@ftisindia/create-app 0.1.6 → 0.3.0

package/template/src/modules/reports/application/services/reports-export-dispatcher.service.ts

@@ -8,8 +8,11 @@ import { ConfigService } from '@nestjs/config';
 import { ReportExportService } from '@ftisindia/report-builder';
 import { PrismaService } from '../../../../database/prisma/prisma.service';
 import { RequestContextService } from '../../../request-context/application/services/request-context.service';
+import { LocalDiskReportExportStorage } from '../../infrastructure/storage/local-disk-export-storage.adapter';
 import { RequestReportsContext } from '../../infrastructure/request-reports-context';
 
+const RETENTION_BATCH_SIZE = 100;
+
 /**
  * Reports-OWNED async export worker (report design §9). Instead of riding the
  * forms transactional outbox, it polls the engine's own `ReportExportJob`
@@ -25,12 +28,15 @@ import { RequestReportsContext } from '../../infrastructure/request-reports-cont
 export class ReportsExportDispatcherService implements OnApplicationBootstrap, OnModuleDestroy {
   private readonly logger = new Logger('ReportsExportWorker');
   private timer?: NodeJS.Timeout;
+  private retentionTimer?: NodeJS.Timeout;
   private ticking = false;
+  private sweepingRetention = false;
 
   constructor(
     private readonly config: ConfigService,
     private readonly prisma: PrismaService,
     private readonly exports: ReportExportService,
+    private readonly storage: LocalDiskReportExportStorage,
     private readonly requestContext: RequestContextService,
     private readonly reportsContext: RequestReportsContext,
   ) {}
@@ -44,11 +50,22 @@ export class ReportsExportDispatcherService implements OnApplicationBootstrap, O
       void this.tick();
     }, pollMs);
     this.timer.unref?.();
+
+    const retentionSweepMs = this.config.get<number>('reports.exportRetentionSweepMs') ?? 3_600_000;
+    this.retentionTimer = setInterval(() => {
+      void this.sweepRetention();
+    }, retentionSweepMs);
+    this.retentionTimer.unref?.();
   }
 
   onModuleDestroy(): void {
     if (this.timer) {
       clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    if (this.retentionTimer) {
+      clearInterval(this.retentionTimer);
+      this.retentionTimer = undefined;
     }
   }
 
@@ -103,6 +120,50 @@ export class ReportsExportDispatcherService implements OnApplicationBootstrap, O
     return { claimed, done, failed };
   }
 
+  /** Delete expired local export files and clear their job fileId. */
+  async cleanupExpiredFiles(now = new Date()): Promise<number> {
+    const retentionDays = this.config.get<number>('reports.exportRetentionDays') ?? 7;
+    if (retentionDays <= 0) {
+      return 0;
+    }
+
+    const cutoff = new Date(now.getTime() - retentionDays * 24 * 60 * 60 * 1000);
+    const jobs = await this.prisma.reportExportJob.findMany({
+      where: {
+        status: 'DONE',
+        fileId: { not: null },
+        updatedAt: { lt: cutoff },
+      },
+      orderBy: { updatedAt: 'asc' },
+      take: RETENTION_BATCH_SIZE,
+      select: { id: true, orgId: true, fileId: true },
+    });
+
+    let removed = 0;
+    for (const job of jobs) {
+      if (job.fileId === null) {
+        continue;
+      }
+      try {
+        await this.storage.delete(job.fileId);
+        const result = await this.prisma.reportExportJob.updateMany({
+          where: { id: job.id, orgId: job.orgId, fileId: job.fileId },
+          data: {
+            fileId: null,
+            error: `Export file expired after ${retentionDays} day(s).`,
+          },
+        });
+        removed += result.count;
+      } catch (error) {
+        this.logger.warn(
+          `Failed to expire export file for job ${job.id}: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+    }
+
+    return removed;
+  }
+
   private async tick(): Promise<void> {
     if (this.ticking) {
       return;
@@ -121,4 +182,24 @@ export class ReportsExportDispatcherService implements OnApplicationBootstrap, O
       this.ticking = false;
     }
   }
+
+  private async sweepRetention(): Promise<void> {
+    if (this.sweepingRetention) {
+      return;
+    }
+    this.sweepingRetention = true;
+    try {
+      const removed = await this.cleanupExpiredFiles();
+      if (removed > 0) {
+        this.logger.log(`Expired ${removed} report export file(s).`);
+      }
+    } catch (error) {
+      this.logger.error(
+        'Export retention sweep failed.',
+        error instanceof Error ? error.stack : String(error),
+      );
+    } finally {
+      this.sweepingRetention = false;
+    }
+  }
 }

package/template/src/modules/reports/application/services/reports-exports.service.ts

@@ -61,12 +61,16 @@ export class ReportsExportsService {
     const job = await withReportsErrorMapping(() =>
       this.engine.getJob(orgId, jobId, this.reportsContext),
     );
-    if (job.status !== 'DONE' || job.fileId === null) {
+    if (job.status !== 'DONE') {
       throw new NotFoundException('The export is not ready for download.');
     }
+    const fileId = job.fileId;
+    if (fileId === null) {
+      throw new NotFoundException('The export file is no longer available.');
+    }
     const format = job.spec.format;
     return {
-      stream: this.storage.read(job.fileId),
+      stream: this.storage.read(fileId),
       fileName: `${job.reportKey}-v${job.reportVersion}.${format}`,
       mimeType: FORMAT_MIME[format] ?? 'application/octet-stream',
     };

package/template/src/modules/reports/infrastructure/storage/local-disk-export-storage.adapter.ts

@@ -1,25 +1,25 @@
-import { createReadStream } from 'node:fs';
-import { mkdir, writeFile } from 'node:fs/promises';
-import { dirname, join, resolve, sep } from 'node:path';
 import { randomUUID } from 'node:crypto';
-import type { Readable } from 'node:stream';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { mkdir, rm } from 'node:fs/promises';
+import { dirname, join, resolve, sep } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import type { ExportFileSink } from '@ftisindia/report-builder';
 
 /**
- * Reports-OWNED export file storage (report design §9). The reports module
- * keeps its own org-scoped export directory so it does not depend on the form
- * builder's UploadedFile machinery — the engine core's standalone guarantee
- * (§3.2) extends to the glue: ReportsModule needs no FormsModule.
- *
- * Files land under `<exportStorageDir>/<orgId>/<uuid>.<ext>`; the returned
- * `fileId` is that storage key, stored on the ReportExportJob row and used by
- * the download route. The key is server-generated (no user input), and reads
- * are guarded against escaping the configured directory.
+ * Reports-owned export file storage. Files land under
+ * `<exportStorageDir>/<orgId>/<uuid>.<ext>`; the returned fileId is that
+ * storage key, stored on the ReportExportJob row and used by the download
+ * route. The key is server-generated, and reads/deletes are guarded against
+ * escaping the configured directory.
  *
- * Swap this for an S3/GCS-backed ExportFileSink without touching the engine or
- * the rest of the glue.
+ * Operational boundary: this local adapter is correct for a single app
+ * instance or for multiple instances sharing the same mounted volume. A
+ * horizontally scaled deployment without shared storage must rebind this
+ * provider to an object-storage adapter so any node can download a file
+ * written by any worker.
  */
 @Injectable()
 export class LocalDiskReportExportStorage implements ExportFileSink {
@@ -33,30 +33,35 @@ export class LocalDiskReportExportStorage implements ExportFileSink {
     opts: { orgId: string; ownerId: string; fileName: string; mimeType: string },
     chunks: AsyncIterable<Uint8Array>,
   ): Promise<{ fileId: string; size: number }> {
-    // Collected into one buffer before writing. Bounded by the engine's export
-    // caps (row limits + the snapshot duration bound, §9) long before memory
-    // becomes a concern.
-    const collected: Buffer[] = [];
-    for await (const chunk of chunks) {
-      collected.push(Buffer.from(chunk));
-    }
-    const bytes = Buffer.concat(collected);
-
     const safeOrg = sanitizeSegment(opts.orgId);
     const storageKey = `${safeOrg}/${randomUUID()}${extensionOf(opts.fileName)}`;
     const absolute = this.resolveKey(storageKey);
     await mkdir(dirname(absolute), { recursive: true });
-    await writeFile(absolute, bytes);
 
-    return { fileId: storageKey, size: bytes.byteLength };
+    let size = 0;
+    try {
+      await pipeline(
+        Readable.from(countingChunks(chunks, (byteLength) => {
+          size += byteLength;
+        })),
+        createWriteStream(absolute),
+      );
+    } catch (error) {
+      await rm(absolute, { force: true });
+      throw error;
+    }
+
+    return { fileId: storageKey, size };
   }
 
-  /** A readable stream for the download route. Guarded against path traversal. */
   read(storageKey: string): Readable {
     return createReadStream(this.resolveKey(storageKey));
   }
 
-  /** Resolve a storage key inside the base dir, rejecting any escape. */
+  async delete(storageKey: string): Promise<void> {
+    await rm(this.resolveKey(storageKey), { force: true });
+  }
+
   private resolveKey(storageKey: string): string {
     const absolute = resolve(join(this.baseDir, storageKey));
     if (absolute !== this.baseDir && !absolute.startsWith(this.baseDir + sep)) {
@@ -66,14 +71,22 @@ export class LocalDiskReportExportStorage implements ExportFileSink {
   }
 }
 
-/** Keep path segments to a safe alphabet (uuids / org ids are already safe). */
 function sanitizeSegment(segment: string): string {
   return segment.replace(/[^A-Za-z0-9_-]/g, '_');
 }
 
-/** Server-controlled extension: a short alphanumeric suffix, or none. */
 function extensionOf(fileName: string): string {
   const dot = fileName.lastIndexOf('.');
   const extension = dot >= 0 ? fileName.slice(dot) : '';
   return /^\.[A-Za-z0-9]+$/.test(extension) ? extension.toLowerCase() : '';
 }
+
+async function* countingChunks(
+  chunks: AsyncIterable<Uint8Array>,
+  onChunk: (byteLength: number) => void,
+): AsyncGenerator<Uint8Array> {
+  for await (const chunk of chunks) {
+    onChunk(chunk.byteLength);
+    yield chunk;
+  }
+}

package/template/src/modules/settings/types/setting-definitions.ts

@@ -57,6 +57,10 @@ export const settingDefinitions = {
     defaultValue: 50,
     parse: parsePositiveInt('forms.maxSubmissionsPerIpPerDay', 100000),
   },
+  'forms.maxPublicUploadsPerIpPerDay': {
+    defaultValue: 100,
+    parse: parsePositiveInt('forms.maxPublicUploadsPerIpPerDay', 100000),
+  },
   // SSRF gate for definition webhooks: empty list = webhooks disabled until
   // an admin explicitly allows destination hosts (changes are audited).
   'forms.webhookAllowedHosts': {

package/template/test/forms-captcha.e2e-spec.ts

@@ -0,0 +1,163 @@
+import { INestApplication, ValidationPipe } from '@nestjs/common';
+import { Test } from '@nestjs/testing';
+import type { CaptchaVerifier } from '@ftisindia/form-builder';
+import request from 'supertest';
+import { HttpExceptionFilter } from '../src/common/filters/http-exception.filter';
+import { FORMS_CAPTCHA_VERIFIER } from '../src/modules/forms/forms.tokens';
+
+describe('Forms captcha seam (e2e)', () => {
+  let app: INestApplication;
+  let owner: { accessToken: string; orgId: string };
+  let formKey: string;
+  const seenTokens: string[] = [];
+
+  beforeAll(async () => {
+    process.env.DATABASE_URL = process.env.TEST_DATABASE_URL ?? process.env.DATABASE_URL;
+    process.env.JWT_SECRET ||= 'test-jwt-secret-at-least-16-chars';
+    process.env.AUTH_GOOGLE_ENABLED ||= 'false';
+    process.env.AUTH_EMAIL_PASSWORD_ENABLED ||= 'true';
+    process.env.NODE_ENV = 'test';
+    process.env.FORMS_OUTBOX_ENABLED = 'false';
+
+    const captcha: CaptchaVerifier = {
+      verify: (token) => {
+        seenTokens.push(token);
+        return Promise.resolve(token === 'valid-token');
+      },
+    };
+
+    const { AppModule } = await import('../src/app.module');
+    const moduleRef = await Test.createTestingModule({ imports: [AppModule] })
+      .overrideProvider(FORMS_CAPTCHA_VERIFIER)
+      .useValue(captcha)
+      .compile();
+
+    app = moduleRef.createNestApplication();
+    app.useGlobalFilters(new HttpExceptionFilter());
+    app.useGlobalPipes(
+      new ValidationPipe({
+        forbidNonWhitelisted: true,
+        transform: true,
+        whitelist: true,
+      }),
+    );
+    await app.init();
+
+    owner = await createUserAndOrg('captcha-owner');
+    formKey = await publishDefinition(publicCaptchaDefinition(uniqueFormKey('captcha')));
+  });
+
+  afterAll(async () => {
+    await app.close();
+  });
+
+  it('rejects missing or invalid tokens and accepts a valid captcha token', async () => {
+    const payload = {
+      data: {
+        fullName: 'Captcha User',
+        email: 'captcha@example.com',
+        ticketType: 'standard',
+      },
+    };
+
+    await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${formKey}/submissions`)
+      .send(payload)
+      .expect(422);
+
+    await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${formKey}/submissions`)
+      .send({ ...payload, captchaToken: 'invalid-token' })
+      .expect(422);
+
+    const submitted = await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${formKey}/submissions`)
+      .send({ ...payload, captchaToken: 'valid-token' })
+      .expect(200);
+
+    expect(submitted.body.submissionId).toBeTruthy();
+    expect(seenTokens).toEqual(['invalid-token', 'valid-token']);
+  });
+
+  function publicCaptchaDefinition(key: string): Record<string, unknown> {
+    return {
+      key,
+      version: 1,
+      title: 'Captcha public form',
+      settings: { access: 'public', captcha: true },
+      fields: [
+        {
+          type: 'text',
+          name: 'fullName',
+          validators: { required: true, maxLength: 120 },
+        },
+        {
+          type: 'email',
+          name: 'email',
+          validators: { required: true },
+        },
+        {
+          type: 'select',
+          name: 'ticketType',
+          validators: { required: true, options: ['standard', 'student', 'vip'] },
+        },
+      ],
+      actions: {
+        submit: ['validateAll', 'persist'],
+        saveDraft: ['persistDraft'],
+      },
+    };
+  }
+
+  async function publishDefinition(definition: Record<string, unknown>): Promise<string> {
+    const created = await request(app.getHttpServer())
+      .post(`/organisations/${owner.orgId}/forms`)
+      .set('Authorization', `Bearer ${owner.accessToken}`)
+      .send({ definition })
+      .expect(201);
+
+    await request(app.getHttpServer())
+      .post(
+        `/organisations/${owner.orgId}/forms/${definition.key as string}/versions/${created.body.version as number}/publish`,
+      )
+      .set('Authorization', `Bearer ${owner.accessToken}`)
+      .expect(200);
+
+    return definition.key as string;
+  }
+
+  async function createUserAndOrg(label: string) {
+    const suffix = uniqueSuffix();
+    const signup = await request(app.getHttpServer())
+      .post('/auth/signup')
+      .send({
+        email: `${label}-${suffix}@example.com`,
+        password: 'test-password-123',
+        displayName: label,
+      })
+      .expect(201);
+
+    const accessToken = signup.body.accessToken as string;
+    const org = await request(app.getHttpServer())
+      .post('/organisations')
+      .set('Authorization', `Bearer ${accessToken}`)
+      .send({
+        name: `${label} ${suffix}`,
+        slug: `${label}-${suffix}`,
+      })
+      .expect(201);
+
+    return {
+      accessToken,
+      orgId: org.body.organisation.id as string,
+    };
+  }
+
+  function uniqueSuffix() {
+    return `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  }
+
+  function uniqueFormKey(label: string) {
+    return `${label}-${uniqueSuffix()}`.toLowerCase();
+  }
+});

package/template/test/forms-files.e2e-spec.ts

@@ -1,6 +1,5 @@
 import { INestApplication, ValidationPipe } from '@nestjs/common';
 import { Test } from '@nestjs/testing';
-import { DataSourceRegistry } from '@ftisindia/form-builder';
 import request from 'supertest';
 import { HttpExceptionFilter } from '../src/common/filters/http-exception.filter';
 import { PrismaService } from '../src/database/prisma/prisma.service';
@@ -43,16 +42,6 @@ describe('Forms file uploads (e2e)', () => {
     await app.init();
     prisma = app.get(PrismaService);
 
-    // The abstract-like definition binds a select to this source; register the
-    // stub BEFORE creating the definition so save-time lint passes.
-    app.get(DataSourceRegistry).register({
-      key: 'conference-tracks',
-      fetch: () => Promise.resolve([
-        { value: 'ai', label: 'Artificial Intelligence' },
-        { value: 'ml', label: 'Machine Learning' },
-      ]),
-    });
-
     owner = await createUserAndOrg('files-owner');
     formKey = await publishDefinition(abstractDefinition(uniqueFormKey('abstract')));
   });
@@ -87,7 +76,7 @@ describe('Forms file uploads (e2e)', () => {
     const draft = await request(app.getHttpServer())
       .post(`/organisations/${owner.orgId}/forms/${key}/submissions`)
       .set('Authorization', `Bearer ${owner.accessToken}`)
-      .send({ mode: 'draft', data: { title: 'Versioned upload', track: 'ai' } })
+      .send({ mode: 'draft', data: { title: 'Versioned upload', track: 'clinical-research' } })
       .expect(200);
 
     const createdV2 = await request(app.getHttpServer())
@@ -136,7 +125,7 @@ describe('Forms file uploads (e2e)', () => {
       .set('Authorization', `Bearer ${owner.accessToken}`)
       .send({
         mode: 'submit',
-        data: { title: 'On Linking Files', track: 'ai', manuscript: { fileId } },
+        data: { title: 'On Linking Files', track: 'clinical-research', manuscript: { fileId } },
       })
       .expect(200);
 
@@ -159,7 +148,7 @@ describe('Forms file uploads (e2e)', () => {
       .set('Authorization', `Bearer ${member.accessToken}`)
       .send({
         mode: 'submit',
-        data: { title: 'Stolen Reference', track: 'ml', manuscript: { fileId } },
+        data: { title: 'Stolen Reference', track: 'public-health', manuscript: { fileId } },
       })
       .expect(422);
 
@@ -176,7 +165,7 @@ describe('Forms file uploads (e2e)', () => {
 
     await prisma.uploadedFile.update({
       where: { id: fileId },
-      data: { createdAt: new Date(Date.now() - 48 * 60 * 60 * 1000) },
+      data: { createdAt: new Date(Date.now() - 365 * 24 * 60 * 60 * 1000) },
     });
 
     const removed = await app.get(FileGcService).runOnce();
@@ -186,15 +175,48 @@ describe('Forms file uploads (e2e)', () => {
     expect(row).toBeNull();
   });
 
-  it('refuses to flip a definition with file fields to public access', async () => {
-    const response = await request(app.getHttpServer())
-      .post(`/organisations/${owner.orgId}/forms/${formKey}/versions/1/public-access`)
+  it('accepts public uploads only with the returned claim token', async () => {
+    const key = await publishDefinition(abstractDefinition(uniqueFormKey('public-file')));
+    await request(app.getHttpServer())
+      .post(`/organisations/${owner.orgId}/forms/${key}/versions/1/public-access`)
       .set('Authorization', `Bearer ${owner.accessToken}`)
       .send({ access: 'public' })
+      .expect(200);
+
+    const upload = await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${key}/files`)
+      .query({ field: 'manuscript' })
+      .attach('file', PDF_BYTES, 'public-manuscript.pdf')
+      .expect(200);
+
+    const fileId = upload.body.fileId as string;
+    const uploadToken = upload.body.uploadToken as string;
+    expect(fileId).toBeTruthy();
+    expect(uploadToken).toBeTruthy();
+    expect(upload.body.status).toBe('TEMPORARY');
+
+    const rejected = await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${key}/submissions`)
+      .send({
+        data: { title: 'Missing token', track: 'clinical-research', manuscript: { fileId } },
+      })
       .expect(422);
+    const errors = rejected.body.error.details.errors as Array<{ code: string; path: string }>;
+    expect(errors.some((error) => error.code === 'FILE_REFERENCE')).toBe(true);
+
+    const submitted = await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${key}/submissions`)
+      .send({
+        data: { title: 'Public file', track: 'clinical-research', manuscript: { fileId } },
+        uploadTokens: { [fileId]: uploadToken },
+      })
+      .expect(200);
 
-    const issues = response.body.error.details.issues as Array<{ code: string }>;
-    expect(issues.map((issue) => issue.code)).toContain('PUBLIC_FILE_FIELD');
+    const submissionId = submitted.body.submissionId as string;
+    const row = await prisma.uploadedFile.findUnique({ where: { id: fileId } });
+    expect(row?.ownerId).toBeNull();
+    expect(row?.status).toBe('LINKED');
+    expect(row?.submissionId).toBe(submissionId);
   });
 
   // ── Helpers ────────────────────────────────────────────────────────────────