@ftisindia/create-app 0.1.6 → 0.2.0

package/template/src/modules/forms/infrastructure/stores/prisma-outbox.store.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto';
 import { Injectable } from '@nestjs/common';
 import { FormOutboxJob as FormOutboxJobRow, Prisma } from '@prisma/client';
 import type {
@@ -20,96 +21,117 @@ export class PrismaOutboxStore implements OutboxStore {
   }
 
   async enqueue(job: OutboxJobInput, tx: EngineTx): Promise<void> {
-    await this.client(tx).formOutboxJob.create({
-      data: {
-        type: job.type,
-        payload: job.payload as unknown as Prisma.InputJsonValue,
-        status: 'PENDING',
-        attempts: 0,
-        maxAttempts: job.maxAttempts ?? 8,
-        idempotencyKey: job.idempotencyKey ?? null,
-        runAfter: job.runAfter ?? null,
-        orgId: job.orgId ?? null,
-        actorUserId: job.actorUserId ?? null,
-        originRequestId: job.originRequestId ?? null,
-      },
-    });
-  }
-
-  /**
-   * Claims due PENDING jobs and stale PROCESSING jobs by selecting candidates
-   * and flipping them to PROCESSING with a guarded updateMany. This is safe
-   * for the template's single in-process dispatcher; multi-instance
-   * deployments should move the claim to `SELECT ... FOR UPDATE SKIP LOCKED`.
-   */
-  async claimDue(now: Date, batchSize: number): Promise<OutboxJobRecord[]> {
-    const staleProcessingBefore = new Date(now.getTime() - PROCESSING_LEASE_MS);
-    const claimableWhere: Prisma.FormOutboxJobWhereInput = {
-      OR: [
+    await this.client(tx).formOutboxJob.createMany({
+      data: [
         {
+          type: job.type,
+          payload: job.payload as unknown as Prisma.InputJsonValue,
           status: 'PENDING',
-          OR: [{ runAfter: null }, { runAfter: { lte: now } }],
-        },
-        {
-          status: 'PROCESSING',
-          updatedAt: { lte: staleProcessingBefore },
+          attempts: 0,
+          maxAttempts: job.maxAttempts ?? 8,
+          idempotencyKey: job.idempotencyKey ?? null,
+          runAfter: job.runAfter ?? null,
+          orgId: job.orgId ?? null,
+          actorUserId: job.actorUserId ?? null,
+          originRequestId: job.originRequestId ?? null,
         },
       ],
-    };
-
-    const due = await this.client().formOutboxJob.findMany({
-      where: claimableWhere,
-      orderBy: { createdAt: 'asc' },
-      take: batchSize,
-      select: { id: true },
+      skipDuplicates: true,
     });
+  }
 
-    if (due.length === 0) {
+  /**
+   * Claims due PENDING jobs and stale PROCESSING jobs with row-level locks so
+   * multiple app instances can poll without double-delivering the same job.
+   */
+  async claimDue(now: Date, batchSize: number): Promise<OutboxJobRecord[]> {
+    if (batchSize <= 0) {
       return [];
     }
 
-    const ids = due.map((row) => row.id);
-
-    await this.client().formOutboxJob.updateMany({
-      where: { id: { in: ids }, ...claimableWhere },
-      data: { status: 'PROCESSING' },
-    });
-
-    const claimed = await this.client().formOutboxJob.findMany({
-      where: { id: { in: ids }, status: 'PROCESSING' },
-    });
+    const staleProcessingBefore = new Date(now.getTime() - PROCESSING_LEASE_MS);
+    const claimedBy = randomUUID();
+    const claimed = await this.prisma.$transaction(
+      (tx) =>
+        tx.$queryRaw<FormOutboxJobRow[]>`
+        WITH due AS (
+          SELECT "id"
+          FROM "FormOutboxJob"
+          WHERE (
+            "status" = 'PENDING'
+            AND ("runAfter" IS NULL OR "runAfter" <= (${now}::timestamptz AT TIME ZONE 'UTC'))
+          )
+          OR (
+            "status" = 'PROCESSING'
+            AND "updatedAt" <= (${staleProcessingBefore}::timestamptz AT TIME ZONE 'UTC')
+          )
+          ORDER BY "createdAt" ASC
+          LIMIT ${batchSize}
+          FOR UPDATE SKIP LOCKED
+        )
+        UPDATE "FormOutboxJob" AS job
+        SET "status" = 'PROCESSING',
+            "claimedBy" = ${claimedBy},
+            "updatedAt" = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
+        FROM due
+        WHERE job."id" = due."id"
+        RETURNING job.*
+      `,
+    );
 
     return claimed.map((row) => this.toRecord(row));
   }
 
-  async markDone(id: string): Promise<void> {
-    await this.client().formOutboxJob.update({
-      where: { id },
-      data: { status: 'DONE' },
+  async markDone(id: string, claimedBy: string): Promise<boolean> {
+    const result = await this.client().formOutboxJob.updateMany({
+      where: { id, status: 'PROCESSING', claimedBy },
+      data: { status: 'DONE', claimedBy: null },
     });
+    return result.count === 1;
   }
 
-  async markRetry(id: string, attempts: number, runAfter: Date, lastError: string): Promise<void> {
-    await this.client().formOutboxJob.update({
-      where: { id },
+  async markRetry(
+    id: string,
+    claimedBy: string,
+    attempts: number,
+    runAfter: Date,
+    lastError: string,
+  ): Promise<boolean> {
+    const result = await this.client().formOutboxJob.updateMany({
+      where: { id, status: 'PROCESSING', claimedBy },
       data: {
         status: 'PENDING',
+        claimedBy: null,
         attempts,
         runAfter,
         lastError,
       },
     });
+    return result.count === 1;
   }
 
-  async markFailed(id: string, attempts: number, lastError: string): Promise<void> {
-    await this.client().formOutboxJob.update({
-      where: { id },
+  async markFailed(id: string, claimedBy: string, attempts: number, lastError: string): Promise<boolean> {
+    const result = await this.client().formOutboxJob.updateMany({
+      where: { id, status: 'PROCESSING', claimedBy },
       data: {
         status: 'FAILED',
+        claimedBy: null,
         attempts,
         lastError,
       },
     });
+    return result.count === 1;
+  }
+
+  async touchProcessing(id: string, claimedBy: string): Promise<boolean> {
+    const touched = await this.prisma.$executeRaw`
+      UPDATE "FormOutboxJob"
+      SET "updatedAt" = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
+      WHERE "id" = ${id}
+        AND "status" = 'PROCESSING'
+        AND "claimedBy" = ${claimedBy}
+    `;
+    return touched === 1;
   }
 
   private toRecord(row: FormOutboxJobRow): OutboxJobRecord {
@@ -118,6 +140,7 @@ export class PrismaOutboxStore implements OutboxStore {
       type: row.type,
       payload: row.payload as unknown as Record<string, unknown>,
       status: row.status as OutboxStatus,
+      claimedBy: row.claimedBy,
       attempts: row.attempts,
       maxAttempts: row.maxAttempts,
       idempotencyKey: row.idempotencyKey,

package/template/src/modules/reports/application/services/reports-export-dispatcher.service.ts

@@ -8,8 +8,11 @@ import { ConfigService } from '@nestjs/config';
 import { ReportExportService } from '@ftisindia/report-builder';
 import { PrismaService } from '../../../../database/prisma/prisma.service';
 import { RequestContextService } from '../../../request-context/application/services/request-context.service';
+import { LocalDiskReportExportStorage } from '../../infrastructure/storage/local-disk-export-storage.adapter';
 import { RequestReportsContext } from '../../infrastructure/request-reports-context';
 
+const RETENTION_BATCH_SIZE = 100;
+
 /**
  * Reports-OWNED async export worker (report design §9). Instead of riding the
  * forms transactional outbox, it polls the engine's own `ReportExportJob`
@@ -25,12 +28,15 @@ import { RequestReportsContext } from '../../infrastructure/request-reports-cont
 export class ReportsExportDispatcherService implements OnApplicationBootstrap, OnModuleDestroy {
   private readonly logger = new Logger('ReportsExportWorker');
   private timer?: NodeJS.Timeout;
+  private retentionTimer?: NodeJS.Timeout;
   private ticking = false;
+  private sweepingRetention = false;
 
   constructor(
     private readonly config: ConfigService,
     private readonly prisma: PrismaService,
     private readonly exports: ReportExportService,
+    private readonly storage: LocalDiskReportExportStorage,
     private readonly requestContext: RequestContextService,
     private readonly reportsContext: RequestReportsContext,
   ) {}
@@ -44,11 +50,22 @@ export class ReportsExportDispatcherService implements OnApplicationBootstrap, O
       void this.tick();
     }, pollMs);
     this.timer.unref?.();
+
+    const retentionSweepMs = this.config.get<number>('reports.exportRetentionSweepMs') ?? 3_600_000;
+    this.retentionTimer = setInterval(() => {
+      void this.sweepRetention();
+    }, retentionSweepMs);
+    this.retentionTimer.unref?.();
   }
 
   onModuleDestroy(): void {
     if (this.timer) {
       clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    if (this.retentionTimer) {
+      clearInterval(this.retentionTimer);
+      this.retentionTimer = undefined;
     }
   }
 
@@ -103,6 +120,50 @@ export class ReportsExportDispatcherService implements OnApplicationBootstrap, O
     return { claimed, done, failed };
   }
 
+  /** Delete expired local export files and clear their job fileId. */
+  async cleanupExpiredFiles(now = new Date()): Promise<number> {
+    const retentionDays = this.config.get<number>('reports.exportRetentionDays') ?? 7;
+    if (retentionDays <= 0) {
+      return 0;
+    }
+
+    const cutoff = new Date(now.getTime() - retentionDays * 24 * 60 * 60 * 1000);
+    const jobs = await this.prisma.reportExportJob.findMany({
+      where: {
+        status: 'DONE',
+        fileId: { not: null },
+        updatedAt: { lt: cutoff },
+      },
+      orderBy: { updatedAt: 'asc' },
+      take: RETENTION_BATCH_SIZE,
+      select: { id: true, orgId: true, fileId: true },
+    });
+
+    let removed = 0;
+    for (const job of jobs) {
+      if (job.fileId === null) {
+        continue;
+      }
+      try {
+        await this.storage.delete(job.fileId);
+        const result = await this.prisma.reportExportJob.updateMany({
+          where: { id: job.id, orgId: job.orgId, fileId: job.fileId },
+          data: {
+            fileId: null,
+            error: `Export file expired after ${retentionDays} day(s).`,
+          },
+        });
+        removed += result.count;
+      } catch (error) {
+        this.logger.warn(
+          `Failed to expire export file for job ${job.id}: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+    }
+
+    return removed;
+  }
+
   private async tick(): Promise<void> {
     if (this.ticking) {
       return;
@@ -121,4 +182,24 @@ export class ReportsExportDispatcherService implements OnApplicationBootstrap, O
       this.ticking = false;
     }
   }
+
+  private async sweepRetention(): Promise<void> {
+    if (this.sweepingRetention) {
+      return;
+    }
+    this.sweepingRetention = true;
+    try {
+      const removed = await this.cleanupExpiredFiles();
+      if (removed > 0) {
+        this.logger.log(`Expired ${removed} report export file(s).`);
+      }
+    } catch (error) {
+      this.logger.error(
+        'Export retention sweep failed.',
+        error instanceof Error ? error.stack : String(error),
+      );
+    } finally {
+      this.sweepingRetention = false;
+    }
+  }
 }

package/template/src/modules/reports/application/services/reports-exports.service.ts

@@ -61,12 +61,16 @@ export class ReportsExportsService {
     const job = await withReportsErrorMapping(() =>
       this.engine.getJob(orgId, jobId, this.reportsContext),
     );
-    if (job.status !== 'DONE' || job.fileId === null) {
+    if (job.status !== 'DONE') {
       throw new NotFoundException('The export is not ready for download.');
     }
+    const fileId = job.fileId;
+    if (fileId === null) {
+      throw new NotFoundException('The export file is no longer available.');
+    }
     const format = job.spec.format;
     return {
-      stream: this.storage.read(job.fileId),
+      stream: this.storage.read(fileId),
       fileName: `${job.reportKey}-v${job.reportVersion}.${format}`,
       mimeType: FORMAT_MIME[format] ?? 'application/octet-stream',
     };

package/template/src/modules/reports/infrastructure/storage/local-disk-export-storage.adapter.ts

@@ -1,25 +1,25 @@
-import { createReadStream } from 'node:fs';
-import { mkdir, writeFile } from 'node:fs/promises';
-import { dirname, join, resolve, sep } from 'node:path';
 import { randomUUID } from 'node:crypto';
-import type { Readable } from 'node:stream';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { mkdir, rm } from 'node:fs/promises';
+import { dirname, join, resolve, sep } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import type { ExportFileSink } from '@ftisindia/report-builder';
 
 /**
- * Reports-OWNED export file storage (report design §9). The reports module
- * keeps its own org-scoped export directory so it does not depend on the form
- * builder's UploadedFile machinery — the engine core's standalone guarantee
- * (§3.2) extends to the glue: ReportsModule needs no FormsModule.
- *
- * Files land under `<exportStorageDir>/<orgId>/<uuid>.<ext>`; the returned
- * `fileId` is that storage key, stored on the ReportExportJob row and used by
- * the download route. The key is server-generated (no user input), and reads
- * are guarded against escaping the configured directory.
+ * Reports-owned export file storage. Files land under
+ * `<exportStorageDir>/<orgId>/<uuid>.<ext>`; the returned fileId is that
+ * storage key, stored on the ReportExportJob row and used by the download
+ * route. The key is server-generated, and reads/deletes are guarded against
+ * escaping the configured directory.
  *
- * Swap this for an S3/GCS-backed ExportFileSink without touching the engine or
- * the rest of the glue.
+ * Operational boundary: this local adapter is correct for a single app
+ * instance or for multiple instances sharing the same mounted volume. A
+ * horizontally scaled deployment without shared storage must rebind this
+ * provider to an object-storage adapter so any node can download a file
+ * written by any worker.
  */
 @Injectable()
 export class LocalDiskReportExportStorage implements ExportFileSink {
@@ -33,30 +33,35 @@ export class LocalDiskReportExportStorage implements ExportFileSink {
     opts: { orgId: string; ownerId: string; fileName: string; mimeType: string },
     chunks: AsyncIterable<Uint8Array>,
   ): Promise<{ fileId: string; size: number }> {
-    // Collected into one buffer before writing. Bounded by the engine's export
-    // caps (row limits + the snapshot duration bound, §9) long before memory
-    // becomes a concern.
-    const collected: Buffer[] = [];
-    for await (const chunk of chunks) {
-      collected.push(Buffer.from(chunk));
-    }
-    const bytes = Buffer.concat(collected);
-
     const safeOrg = sanitizeSegment(opts.orgId);
     const storageKey = `${safeOrg}/${randomUUID()}${extensionOf(opts.fileName)}`;
     const absolute = this.resolveKey(storageKey);
     await mkdir(dirname(absolute), { recursive: true });
-    await writeFile(absolute, bytes);
 
-    return { fileId: storageKey, size: bytes.byteLength };
+    let size = 0;
+    try {
+      await pipeline(
+        Readable.from(countingChunks(chunks, (byteLength) => {
+          size += byteLength;
+        })),
+        createWriteStream(absolute),
+      );
+    } catch (error) {
+      await rm(absolute, { force: true });
+      throw error;
+    }
+
+    return { fileId: storageKey, size };
   }
 
-  /** A readable stream for the download route. Guarded against path traversal. */
   read(storageKey: string): Readable {
     return createReadStream(this.resolveKey(storageKey));
   }
 
-  /** Resolve a storage key inside the base dir, rejecting any escape. */
+  async delete(storageKey: string): Promise<void> {
+    await rm(this.resolveKey(storageKey), { force: true });
+  }
+
   private resolveKey(storageKey: string): string {
     const absolute = resolve(join(this.baseDir, storageKey));
     if (absolute !== this.baseDir && !absolute.startsWith(this.baseDir + sep)) {
@@ -66,14 +71,22 @@ export class LocalDiskReportExportStorage implements ExportFileSink {
   }
 }
 
-/** Keep path segments to a safe alphabet (uuids / org ids are already safe). */
 function sanitizeSegment(segment: string): string {
   return segment.replace(/[^A-Za-z0-9_-]/g, '_');
 }
 
-/** Server-controlled extension: a short alphanumeric suffix, or none. */
 function extensionOf(fileName: string): string {
   const dot = fileName.lastIndexOf('.');
   const extension = dot >= 0 ? fileName.slice(dot) : '';
   return /^\.[A-Za-z0-9]+$/.test(extension) ? extension.toLowerCase() : '';
 }
+
+async function* countingChunks(
+  chunks: AsyncIterable<Uint8Array>,
+  onChunk: (byteLength: number) => void,
+): AsyncGenerator<Uint8Array> {
+  for await (const chunk of chunks) {
+    onChunk(chunk.byteLength);
+    yield chunk;
+  }
+}

package/template/test/forms-captcha.e2e-spec.ts

@@ -0,0 +1,163 @@
+import { INestApplication, ValidationPipe } from '@nestjs/common';
+import { Test } from '@nestjs/testing';
+import type { CaptchaVerifier } from '@ftisindia/form-builder';
+import request from 'supertest';
+import { HttpExceptionFilter } from '../src/common/filters/http-exception.filter';
+import { FORMS_CAPTCHA_VERIFIER } from '../src/modules/forms/forms.tokens';
+
+describe('Forms captcha seam (e2e)', () => {
+  let app: INestApplication;
+  let owner: { accessToken: string; orgId: string };
+  let formKey: string;
+  const seenTokens: string[] = [];
+
+  beforeAll(async () => {
+    process.env.DATABASE_URL = process.env.TEST_DATABASE_URL ?? process.env.DATABASE_URL;
+    process.env.JWT_SECRET ||= 'test-jwt-secret-at-least-16-chars';
+    process.env.AUTH_GOOGLE_ENABLED ||= 'false';
+    process.env.AUTH_EMAIL_PASSWORD_ENABLED ||= 'true';
+    process.env.NODE_ENV = 'test';
+    process.env.FORMS_OUTBOX_ENABLED = 'false';
+
+    const captcha: CaptchaVerifier = {
+      verify: (token) => {
+        seenTokens.push(token);
+        return Promise.resolve(token === 'valid-token');
+      },
+    };
+
+    const { AppModule } = await import('../src/app.module');
+    const moduleRef = await Test.createTestingModule({ imports: [AppModule] })
+      .overrideProvider(FORMS_CAPTCHA_VERIFIER)
+      .useValue(captcha)
+      .compile();
+
+    app = moduleRef.createNestApplication();
+    app.useGlobalFilters(new HttpExceptionFilter());
+    app.useGlobalPipes(
+      new ValidationPipe({
+        forbidNonWhitelisted: true,
+        transform: true,
+        whitelist: true,
+      }),
+    );
+    await app.init();
+
+    owner = await createUserAndOrg('captcha-owner');
+    formKey = await publishDefinition(publicCaptchaDefinition(uniqueFormKey('captcha')));
+  });
+
+  afterAll(async () => {
+    await app.close();
+  });
+
+  it('rejects missing or invalid tokens and accepts a valid captcha token', async () => {
+    const payload = {
+      data: {
+        fullName: 'Captcha User',
+        email: 'captcha@example.com',
+        ticketType: 'standard',
+      },
+    };
+
+    await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${formKey}/submissions`)
+      .send(payload)
+      .expect(422);
+
+    await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${formKey}/submissions`)
+      .send({ ...payload, captchaToken: 'invalid-token' })
+      .expect(422);
+
+    const submitted = await request(app.getHttpServer())
+      .post(`/public/organisations/${owner.orgId}/forms/${formKey}/submissions`)
+      .send({ ...payload, captchaToken: 'valid-token' })
+      .expect(200);
+
+    expect(submitted.body.submissionId).toBeTruthy();
+    expect(seenTokens).toEqual(['invalid-token', 'valid-token']);
+  });
+
+  function publicCaptchaDefinition(key: string): Record<string, unknown> {
+    return {
+      key,
+      version: 1,
+      title: 'Captcha public form',
+      settings: { access: 'public', captcha: true },
+      fields: [
+        {
+          type: 'text',
+          name: 'fullName',
+          validators: { required: true, maxLength: 120 },
+        },
+        {
+          type: 'email',
+          name: 'email',
+          validators: { required: true },
+        },
+        {
+          type: 'select',
+          name: 'ticketType',
+          validators: { required: true, options: ['standard', 'student', 'vip'] },
+        },
+      ],
+      actions: {
+        submit: ['validateAll', 'persist'],
+        saveDraft: ['persistDraft'],
+      },
+    };
+  }
+
+  async function publishDefinition(definition: Record<string, unknown>): Promise<string> {
+    const created = await request(app.getHttpServer())
+      .post(`/organisations/${owner.orgId}/forms`)
+      .set('Authorization', `Bearer ${owner.accessToken}`)
+      .send({ definition })
+      .expect(201);
+
+    await request(app.getHttpServer())
+      .post(
+        `/organisations/${owner.orgId}/forms/${definition.key as string}/versions/${created.body.version as number}/publish`,
+      )
+      .set('Authorization', `Bearer ${owner.accessToken}`)
+      .expect(200);
+
+    return definition.key as string;
+  }
+
+  async function createUserAndOrg(label: string) {
+    const suffix = uniqueSuffix();
+    const signup = await request(app.getHttpServer())
+      .post('/auth/signup')
+      .send({
+        email: `${label}-${suffix}@example.com`,
+        password: 'test-password-123',
+        displayName: label,
+      })
+      .expect(201);
+
+    const accessToken = signup.body.accessToken as string;
+    const org = await request(app.getHttpServer())
+      .post('/organisations')
+      .set('Authorization', `Bearer ${accessToken}`)
+      .send({
+        name: `${label} ${suffix}`,
+        slug: `${label}-${suffix}`,
+      })
+      .expect(201);
+
+    return {
+      accessToken,
+      orgId: org.body.organisation.id as string,
+    };
+  }
+
+  function uniqueSuffix() {
+    return `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  }
+
+  function uniqueFormKey(label: string) {
+    return `${label}-${uniqueSuffix()}`.toLowerCase();
+  }
+});

package/template/test/forms-files.e2e-spec.ts

@@ -176,7 +176,7 @@ describe('Forms file uploads (e2e)', () => {
 
     await prisma.uploadedFile.update({
       where: { id: fileId },
-      data: { createdAt: new Date(Date.now() - 48 * 60 * 60 * 1000) },
+      data: { createdAt: new Date(Date.now() - 365 * 24 * 60 * 60 * 1000) },
     });
 
     const removed = await app.get(FileGcService).runOnce();