@ftisindia/create-app 0.1.6 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ftisindia/create-app",
-  "version": "0.1.6",
+  "version": "0.3.0",
   "description": "One-command scaffolder for the Phase 1 NestJS foundation starter.",
   "type": "module",
   "bin": {

package/template/.env.example

@@ -48,6 +48,7 @@ TEST_ORG_SLUG=demo
 # Form builder (@ftisindia/form-builder)
 FORMS_OUTBOX_ENABLED=true
 FORMS_OUTBOX_POLL_MS=5000
+FORMS_OUTBOX_HEARTBEAT_MS=60000
 FORMS_FILE_GC_INTERVAL_MS=3600000
 FORMS_FILE_TEMP_TTL_HOURS=24
 FORMS_MAX_UPLOAD_MB=25
@@ -65,3 +66,5 @@ REPORTS_TOKEN_SECRET=
 REPORTS_EXPORT_WORKER_ENABLED=true
 REPORTS_EXPORT_POLL_MS=5000
 REPORTS_EXPORT_STORAGE_DIR=./var/report-exports
+REPORTS_EXPORT_RETENTION_DAYS=7
+REPORTS_EXPORT_RETENTION_SWEEP_MS=3600000

package/template/_package.json

@@ -10,6 +10,7 @@
     "format": "prettier --write .",
     "format:check": "prettier --check .",
     "forms:push": "ts-node scripts/push-form.ts",
+    "reports:push": "ts-node scripts/push-report.ts",
     "gen:form": "node scripts/gen-form.mjs",
     "gen:module": "node scripts/gen-module.mjs",
     "lint": "eslint .",
@@ -27,14 +28,14 @@
     "setup:ci": "node scripts/setup-local.mjs --yes",
     "test": "jest",
     "test:cov": "jest --coverage",
-    "test:e2e": "node scripts/test-db.mjs && jest --config test/jest-e2e.json --runInBand",
+    "test:e2e": "node scripts/test-db.mjs && node ./node_modules/jest/bin/jest.js --config test/jest-e2e.json --runInBand",
     "test:watch": "jest --watch",
     "db:reset": "prisma migrate reset"
   },
   "dependencies": {
     "@casl/ability": "6.8.1",
-    "@ftisindia/form-builder": "^0.1.0",
-    "@ftisindia/report-builder": "^0.1.0",
+    "@ftisindia/form-builder": "^1.0.0",
+    "@ftisindia/report-builder": "^0.3.0",
     "@nestjs/common": "11.1.24",
     "@nestjs/config": "4.0.4",
     "@nestjs/core": "11.1.24",
@@ -84,6 +85,9 @@
     "typescript-eslint": "8.60.0",
     "typescript": "5.9.3"
   },
+  "overrides": {
+    "js-yaml": "4.2.0"
+  },
   "lint-staged": {
     "*.{js,mjs,ts,json,md,yml,yaml}": "prettier --write",
     "*.{js,mjs,ts}": "eslint --fix"

package/template/docs/FORMS.md

@@ -38,16 +38,25 @@ json-logic rules + data-source membership + file references) → the action
 pipeline: validation actions, then **all transactional actions in ONE
 database transaction** (submission, action logs, audit rows, outbox rows
 commit or roll back together), then post-commit side effects via the
-**transactional outbox** (in-process poller, retries with backoff, parks as
-FAILED after max attempts; delivery writes an audit row). A failed email
-never rolls back a valid submission; a failed transactional step rolls back
-everything.
+**transactional outbox** (in-process poller with Postgres row-lock claiming,
+idempotent enqueue, claim-token fencing, active-job heartbeat, retries with
+backoff, parks as FAILED after max attempts; delivery writes an audit row). A
+failed email never rolls back a valid submission; a failed transactional step
+rolls back everything.
 
 ## Adding behavior (the three registries)
 
 Implement the engine interface, decorate the provider, list it in a module —
 the registry bootstrap finds it at startup:
 
+Definitions can reference a `dataSource.key` only after a matching provider is
+registered. This template ships `ConferenceTracksDataSource` in
+`src/modules/forms/application/services/data-sources/` for the bundled
+`abstract-submission` example (`conference-tracks`). It is intentionally static
+so fresh apps can publish and render the example immediately; replace it with an
+org-scoped Prisma/API lookup before using real production master data. An
+unregistered source fails save/publish lint with `UNKNOWN_DATA_SOURCE`.
+
 ```ts
 import { FormDataSource } from './modules/forms/infrastructure/registry/form-extension.decorators';
 import type { DataSourceDef, DataSourceContext, DataSourceOption } from '@ftisindia/form-builder';
@@ -57,11 +66,16 @@ import type { DataSourceDef, DataSourceContext, DataSourceOption } from '@ftisin
 export class TrackSource implements DataSourceDef {
   key = 'conference-tracks';
   constructor(private readonly prisma: PrismaService) {}
-  async fetch(_params: Record<string, unknown>, ctx: DataSourceContext): Promise<DataSourceOption[]> {
-    return this.prisma.track.findMany({
-      where: { orgId: ctx.orgId, active: true },
-      select: { id: true, name: true },
-    }).then((rows) => rows.map((row) => ({ value: row.id, label: row.name })));
+  async fetch(
+    _params: Record<string, unknown>,
+    ctx: DataSourceContext,
+  ): Promise<DataSourceOption[]> {
+    return this.prisma.track
+      .findMany({
+        where: { orgId: ctx.orgId, active: true },
+        select: { id: true, name: true },
+      })
+      .then((rows) => rows.map((row) => ({ value: row.id, label: row.name })));
   }
 }
 ```
@@ -89,9 +103,51 @@ Flip a published definition with `POST .../public-access { "access": "public" }`
 (requires `forms.managePublicAccess`, writes an audit row). Public routes are
 throttled tighter than the app default; the engine enforces the per-IP/day
 cap (`forms.maxSubmissionsPerIpPerDay` setting, per-form override in the
-definition's `settings`) and the captcha seam (`settings.captcha: true` —
+definition's `settings`) and the captcha seam (`settings.captcha: true` -
 publish fails closed unless a `CaptchaVerifier` is bound to the
-`FORMS_CAPTCHA_VERIFIER` token). Public forms cannot contain file fields.
+`FORMS_CAPTCHA_VERIFIER` token). Public file fields are allowed through the
+public upload endpoint: upload first, then submit `{ fileId }` in `data` and the
+returned token in `uploadTokens[fileId]`. Anonymous upload volume is capped by
+`forms.maxPublicUploadsPerIpPerDay`. Like the public submission cap, it depends on a resolvable request IP; configure proxy/IP handling correctly in deployments behind a reverse proxy.
+
+A captcha-enabled public form keeps failing closed until the app binds a real
+verifier. Example Turnstile-style adapter:
+
+```ts
+import { Injectable, Module } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import type { CaptchaVerifier, FormsContext } from '@ftisindia/form-builder';
+import { FORMS_CAPTCHA_VERIFIER } from './modules/forms/forms.tokens';
+
+@Injectable()
+export class TurnstileCaptchaVerifier implements CaptchaVerifier {
+  constructor(private readonly config: ConfigService) {}
+
+  async verify(token: string, _ctx: FormsContext): Promise<boolean> {
+    const body = new URLSearchParams({
+      secret: this.config.getOrThrow<string>('TURNSTILE_SECRET_KEY'),
+      response: token,
+    });
+    const response = await fetch(
+      'https://challenges.cloudflare.com/turnstile/v0/siteverify',
+      { method: 'POST', body },
+    );
+    const result = (await response.json()) as { success?: boolean };
+    return result.success === true;
+  }
+}
+
+@Module({
+  providers: [
+    { provide: FORMS_CAPTCHA_VERIFIER, useClass: TurnstileCaptchaVerifier },
+  ],
+})
+export class AppCaptchaModule {}
+```
+
+After that binding is present, a definition can opt in with
+`settings.captcha: true`; without the binding, publish fails before the form can
+go public.
 
 ## Files
 
@@ -119,19 +175,32 @@ A definition can declare webhook destinations that fire when a submission is
 - **Reliable by construction:** the job is written to the transactional outbox
   inside the same transaction as the submission, then delivered post-commit
   with retries/backoff (parks as FAILED after max attempts; success writes a
-  `forms.outbox.delivered` audit row). Idempotency key: `<submissionId>:webhook:<index>`.
+  `forms.outbox.delivered` audit row). The Prisma store claims jobs with
+  `FOR UPDATE SKIP LOCKED`, assigns an opaque claim token, heartbeats active
+  `PROCESSING` rows with that token, and guards `DONE`/`PENDING`/`FAILED`
+  transitions with the same token so stale workers cannot settle reclaimed
+  jobs. A duplicate idempotency key is a no-op so retries do not roll back the
+  submission. Idempotency key: `<submissionId>:webhook:<index>`, also sent as
+  `x-forms-idempotency-key` for receiver-side dedupe.
 - **SSRF-gated:** the destination host must be in the org's audited
   `forms.webhookAllowedHosts` typed setting (default empty = webhooks
   disabled), and the URL must be `https` — `http` is allowed for loopback
   hosts only (local development). Enforced at save AND publish lint
-  (`WEBHOOK_HOST_NOT_ALLOWED`, `WEBHOOK_URL_INSECURE`, …).
+  (`WEBHOOK_HOST_NOT_ALLOWED`, `WEBHOOK_URL_INSECURE`, …). Delivery does a
+  fresh DNS resolution, rejects private/reserved/link-local destinations in
+  `NODE_ENV=production`, pins the request to the resolved IP while preserving
+  Host/SNI, and refuses redirects instead of following them.
 - **Payload:** `{ event: 'submitted', orgId, formKey, formVersion, submissionId,
-  occurredAt, data? }` — `data` passes sensitive-field redaction; set
+occurredAt, data? }` — `data` passes sensitive-field redaction; set
   `includeData: false` to send a pure notification.
 - **Signed:** with a `secret`, deliveries carry
-  `x-forms-signature: sha256=<hmac-sha256(rawBody)>`. The secret lives in the
-  definition document (DB) — treat it as a rotatable shared token, not a
-  master credential. Max 5 webhooks per definition.
+  `x-forms-signature: sha256=<hmac-sha256(rawBody)>`. The engine computes the
+  signature before enqueueing; the outbox payload stores the raw body and
+  signature, not the shared secret. Old-format jobs containing `secret` are
+  sanitized to the new `{ rawBody, signature }` payload before delivery. The
+  secret lives only in the definition
+  document (DB) — treat it as a rotatable shared token, not a master
+  credential. Max 5 webhooks per definition.
 
 ## Definitions as code (`gen:form` + `forms:push`)
 
@@ -142,7 +211,7 @@ npm run forms:push -- --file src/modules/forms/definitions/customer-feedback.for
     --org demo --user owner@example.com --publish
 ```
 
-`gen:form` writes `src/modules/forms/definitions/<key>.form.json` and
+Example files under `src/modules/forms/examples/` are reference fixtures; copy or scaffold into `src/modules/forms/definitions/` before pushing. `gen:form` writes `src/modules/forms/definitions/<key>.form.json` and
 `test/<key>.form.spec.ts` (meta-schema + publish-stage lint — fails `npm test`
 the moment the definition would be rejected by the API). `forms:push` loads a
 definition file into an org through the real engine path: it impersonates an

package/template/docs/FORMS_CHECKLIST.md

@@ -1,61 +1,71 @@
 # Forms Module Completion Checklist
 
-The forms module must pass `MODULE_COMPLETION_CHECKLIST.md` verbatim, **plus**
-these form-builder-specific items (ecosystem guide §11). Re-verify after any
-engine upgrade or forms-module change.
+The forms module must pass `MODULE_COMPLETION_CHECKLIST.md` verbatim, plus
+these form-builder-specific items from the ecosystem guide. Re-verify after
+any engine upgrade or forms-module change.
+
+Last verified on 2026-06-13 against the generated dogfood app (`test-out`) with
+local PostgreSQL `postgres/root`: `npm run dogfood`, generated app
+`npm run build`, `npm run lint`, `npm test`, `npm run test:e2e`; engine
+`npm test`, `npm run lint`, `npm run typecheck`.
 
 ## Boundary
 
-- [ ] The engine package (`@ftisindia/form-builder`) compiles and unit-tests **without** the template; only `src/modules/forms` imports template services.
-- [ ] The engine's boundary lint passes: no `@nestjs/*`, `@prisma/*`, `@casl/*`, or template imports anywhere in the engine core; engine errors are engine-typed and mapped to HTTP exceptions only in the glue (`forms-error.mapper.ts`).
+- [x] The engine package (`@ftisindia/form-builder`) compiles and unit-tests without the template; only `src/modules/forms` imports template services.
+- [x] The engine's boundary lint passes: no `@nestjs/*`, `@prisma/*`, `@casl/*`, or template imports anywhere in the engine core; engine errors are engine-typed and mapped to HTTP exceptions only in the glue (`forms-error.mapper.ts`).
 
 ## Schema ownership
 
-- [ ] Boot-time schema check proven both ways: green on a freshly migrated app; clear fail-fast message (naming the engine schema version and the snippet path) on a deliberately stale schema.
-- [ ] `prisma/forms.prisma` in the engine matches the models in the app's `schema.prisma`; upgrades follow copy-snippet → `prisma migrate dev`.
+- [x] Boot-time schema check proven both ways: green on a freshly migrated app; clear fail-fast message naming the engine schema version and snippet path on a deliberately stale schema.
+- [x] `prisma/forms.prisma` in the engine matches the models in the app's `schema.prisma`; upgrades follow copy-snippet -> `prisma migrate dev`.
 
 ## Permissions & registry
 
-- [ ] All 11 form permission keys exist in `permissionKeys`, are seeded, and `test/forms-permission-sync.spec.ts` passes (engine ↔ template set equality).
-- [ ] Every protected form route is in `routePermissionRegistry` (the boot-time registry validator passes).
+- [x] All 11 form permission keys exist in `permissionKeys`, are seeded, and `test/forms-permission-sync.spec.ts` passes (engine <-> template set equality).
+- [x] Every protected form route is in `routePermissionRegistry` (the boot-time registry validator passes).
+- [x] The bundled `ConferenceTracksDataSource` backs the abstract-submission example; production data sources remain app-provided `@FormDataSource()` providers.
 
 ## Gating
 
-- [ ] Attach-time gating proven by test: lacking `forms.wireDangerous` ⇒ cannot save/publish a definition wiring a `dangerous` action; the `forms.allowedDangerousActions` org setting is respected at save/publish AND at execute time.
-- [ ] `forms.managePublicAccess` gating proven: lacking it ⇒ cannot set or unset `settings.access = 'public'` (including publishing an already-public draft); every flip writes an audit row with previous/next.
+- [x] Attach-time gating proven by test: lacking `forms.wireDangerous` cannot save/publish a definition wiring a `dangerous` action; the `forms.allowedDangerousActions` org setting is respected at save/publish AND at execute time.
+- [x] `forms.managePublicAccess` gating proven: lacking it cannot set or unset `settings.access = 'public'` (including publishing an already-public draft); every flip writes an audit row with previous/next.
 
 ## Tenancy
 
-- [ ] Cross-org tests: definition fetch, submission read, and file reference across orgs are rejected as not-found.
+- [x] Cross-org tests: definition fetch, submission read, and file reference across orgs are rejected as not-found.
 
 ## Transactionality & outbox
 
-- [ ] Phase-2 transactionality proven: an induced `lockEditing` failure rolls back persist **and** in-tx action logs **and** enqueued outbox rows.
-- [ ] Outbox worker runs inside `requestContext.run({ source: 'worker', jobId, orgId, userId, requestId })`; retries honor backoff; exhausted jobs park as `FAILED`; an audit row (`forms.outbox.delivered`) is written on final success.
+- [x] Phase-2 transactionality proven: an induced `lockEditing` failure rolls back persist and in-tx action logs and enqueued outbox rows.
+- [x] Outbox worker runs inside `requestContext.run({ source: 'worker', jobId, orgId, userId, requestId })`; retries honor backoff; exhausted jobs park as `FAILED`; an audit row (`forms.outbox.delivered`) is written on final success.
+- [x] Duplicate outbox enqueue with the same idempotency key is a no-op, not a submission-rolling unique violation.
+- [x] Claimed jobs carry an opaque `claimedBy` lease token; heartbeat and `DONE`/retry/failed terminal transitions are guarded by that token so stale workers cannot settle reclaimed jobs.
+- [x] Active `PROCESSING` jobs heartbeat while handlers run; e2e covers both the store primitive and the actual dispatcher wiring with a slow handler.
 
 ## Public forms
 
-- [ ] Public path: `@Public()` render/submit works without membership; a non-public form 404s on the public routes; org-scoped storage still enforced; permissioned paths unaffected.
-- [ ] Tier-1 abuse baseline active: public routes throttled tighter than the app default; per-IP/day caps read from typed settings (with per-form override) and enforced; IP/UA stamped on anonymous submissions; captcha seam invoked when enabled; captcha-enabled-without-adapter fails at publish, not at submit.
+- [x] Public path: `@Public()` render/submit works without membership; a non-public form 404s on the public routes; org-scoped storage still enforced; permissioned paths unaffected.
+- [x] Tier-1 abuse baseline active: public routes throttled tighter than the app default; per-IP/day caps read from typed settings (with per-form override) and enforced; IP/UA stamped on anonymous submissions; captcha seam invoked when enabled; captcha-enabled-without-adapter fails at publish, not at submit.
+- [x] Captcha remains fail-closed by default: apps must bind a real `CaptchaVerifier` to `FORMS_CAPTCHA_VERIFIER` before publishing public forms with `settings.captcha: true`.
 
 ## Files
 
-- [ ] Upload → submit → `LINKED` flow proven; a foreign user's `fileId` is rejected (IDOR); GC sweeps orphaned `TEMPORARY` files past TTL; public forms with file fields are rejected at publish.
+- [x] Upload -> submit -> `LINKED` flow proven; a foreign user's `fileId` is rejected (IDOR); public uploads require the per-file token returned by the public upload endpoint; GC sweeps orphaned `TEMPORARY` files past TTL.
 
 ## Webhooks
 
-- [ ] Webhook destinations are gated by the `forms.webhookAllowedHosts` setting at save AND publish (`WEBHOOK_HOST_NOT_ALLOWED`); https required except loopback (`WEBHOOK_URL_INSECURE`).
-- [ ] Webhook jobs enqueue inside the submit transaction (idempotency key `<submissionId>:webhook:<i>`), never on draft saves or persist-less pipelines.
-- [ ] Deliveries are signed (`x-forms-signature`) when a secret is set; payload data passes sensitive-field redaction; failures retry then park as FAILED.
-- [ ] `forms:push` loads definitions only through the audited service path (real member RBAC, save-time lint, versioning) — never raw DB writes.
+- [x] Webhook destinations are gated by the `forms.webhookAllowedHosts` setting at save AND publish (`WEBHOOK_HOST_NOT_ALLOWED`); https required except loopback (`WEBHOOK_URL_INSECURE`).
+- [x] Webhook jobs enqueue inside the submit transaction (idempotency key `<submissionId>:webhook:<i>`), never on draft saves or persist-less pipelines.
+- [x] Deliveries are signed (`x-forms-signature`) when a secret is set and carry `x-forms-idempotency-key` for receiver-side dedupe; new outbox payloads store the precomputed signature, not the shared secret; legacy secret-bearing payloads are sanitized before delivery; payload data passes sensitive-field redaction; failures retry then park as FAILED.
+- [x] `forms:push` loads definitions only through the audited service path (real member RBAC, save-time lint, versioning) and never raw DB writes.
 
 ## Export & redaction
 
-- [ ] `formSubmissions.export` enforced independently of `.read`; **every** export writes an audit row with form key, filters, row count, and included fields.
-- [ ] Credential/sensitive fields never appear in `FormActionLog` or `AuditLog` metadata (redaction test); dangerous actions log `[REDACTED]` for input and output.
+- [x] `formSubmissions.export` enforced independently of `.read`; every export writes an audit row with form key, filters, row count, and included fields.
+- [x] Credential/sensitive fields never appear in `FormActionLog` or `AuditLog` metadata (redaction test); dangerous actions log `[REDACTED]` for input and output; unknown credential-like keys (`apiKey`, `token`, etc.) are defensively redacted.
 
 ## Envelope & docs
 
-- [ ] Engine validation errors arrive as `{ error: { code, message, details: { errors: [...] } } }` through the global filter; lint failures carry `details.issues`.
-- [ ] Engine-thrown denials and not-founds are indistinguishable in shape from the template's own.
-- [ ] Swagger documents all form routes (dynamic payloads as `object`).
+- [x] Engine validation errors arrive as `{ error: { code, message, details: { errors: [...] } } }` through the global filter; lint failures carry `details.issues`.
+- [x] Engine-thrown denials and not-founds are indistinguishable in shape from the template's own.
+- [x] Swagger documents all form routes (dynamic payloads as `object`).

package/template/docs/REPORTS.md

@@ -50,8 +50,17 @@ definition AND again at execute time.
 
 Example definitions live in `src/modules/reports/examples/` — `org-members`
 (custom source, no form builder anywhere) and `abstract-review-board` (the
-form-backed quick path, indexed tier).
+form-backed quick path, indexed tier).
+
+## Definitions as code (`reports:push`)
 
+```bash
+npm run reports:push -- --file src/modules/reports/definitions/member-directory.report.json \
+    --org demo --user owner@example.com --publish
+```
+
+`src/modules/reports/definitions/*.report.json` files are source files; they are not loaded automatically at boot. `reports:push` loads a definition into an org through the real service path, so member RBAC, save/publish linting, versioning, and audit rows match the HTTP API. Re-pushing unchanged content is a no-op; changed content creates the next draft version, and `--publish` publishes it.
+
 ## One contract: the QuerySpec
 
 The frontend never builds queries. Every grid interaction posts a declarative
@@ -150,8 +159,8 @@ the thinking:
   against the submission's stamped `formVersion`; `updateStatus` runs the same
   pipeline as the form. Both require `formSubmissions.update`. There is no
   generic UPDATE — the report engine never mutates source rows itself.
-- Version note: columns resolve against the latest **published** form version.
-  Renamed-field mapping across versions needs rename metadata the form
+- Known limitation: columns resolve against the latest **published** form
+  version. Renamed-field mapping across versions needs rename metadata the form
   definition format does not yet carry.
 
 ## Performance tiers
@@ -202,6 +211,13 @@ row family's previous `label:*`). Curate the vocabulary with the
   streamed into reports-owned org-scoped file storage
   (`REPORTS_EXPORT_STORAGE_DIR`), polled at `GET /reports/exports/:jobId`, and
   downloaded from `GET /reports/exports/:jobId/download`.
+- The default local-disk storage is for a single app instance or for multiple
+  instances sharing the same mounted volume. For horizontal scaling without a
+  shared volume, bind an S3/GCS-backed `ExportFileSink` so downloads can land on
+  any node.
+- Finished async export files are retained for `REPORTS_EXPORT_RETENTION_DAYS`
+  (default 7) and swept by the reports export worker. After expiry, the job row
+  remains for audit/status history but the file is no longer downloadable.
 - Every export — sync or async — runs inside ONE `REPEATABLE READ` snapshot
   (duration-bounded by `reports.exportMaxSnapshotSeconds`); an export that
   cannot finish in time is rejected with guidance to tighten filters or move
@@ -232,7 +248,9 @@ Env: `REPORTS_SCHEMA_CHECK` (`on`/`off`); `REPORTS_TOKEN_SECRET` (optional, min
 `JWT_SECRET` via HKDF; rotating it invalidates outstanding cursors/tokens —
 clients restart from page one); and the reports-owned async-export worker:
 `REPORTS_EXPORT_WORKER_ENABLED` (default true), `REPORTS_EXPORT_POLL_MS`
-(default 5000), `REPORTS_EXPORT_STORAGE_DIR` (default `./var/report-exports`).
+(default 5000), `REPORTS_EXPORT_STORAGE_DIR` (default `./var/report-exports`),
+`REPORTS_EXPORT_RETENTION_DAYS` (default 7; 0 disables cleanup), and
+`REPORTS_EXPORT_RETENTION_SWEEP_MS` (default 3600000).
 
 ## Engine upgrades (schema ownership)