@ftisindia/create-app 0.1.5 → 0.2.0

package/template/src/modules/reports/application/services/reports-export-dispatcher.service.ts

@@ -0,0 +1,205 @@
+import {
+  Injectable,
+  Logger,
+  OnApplicationBootstrap,
+  OnModuleDestroy,
+} from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { ReportExportService } from '@ftisindia/report-builder';
+import { PrismaService } from '../../../../database/prisma/prisma.service';
+import { RequestContextService } from '../../../request-context/application/services/request-context.service';
+import { LocalDiskReportExportStorage } from '../../infrastructure/storage/local-disk-export-storage.adapter';
+import { RequestReportsContext } from '../../infrastructure/request-reports-context';
+
+const RETENTION_BATCH_SIZE = 100;
+
+/**
+ * Reports-OWNED async export worker (report design §9). Instead of riding the
+ * forms transactional outbox, it polls the engine's own `ReportExportJob`
+ * table for PENDING rows — so the reports module needs no FormsModule. Each
+ * job is claimed atomically (PENDING → RUNNING) to prevent double-processing,
+ * then run inside a restored worker request context (source: 'worker', orgId,
+ * userId) exactly as the forms dispatcher restores context for its handlers.
+ *
+ * The engine's ReportExportService.runJob streams from one REPEATABLE READ
+ * snapshot into the reports export storage and marks the row DONE/FAILED.
+ */
+@Injectable()
+export class ReportsExportDispatcherService implements OnApplicationBootstrap, OnModuleDestroy {
+  private readonly logger = new Logger('ReportsExportWorker');
+  private timer?: NodeJS.Timeout;
+  private retentionTimer?: NodeJS.Timeout;
+  private ticking = false;
+  private sweepingRetention = false;
+
+  constructor(
+    private readonly config: ConfigService,
+    private readonly prisma: PrismaService,
+    private readonly exports: ReportExportService,
+    private readonly storage: LocalDiskReportExportStorage,
+    private readonly requestContext: RequestContextService,
+    private readonly reportsContext: RequestReportsContext,
+  ) {}
+
+  onApplicationBootstrap(): void {
+    if (this.config.get<boolean>('reports.exportWorkerEnabled') === false) {
+      return;
+    }
+    const pollMs = this.config.get<number>('reports.exportPollMs') ?? 5000;
+    this.timer = setInterval(() => {
+      void this.tick();
+    }, pollMs);
+    this.timer.unref?.();
+
+    const retentionSweepMs = this.config.get<number>('reports.exportRetentionSweepMs') ?? 3_600_000;
+    this.retentionTimer = setInterval(() => {
+      void this.sweepRetention();
+    }, retentionSweepMs);
+    this.retentionTimer.unref?.();
+  }
+
+  onModuleDestroy(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    if (this.retentionTimer) {
+      clearInterval(this.retentionTimer);
+      this.retentionTimer = undefined;
+    }
+  }
+
+  /** One poll cycle — exposed for tests / manual draining. */
+  async runOnce(batchSize = 10): Promise<{ claimed: number; done: number; failed: number }> {
+    const pending = await this.prisma.reportExportJob.findMany({
+      where: { status: 'PENDING' },
+      orderBy: { createdAt: 'asc' },
+      take: batchSize,
+      select: { id: true, orgId: true, requestedBy: true },
+    });
+
+    let claimed = 0;
+    let done = 0;
+    let failed = 0;
+    for (const job of pending) {
+      // Atomic claim: only the worker that flips PENDING → RUNNING runs it.
+      const claim = await this.prisma.reportExportJob.updateMany({
+        where: { id: job.id, orgId: job.orgId, status: 'PENDING' },
+        data: { status: 'RUNNING' },
+      });
+      if (claim.count !== 1) {
+        continue;
+      }
+      claimed += 1;
+
+      const outcome = await this.requestContext.run(
+        {
+          source: 'worker',
+          orgId: job.orgId,
+          userId: job.requestedBy,
+        },
+        async () => {
+          try {
+            await this.exports.runJob(job.orgId, job.id, this.reportsContext);
+            return 'done' as const;
+          } catch (error) {
+            this.logger.error(
+              `Export job ${job.id} failed.`,
+              error instanceof Error ? error.stack : String(error),
+            );
+            return 'failed' as const;
+          }
+        },
+      );
+      if (outcome === 'done') {
+        done += 1;
+      } else {
+        failed += 1;
+      }
+    }
+    return { claimed, done, failed };
+  }
+
+  /** Delete expired local export files and clear their job fileId. */
+  async cleanupExpiredFiles(now = new Date()): Promise<number> {
+    const retentionDays = this.config.get<number>('reports.exportRetentionDays') ?? 7;
+    if (retentionDays <= 0) {
+      return 0;
+    }
+
+    const cutoff = new Date(now.getTime() - retentionDays * 24 * 60 * 60 * 1000);
+    const jobs = await this.prisma.reportExportJob.findMany({
+      where: {
+        status: 'DONE',
+        fileId: { not: null },
+        updatedAt: { lt: cutoff },
+      },
+      orderBy: { updatedAt: 'asc' },
+      take: RETENTION_BATCH_SIZE,
+      select: { id: true, orgId: true, fileId: true },
+    });
+
+    let removed = 0;
+    for (const job of jobs) {
+      if (job.fileId === null) {
+        continue;
+      }
+      try {
+        await this.storage.delete(job.fileId);
+        const result = await this.prisma.reportExportJob.updateMany({
+          where: { id: job.id, orgId: job.orgId, fileId: job.fileId },
+          data: {
+            fileId: null,
+            error: `Export file expired after ${retentionDays} day(s).`,
+          },
+        });
+        removed += result.count;
+      } catch (error) {
+        this.logger.warn(
+          `Failed to expire export file for job ${job.id}: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+    }
+
+    return removed;
+  }
+
+  private async tick(): Promise<void> {
+    if (this.ticking) {
+      return;
+    }
+    this.ticking = true;
+    try {
+      const result = await this.runOnce();
+      if (result.claimed > 0) {
+        this.logger.log(
+          `Export cycle: claimed ${result.claimed}, done ${result.done}, failed ${result.failed}.`,
+        );
+      }
+    } catch (error) {
+      this.logger.error('Export cycle failed.', error instanceof Error ? error.stack : String(error));
+    } finally {
+      this.ticking = false;
+    }
+  }
+
+  private async sweepRetention(): Promise<void> {
+    if (this.sweepingRetention) {
+      return;
+    }
+    this.sweepingRetention = true;
+    try {
+      const removed = await this.cleanupExpiredFiles();
+      if (removed > 0) {
+        this.logger.log(`Expired ${removed} report export file(s).`);
+      }
+    } catch (error) {
+      this.logger.error(
+        'Export retention sweep failed.',
+        error instanceof Error ? error.stack : String(error),
+      );
+    } finally {
+      this.sweepingRetention = false;
+    }
+  }
+}

package/template/src/modules/reports/application/services/reports-exports.service.ts

@@ -0,0 +1,78 @@
+import { Injectable, NotFoundException } from '@nestjs/common';
+import type { Readable } from 'node:stream';
+import { ReportExportService } from '@ftisindia/report-builder';
+import type { ExportOutcome, QuerySpec } from '@ftisindia/report-builder';
+import { RequestContextService } from '../../../request-context/application/services/request-context.service';
+import { ExportRequestDto } from '../../dto/export-request.dto';
+import { RequestReportsContext } from '../../infrastructure/request-reports-context';
+import { LocalDiskReportExportStorage } from '../../infrastructure/storage/local-disk-export-storage.adapter';
+import { withReportsErrorMapping } from './reports-error.mapper';
+
+const FORMAT_MIME: Record<string, string> = {
+  csv: 'text/csv',
+  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+};
+
+/**
+ * Thin HTTP-facing wrapper over the engine's ReportExportService. The sync-vs-
+ * async split by ACTUAL size, snapshot consistency, exportable:false stripping,
+ * and the PII-egress audit all live in the engine (design §9). The controller
+ * turns the ExportOutcome union into a StreamableFile or a 202 job response.
+ *
+ * Async export files land in the reports-OWNED export storage (no dependency
+ * on the forms file machinery); `download` streams a finished job's file back.
+ */
+@Injectable()
+export class ReportsExportsService {
+  constructor(
+    private readonly engine: ReportExportService,
+    private readonly reportsContext: RequestReportsContext,
+    private readonly requestContext: RequestContextService,
+    private readonly storage: LocalDiskReportExportStorage,
+  ) {}
+
+  async requestExport(orgId: string, key: string, dto: ExportRequestDto): Promise<ExportOutcome> {
+    this.requestContext.assertOrgScope(orgId);
+    return withReportsErrorMapping(() =>
+      this.engine.requestExport(
+        orgId,
+        key,
+        {
+          format: dto.format,
+          spec: dto.spec as unknown as QuerySpec | undefined,
+          columns: dto.columns,
+        },
+        this.reportsContext,
+      ),
+    );
+  }
+
+  async getJob(orgId: string, jobId: string) {
+    this.requestContext.assertOrgScope(orgId);
+    return withReportsErrorMapping(() => this.engine.getJob(orgId, jobId, this.reportsContext));
+  }
+
+  /** Stream a finished async-export file (the job's fileId is its storage key). */
+  async download(
+    orgId: string,
+    jobId: string,
+  ): Promise<{ stream: Readable; fileName: string; mimeType: string }> {
+    this.requestContext.assertOrgScope(orgId);
+    const job = await withReportsErrorMapping(() =>
+      this.engine.getJob(orgId, jobId, this.reportsContext),
+    );
+    if (job.status !== 'DONE') {
+      throw new NotFoundException('The export is not ready for download.');
+    }
+    const fileId = job.fileId;
+    if (fileId === null) {
+      throw new NotFoundException('The export file is no longer available.');
+    }
+    const format = job.spec.format;
+    return {
+      stream: this.storage.read(fileId),
+      fileName: `${job.reportKey}-v${job.reportVersion}.${format}`,
+      mimeType: FORMAT_MIME[format] ?? 'application/octet-stream',
+    };
+  }
+}

package/template/src/modules/reports/application/services/reports-queries.service.ts

@@ -0,0 +1,35 @@
+import { Injectable } from '@nestjs/common';
+import { ReportQueryService } from '@ftisindia/report-builder';
+import type { QuerySpec } from '@ftisindia/report-builder';
+import { RequestContextService } from '../../../request-context/application/services/request-context.service';
+import { QuerySpecDto } from '../../dto/query-spec.dto';
+import { RequestReportsContext } from '../../infrastructure/request-reports-context';
+import { withReportsErrorMapping } from './reports-error.mapper';
+
+/**
+ * Thin HTTP-facing wrapper over the engine's ReportQueryService — THE grid
+ * endpoint plus the self-describing meta descriptor (design §4). The DTO
+ * validated the envelope; the engine's compiler validates the semantics.
+ */
+@Injectable()
+export class ReportsQueriesService {
+  constructor(
+    private readonly engine: ReportQueryService,
+    private readonly reportsContext: RequestReportsContext,
+    private readonly requestContext: RequestContextService,
+  ) {}
+
+  async query(orgId: string, key: string, spec: QuerySpecDto) {
+    this.requestContext.assertOrgScope(orgId);
+    return withReportsErrorMapping(() =>
+      this.engine.query(orgId, key, spec as unknown as QuerySpec, this.reportsContext),
+    );
+  }
+
+  async meta(orgId: string, key: string, version: number | null) {
+    this.requestContext.assertOrgScope(orgId);
+    return withReportsErrorMapping(() =>
+      this.engine.meta(orgId, key, version, this.reportsContext),
+    );
+  }
+}

package/template/src/modules/reports/application/services/reports-settings-reader.service.ts

@@ -0,0 +1,49 @@
+import { Injectable } from '@nestjs/common';
+import { DEFAULT_REPORTS_POLICY } from '@ftisindia/report-builder';
+import type { ReportsPolicy } from '@ftisindia/report-builder';
+import { PrismaService } from '../../../../database/prisma/prisma.service';
+import { settingDefinitions } from '../../../settings/types/setting-definitions';
+
+const REPORTS_SETTING_KEYS = [
+  'reports.statementTimeoutMs',
+  'reports.exportMaxSnapshotSeconds',
+  'reports.maxRowsSync',
+  'reports.countCap',
+  'reports.liveTierMaxRows',
+  'reports.resultCacheTtlMs',
+  'reports.tagVocabulary',
+] as const;
+
+/**
+ * Reads the org's reports policy from typed settings, falling back to the
+ * setting definitions' defaults for unset keys (the SettingsService 404s on
+ * unset keys, which is wrong for policy reads). Keys without a setting (page
+ * sizes, batch sizes, token TTLs, ...) keep the engine's DEFAULT_REPORTS_POLICY
+ * values. The result feeds every engine call that takes a ReportsPolicy —
+ * statement budgets, count caps, tier gates, export bounds (design §5.2/§8/§9).
+ */
+@Injectable()
+export class ReportsSettingsReader {
+  constructor(private readonly prisma: PrismaService) {}
+
+  async policyFor(orgId: string): Promise<ReportsPolicy> {
+    const rows = await this.prisma.organisationSetting.findMany({
+      where: { orgId, key: { in: [...REPORTS_SETTING_KEYS] } },
+      select: { key: true, value: true },
+    });
+    const values = new Map(rows.map((row) => [row.key, row.value as unknown]));
+    const read = <T>(key: (typeof REPORTS_SETTING_KEYS)[number]): T =>
+      (values.has(key) ? values.get(key) : settingDefinitions[key].defaultValue) as T;
+
+    return {
+      ...DEFAULT_REPORTS_POLICY,
+      statementTimeoutMs: read<number>('reports.statementTimeoutMs'),
+      exportMaxSnapshotSeconds: read<number>('reports.exportMaxSnapshotSeconds'),
+      maxRowsSync: read<number>('reports.maxRowsSync'),
+      countCap: read<number>('reports.countCap'),
+      liveTierMaxRows: read<number>('reports.liveTierMaxRows'),
+      resultCacheTtlMs: read<number>('reports.resultCacheTtlMs'),
+      tagVocabulary: read<string[]>('reports.tagVocabulary'),
+    };
+  }
+}

package/template/src/modules/reports/application/services/reports-views.service.ts

@@ -0,0 +1,79 @@
+import { Injectable } from '@nestjs/common';
+import { ReportViewService } from '@ftisindia/report-builder';
+import type { QuerySpec } from '@ftisindia/report-builder';
+import { RequestContextService } from '../../../request-context/application/services/request-context.service';
+import { CreateSavedViewDto } from '../../dto/create-saved-view.dto';
+import { ListViewsQueryDto } from '../../dto/list-views-query.dto';
+import { UpdateSavedViewDto } from '../../dto/update-saved-view.dto';
+import { RequestReportsContext } from '../../infrastructure/request-reports-context';
+import { withReportsErrorMapping } from './reports-error.mapper';
+
+/**
+ * Thin HTTP-facing wrapper over the engine's ReportViewService. Personal vs
+ * shared semantics (ownerId null ⇒ shared, gated by reports.update) and the
+ * per-view compatibility verdicts live in the engine (design §4, finding #9).
+ */
+@Injectable()
+export class ReportsViewsService {
+  constructor(
+    private readonly engine: ReportViewService,
+    private readonly reportsContext: RequestReportsContext,
+    private readonly requestContext: RequestContextService,
+  ) {}
+
+  async list(orgId: string, key: string, query: ListViewsQueryDto) {
+    this.requestContext.assertOrgScope(orgId);
+    const items = await withReportsErrorMapping(() =>
+      this.engine.list(orgId, key, query.version ?? null, this.reportsContext),
+    );
+    return { items };
+  }
+
+  async createPersonal(orgId: string, key: string, dto: CreateSavedViewDto) {
+    this.requestContext.assertOrgScope(orgId);
+    return withReportsErrorMapping(() =>
+      this.engine.createPersonal(
+        orgId,
+        key,
+        { name: dto.name, spec: dto.spec as unknown as QuerySpec },
+        this.reportsContext,
+      ),
+    );
+  }
+
+  async createShared(orgId: string, key: string, dto: CreateSavedViewDto) {
+    this.requestContext.assertOrgScope(orgId);
+    return withReportsErrorMapping(() =>
+      this.engine.createShared(
+        orgId,
+        key,
+        { name: dto.name, spec: dto.spec as unknown as QuerySpec },
+        this.reportsContext,
+      ),
+    );
+  }
+
+  async update(orgId: string, key: string, viewId: string, dto: UpdateSavedViewDto) {
+    this.requestContext.assertOrgScope(orgId);
+    return withReportsErrorMapping(() =>
+      this.engine.update(
+        orgId,
+        key,
+        viewId,
+        {
+          ...(dto.name === undefined ? {} : { name: dto.name }),
+          ...(dto.spec === undefined ? {} : { spec: dto.spec as unknown as QuerySpec }),
+        },
+        this.reportsContext,
+      ),
+    );
+  }
+
+  async delete(orgId: string, key: string, viewId: string) {
+    this.requestContext.assertOrgScope(orgId);
+    await withReportsErrorMapping(() =>
+      this.engine.delete(orgId, key, viewId, this.reportsContext),
+    );
+    return { deleted: true };
+  }
+}

package/template/src/modules/reports/dto/action-result-response.dto.ts

@@ -0,0 +1,21 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class ActionResultResponseDto {
+  @ApiProperty({ enum: ['DONE'], example: 'DONE' })
+  status!: 'DONE';
+
+  @ApiProperty({ example: 240 })
+  affectedRows!: number;
+
+  @ApiProperty({
+    nullable: true,
+    description: 'Whatever the delegated handler returned (e.g. { updated: 12 }).',
+  })
+  result!: unknown;
+
+  @ApiProperty({
+    example: false,
+    description: 'True when this call replayed a previously recorded idempotent run (§6.3).',
+  })
+  replayed!: boolean;
+}

package/template/src/modules/reports/dto/create-report-definition.dto.ts

@@ -0,0 +1,86 @@
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import {
+  Allow,
+  ArrayMinSize,
+  IsArray,
+  IsIn,
+  IsObject,
+  IsOptional,
+  IsString,
+  Matches,
+} from 'class-validator';
+
+/**
+ * The request body IS the ReportDefinition document (report design §2/§10).
+ * Envelope-only validation — the same two-layer split as forms: this DTO
+ * whitelists the document's top-level keys (the global ValidationPipe runs
+ * with forbidNonWhitelisted) and checks the cheap invariants; the engine's
+ * meta-schema + linter are the real gate for everything nested (columns,
+ * source binding, export config, ...), so nested content passes through
+ * untouched.
+ */
+export class CreateReportDefinitionDto {
+  @ApiProperty({ example: 'abstract-review-board' })
+  @IsString()
+  @Matches(/^[a-z][a-z0-9-]*$/)
+  key!: string;
+
+  @ApiPropertyOptional({ example: 'Abstract review board' })
+  @IsOptional()
+  @IsString()
+  title?: string;
+
+  @ApiPropertyOptional({ example: 'All submitted abstracts with review status.' })
+  @IsOptional()
+  @IsString()
+  description?: string;
+
+  @ApiProperty({
+    type: 'object',
+    additionalProperties: true,
+    description: 'Source binding ({ kind, key, options? }) — resolved and validated by the engine.',
+  })
+  @IsObject()
+  source!: Record<string, unknown>;
+
+  @ApiProperty({
+    type: 'array',
+    items: { type: 'object', additionalProperties: true },
+    description: 'Column definitions — validated against the engine meta-schema and linter.',
+  })
+  @IsArray()
+  @ArrayMinSize(1)
+  columns!: Record<string, unknown>[];
+
+  @ApiPropertyOptional({
+    type: 'array',
+    items: { type: 'object', additionalProperties: true },
+  })
+  @Allow()
+  defaultSort?: unknown;
+
+  @ApiPropertyOptional({ type: 'object', additionalProperties: true })
+  @Allow()
+  search?: unknown;
+
+  @ApiPropertyOptional({ type: [String] })
+  @Allow()
+  rowActions?: unknown;
+
+  @ApiPropertyOptional({ type: 'object', additionalProperties: true })
+  @Allow()
+  export?: unknown;
+
+  @ApiProperty({ enum: ['live', 'indexed', 'materialized'], example: 'indexed' })
+  @IsIn(['live', 'indexed', 'materialized'])
+  performanceTier!: 'live' | 'indexed' | 'materialized';
+
+  // Server-assigned; tolerated on input so fetched documents round-trip.
+  @ApiPropertyOptional({ description: 'Server-assigned; ignored on input.' })
+  @Allow()
+  version?: unknown;
+
+  @ApiPropertyOptional({ description: 'Server-assigned; ignored on input.' })
+  @Allow()
+  status?: unknown;
+}

package/template/src/modules/reports/dto/create-saved-view.dto.ts

@@ -0,0 +1,26 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { Type } from 'class-transformer';
+import {
+  IsDefined,
+  IsObject,
+  IsString,
+  MaxLength,
+  MinLength,
+  ValidateNested,
+} from 'class-validator';
+import { QuerySpecDto } from './query-spec.dto';
+
+export class CreateSavedViewDto {
+  @ApiProperty({ example: 'My ML shortlist', minLength: 1, maxLength: 120 })
+  @IsString()
+  @MinLength(1)
+  @MaxLength(120)
+  name!: string;
+
+  @ApiProperty({ type: QuerySpecDto })
+  @IsDefined()
+  @IsObject()
+  @ValidateNested()
+  @Type(() => QuerySpecDto)
+  spec!: QuerySpecDto;
+}

package/template/src/modules/reports/dto/execute-action.dto.ts

@@ -0,0 +1,71 @@
+import { ApiPropertyOptional } from '@nestjs/swagger';
+import { Type } from 'class-transformer';
+import {
+  ArrayMaxSize,
+  IsArray,
+  IsObject,
+  IsOptional,
+  IsString,
+  MaxLength,
+  MinLength,
+  ValidateNested,
+} from 'class-validator';
+import { QuerySpecDto } from './query-spec.dto';
+
+/**
+ * Bulk selection (§6.3): exactly one of byIds / byFilter — enforced by the
+ * engine. byIds is self-snapshotting and capped; byFilter requires the
+ * prepared action token because rows matching at confirmation may differ at
+ * execution.
+ */
+export class ExecuteActionSelectionDto {
+  @ApiPropertyOptional({ type: [String], maxItems: 1000 })
+  @IsOptional()
+  @IsArray()
+  @ArrayMaxSize(1000)
+  @IsString({ each: true })
+  byIds?: string[];
+
+  @ApiPropertyOptional({ type: QuerySpecDto })
+  @IsOptional()
+  @IsObject()
+  @ValidateNested()
+  @Type(() => QuerySpecDto)
+  byFilter?: QuerySpecDto;
+}
+
+export class ExecuteActionDto {
+  @ApiPropertyOptional({ type: ExecuteActionSelectionDto })
+  @IsOptional()
+  @IsObject()
+  @ValidateNested()
+  @Type(() => ExecuteActionSelectionDto)
+  selection?: ExecuteActionSelectionDto;
+
+  @ApiPropertyOptional({
+    description: 'Prepared action token — REQUIRED for byFilter selections (§6.3).',
+  })
+  @IsOptional()
+  @IsString()
+  actionToken?: string;
+
+  @ApiPropertyOptional({
+    type: 'object',
+    additionalProperties: true,
+    description: "Action input — validated against the action's JSON Schema by the engine.",
+  })
+  @IsOptional()
+  @IsObject()
+  input?: Record<string, unknown>;
+
+  @ApiPropertyOptional({
+    minLength: 1,
+    maxLength: 120,
+    description: 'Idempotency ledger key — retries return the recorded outcome (§6.3).',
+  })
+  @IsOptional()
+  @IsString()
+  @MinLength(1)
+  @MaxLength(120)
+  idempotencyKey?: string;
+}