@pattern-stack/codegen 0.8.1 → 0.9.0

package/runtime/subsystems/jobs/job-run-service.protocol.ts

@@ -54,6 +54,75 @@ export interface PoolStatusCount {
   count: number;
 }
 
+/**
+ * Filter + keyset-pagination input for `IJobRunService.listJobRuns`
+ * (OBS-LIST-1). The combiner's `listJobRuns` forwards this verbatim.
+ *
+ * Pagination is keyset (a.k.a. seek) on `created_at` descending: pass the
+ * previous page's `nextCursor` as `cursor` to fetch the following page.
+ * Keyset (not offset) so deep pages stay O(log n) and don't drift as new
+ * rows arrive at the head.
+ */
+export interface ListJobRunsQuery {
+  /** Filter to a single `pool`. */
+  poolId?: string;
+  /**
+   * Filter to a single run tree by `root_run_id`. Used by the correlation
+   * timeline to gather every run sharing a root.
+   */
+  rootRunId?: string;
+  /** Filter to a single status. Accepts any `JobRun['status']`. */
+  status?: JobRun['status'];
+  /** Lower bound on `created_at` (inclusive). */
+  since?: Date;
+  /**
+   * Opaque keyset cursor returned as `nextCursor` from a previous page.
+   * Encodes the `(createdAt, id)` of the last row seen.
+   */
+  cursor?: string;
+  /** Page size. Backend clamps to a sane default + max. */
+  limit?: number;
+  /**
+   * Multi-tenancy gate, same semantics as `countByPoolAndStatus`:
+   *   - `multiTenant` off → ignored.
+   *   - on + string → filters `tenant_id = :tenantId`.
+   *   - on + null   → filters `tenant_id IS NULL`.
+   *   - on + undefined → throws `MissingTenantIdError`.
+   */
+  tenantId?: string | null;
+}
+
+/**
+ * Summary row for the `job_run` list (OBS-LIST-1). A narrow projection over
+ * `JobRun` carrying the columns a runs viewer renders. `rootRunId` is
+ * included so the correlation timeline can stitch runs to events.
+ */
+export interface JobRunSummary {
+  runId: string;
+  rootRunId: string;
+  jobType: string;
+  pool: string;
+  status: JobRun['status'];
+  scopeEntityType: string | null;
+  scopeEntityId: string | null;
+  tenantId: string | null;
+  attempts: number;
+  errorMessage: string | null;
+  runAt: Date;
+  startedAt: Date | null;
+  finishedAt: Date | null;
+  createdAt: Date;
+}
+
+/**
+ * One page of `listJobRuns` results. `nextCursor` is `null` when there are
+ * no more rows; otherwise pass it back as `query.cursor` for the next page.
+ */
+export interface JobRunPage {
+  items: JobRunSummary[];
+  nextCursor: string | null;
+}
+
 /**
  * Summary row for the "recent failed runs" observability widget (OBS-2). A
  * narrow projection over `JobRun` — just the fields a dashboard needs.
@@ -122,4 +191,12 @@ export interface IJobRunService {
     limit: number,
     tenantId?: string | null,
   ): Promise<JobRunFailure[]>;
+
+  /**
+   * Paginated, filterable list of `job_run` rows for the observability runs
+   * viewer (OBS-LIST-1). Newest first (`created_at` desc, `id` desc as the
+   * keyset tie-break). Returns a `JobRunPage` with an opaque `nextCursor`
+   * for keyset pagination. Tenant gate follows `countByPoolAndStatus`.
+   */
+  listJobRuns(query?: ListJobRunsQuery): Promise<JobRunPage>;
 }

package/runtime/subsystems/jobs/job-worker.bullmq-backend.ts

@@ -0,0 +1,311 @@
+/**
+ * BullMQJobWorker — BullMQ-backed claim/dispatch worker (BULLMQ-1).
+ *
+ * Replaces the Drizzle `JobWorker` polling loop with one BullMQ `Worker` per
+ * active pool. BullMQ owns claim (its native atomic BRPOPLPUSH), concurrency
+ * (`{ concurrency }`), and retry/backoff (job opts set by the orchestrator) —
+ * so this class is thinner than the Drizzle poller: no claim query, no stale
+ * sweeper, no backoff math.
+ *
+ * The processor still drives the domain through Postgres `job_run` (the
+ * source of truth) and runs the user handler through the existing
+ * `JobHandlerBase` contract (`ctx.input` / `ctx.step` / `ctx.spawnChild`),
+ * identical to the Drizzle path — only the claim mechanism differs.
+ *
+ *   BullMQ job (runId) → load job_run → mark running → resolve handler via
+ *   ModuleRef → run(ctx) → mark completed / let BullMQ retry on throw.
+ *
+ * On a thrown handler error we rethrow so BullMQ applies the job's `attempts`/
+ * `backoff` policy; the final failure (attempts exhausted) is mirrored to
+ * `job_run.status='failed'` in the `failed` event handler.
+ */
+import { Logger } from '@nestjs/common';
+import type { ModuleRef } from '@nestjs/core';
+// `bullmq` is an OPTIONAL peer dependency — TYPE imports ONLY here. `Worker`,
+// `Job`, `ConnectionOptions` are erased at compile time and never resolve
+// `'bullmq'` at runtime. The `Worker` VALUE constructor is loaded lazily via
+// `await import('bullmq')` in `onModuleInit` (mirrors
+// `event-bus.redis-backend.ts:createRedisClient`). See BULLMQ-1 §Lazy import.
+import type { Worker, Job, ConnectionOptions } from 'bullmq';
+import { eq } from 'drizzle-orm';
+import type { DrizzleClient } from '../../types/drizzle';
+import { jobRuns, type JobRunRow } from './job-orchestration.schema';
+import type { IJobOrchestrator, JobRun } from './job-orchestrator.protocol';
+import type { IJobStepService } from './job-step-service.protocol';
+import {
+  JOB_HANDLER_REGISTRY,
+  type JobContext,
+  type JobHandlerBase,
+  type SpawnChildOptions,
+  type StepOptions,
+} from './job-handler.base';
+
+interface BullJobPayload {
+  runId: string;
+  type: string;
+  input: unknown;
+}
+
+function serialiseError(err: unknown, attempt: number, retryable: boolean) {
+  const e = err as { message?: string; stack?: string } | undefined;
+  return {
+    message: (e?.message ?? String(err)) as string,
+    stack: e?.stack,
+    retryable,
+    attempt,
+  };
+}
+
+/**
+ * Options for a single per-pool BullMQ worker.
+ */
+export interface BullMQJobWorkerOptions {
+  /** Logical pool name (matches `job_run.pool`). */
+  pool: string;
+  /** Fully-resolved BullMQ queue name to consume. */
+  queueName: string;
+  /** Max concurrent in-flight processors. */
+  concurrency: number;
+  /** ioredis-compatible connection. */
+  connection: ConnectionOptions;
+}
+
+export class BullMQJobWorker {
+  private readonly logger = new Logger(BullMQJobWorker.name);
+  private worker: Worker | null = null;
+
+  constructor(
+    private readonly db: DrizzleClient,
+    private readonly orchestrator: IJobOrchestrator,
+    private readonly stepService: IJobStepService,
+    private readonly options: BullMQJobWorkerOptions,
+    private readonly moduleRef: ModuleRef,
+  ) {}
+
+  async onModuleInit(): Promise<void> {
+    let WorkerCtor: typeof import('bullmq').Worker;
+    try {
+      const mod = await import('bullmq');
+      WorkerCtor = mod.Worker;
+    } catch {
+      throw new Error(
+        'BullMQ backend requires the "bullmq" package. Install it with: npm install bullmq',
+      );
+    }
+    this.worker = new WorkerCtor(
+      this.options.queueName,
+      (job) => this.process(job as Job<BullJobPayload>),
+      {
+        connection: this.options.connection,
+        concurrency: this.options.concurrency,
+      },
+    );
+    this.worker.on('failed', (job, err) => {
+      // BullMQ fires `failed` after EACH attempt; only mirror to job_run when
+      // attempts are exhausted (BullMQ will not retry further).
+      if (!job) return;
+      const attemptsMade = job.attemptsMade;
+      const maxAttempts = job.opts.attempts ?? 1;
+      if (attemptsMade >= maxAttempts) {
+        void this.markFailed(job.data.runId, err, attemptsMade);
+      }
+    });
+    this.logger.log(
+      `BullMQ worker started: pool='${this.options.pool}' queue='${this.options.queueName}' concurrency=${this.options.concurrency}`,
+    );
+  }
+
+  async onModuleDestroy(): Promise<void> {
+    if (this.worker) {
+      await this.worker.close();
+      this.worker = null;
+    }
+  }
+
+  /**
+   * Process one BullMQ job. Returns the handler output (stored by BullMQ as
+   * the job return value AND written to `job_run.output`). Throws on handler
+   * failure so BullMQ applies the retry policy.
+   */
+  private async process(job: Job<BullJobPayload>): Promise<unknown> {
+    const { runId } = job.data;
+    const [row] = await this.db
+      .select()
+      .from(jobRuns)
+      .where(eq(jobRuns.id, runId))
+      .limit(1);
+    if (!row) {
+      // Domain row vanished (canceled + removed). Treat as a no-op success so
+      // BullMQ doesn't retry a job whose authoritative state is gone.
+      this.logger.warn(`process: job_run ${runId} not found; skipping`);
+      return {};
+    }
+    const run = row as JobRunRow;
+
+    // Canceled in Postgres after enqueue but before claim — honour the domain
+    // decision and skip without running the handler.
+    if (run.status === 'canceled') {
+      return {};
+    }
+
+    const registryEntry = JOB_HANDLER_REGISTRY.get(run.jobType);
+    if (!registryEntry) {
+      throw new Error(
+        `No handler registered for jobType='${run.jobType}' (run ${run.id})`,
+      );
+    }
+
+    // Mark running (mirrors the Drizzle worker's claim transition).
+    await this.db
+      .update(jobRuns)
+      .set({
+        status: 'running',
+        claimedAt: new Date(),
+        startedAt: new Date(),
+        attempts: job.attemptsMade + 1,
+        updatedAt: new Date(),
+      })
+      .where(eq(jobRuns.id, run.id));
+
+    const HandlerClass = registryEntry.handlerClass;
+    const handler = this.moduleRef.get(
+      HandlerClass as unknown as new (...args: unknown[]) => unknown,
+      { strict: false },
+    ) as JobHandlerBase<unknown>;
+
+    const ctx: JobContext<unknown> = {
+      input: run.input,
+      run: run as JobRun,
+      step: this.makeStepFn(run),
+      spawnChild: this.makeSpawnFn(run),
+      logger: new Logger(`JobRun:${run.id}`),
+    };
+
+    const output = (await handler.run(ctx)) as
+      | Record<string, unknown>
+      | undefined;
+
+    await this.db
+      .update(jobRuns)
+      .set({
+        status: 'completed',
+        output: (output ?? {}) as Record<string, unknown>,
+        finishedAt: new Date(),
+        updatedAt: new Date(),
+      })
+      .where(eq(jobRuns.id, run.id));
+
+    return output ?? {};
+  }
+
+  private async markFailed(
+    runId: string,
+    err: unknown,
+    finalAttempts: number,
+  ): Promise<void> {
+    const [row] = await this.db
+      .select()
+      .from(jobRuns)
+      .where(eq(jobRuns.id, runId))
+      .limit(1);
+    if (!row) return;
+    const run = row as JobRunRow;
+    await this.db
+      .update(jobRuns)
+      .set({
+        status: 'failed',
+        attempts: finalAttempts,
+        finishedAt: new Date(),
+        error: serialiseError(err, finalAttempts, false),
+        updatedAt: new Date(),
+      })
+      .where(eq(jobRuns.id, runId));
+
+    // Parent-close-policy cascade — identical semantics to the Drizzle worker.
+    if (run.parentClosePolicy === 'terminate') {
+      try {
+        await this.orchestrator.cancel(run.id, {
+          cascade: true,
+          reason: 'parent-failed',
+          tenantId: run.tenantId,
+        });
+      } catch (cascadeErr) {
+        this.logger.warn(
+          `cascade on failed run ${run.id}: ${(cascadeErr as Error).message}`,
+        );
+      }
+    }
+  }
+
+  // ── ctx.step / ctx.spawnChild (mirror JobWorker) ──────────────────────────
+
+  private makeStepFn(run: JobRunRow) {
+    return async <TOutput>(
+      stepId: string,
+      fn: () => Promise<TOutput>,
+      _opts?: StepOptions,
+    ): Promise<TOutput> => {
+      void _opts;
+      const existing = await this.stepService.findStep(run.id, stepId);
+      if (existing?.status === 'completed') {
+        return existing.output as TOutput;
+      }
+      const nextAttempts = (existing?.attempts ?? 0) + 1;
+      const seq = nextAttempts; // BullMQ path: seq is per-step attempt index
+      await this.stepService.recordStep({
+        jobRunId: run.id,
+        stepId,
+        kind: 'task',
+        seq,
+        status: 'running',
+        startedAt: new Date(),
+        attempts: nextAttempts,
+      });
+      try {
+        const output = await fn();
+        await this.stepService.recordStep({
+          jobRunId: run.id,
+          stepId,
+          kind: 'task',
+          seq,
+          status: 'completed',
+          output: output as Record<string, unknown> | undefined,
+          finishedAt: new Date(),
+          attempts: nextAttempts,
+        });
+        return output;
+      } catch (err) {
+        await this.stepService.recordStep({
+          jobRunId: run.id,
+          stepId,
+          kind: 'task',
+          seq,
+          status: 'failed',
+          error: serialiseError(err, nextAttempts, false),
+          finishedAt: new Date(),
+          attempts: nextAttempts,
+        });
+        throw err;
+      }
+    };
+  }
+
+  private makeSpawnFn(run: JobRunRow) {
+    return async (
+      type: string,
+      input: unknown,
+      opts?: SpawnChildOptions,
+    ): Promise<JobRun> => {
+      return this.orchestrator.start(type, input, {
+        parentRunId: run.id,
+        parentClosePolicy: opts?.closePolicy,
+        runAt: opts?.runAt,
+        priority: opts?.priority,
+        tags: opts?.tags,
+        triggerSource: 'parent',
+        triggerRef: run.id,
+        tenantId: run.tenantId,
+      });
+    };
+  }
+}

package/runtime/subsystems/jobs/job-worker.module.ts

@@ -47,10 +47,19 @@ import type { IJobRunService } from './job-run-service.protocol';
 import type { IJobStepService } from './job-step-service.protocol';
 import {
   allNonReservedPoolNames,
+  allPoolNames,
   loadPoolConfig,
   type PoolConfig,
 } from './pool-config.loader';
 import { JobWorker, type JobWorkerOptions } from './job-worker';
+import { BullMQJobWorker } from './job-worker.bullmq-backend';
+import type { ConnectionOptions } from 'bullmq';
+import {
+  BULLMQ_CONNECTION,
+  BULLMQ_RESOLVED_CONFIG,
+  resolvePoolQueueName,
+  type BullMqResolvedConfig,
+} from './bullmq.config';
 import {
   BootValidationError,
   ReservedPoolViolationError,
@@ -62,10 +71,11 @@ export interface JobWorkerModuleOptions {
   mode: 'embedded' | 'standalone';
   /**
    * Threads into the internal `JobsDomainModule.forRoot({ backend })`
-   * import. Default `'drizzle'`. The boot-time validator runs only when
-   * this is `'drizzle'`.
+   * import. Default `'drizzle'`. The boot-time validator runs for both
+   * `'drizzle'` and `'bullmq'` (both persist `job` rows to Postgres);
+   * `'memory'` skips it.
    */
-  backend?: 'drizzle' | 'memory';
+  backend?: 'drizzle' | 'memory' | 'bullmq';
   /**
    * Active pool names. Defaults to every non-reserved pool in the resolved
    * config (i.e. `interactive`, `batch`, plus any user-defined pools).
@@ -73,6 +83,18 @@ export interface JobWorkerModuleOptions {
    * horizontally.
    */
   pools?: string[];
+  /**
+   * BULLMQ-1 Phase 1 — when `true`, `onModuleInit` activates **every** pool
+   * in the resolved config, including the reserved `events_*` lanes. This is
+   * how the standalone worker (`worker.ts`) drains bridge wrappers without
+   * the consumer hand-listing `...BRIDGE_RESERVED_POOLS`. Mutually exclusive
+   * with an explicit `pools` list — when both are set, `pools` wins (explicit
+   * beats blanket) and `allPools` is ignored.
+   *
+   * `BridgeModule`'s reserved-pool guard short-circuits to "pass" when this
+   * is `true`, since every reserved pool is provably being polled.
+   */
+  allPools?: boolean;
   /** SIGTERM drain budget. Default 30_000 ms. */
   shutdownTimeoutMs?: number;
   /**
@@ -128,6 +150,17 @@ export class JobWorkerOrchestrator implements OnModuleInit, OnModuleDestroy {
      */
     @Optional() @Inject(DRIZZLE) private readonly db: DrizzleClient | null = null,
     private readonly moduleRef?: ModuleRef,
+    /**
+     * BULLMQ-1 — resolved BullMQ connection + config, only bound when the
+     * inner `JobsDomainModule` was booted with `backend: 'bullmq'`. `@Optional()`
+     * so drizzle/memory boots see `null`.
+     */
+    @Optional()
+    @Inject(BULLMQ_CONNECTION)
+    private readonly bullConnection: ConnectionOptions | null = null,
+    @Optional()
+    @Inject(BULLMQ_RESOLVED_CONFIG)
+    private readonly bullConfig: BullMqResolvedConfig | null = null,
   ) {}
 
   // ============================================================================
@@ -166,8 +199,14 @@ export class JobWorkerOrchestrator implements OnModuleInit, OnModuleDestroy {
     }
 
     // (6) Resolve active pool list and spawn one worker per pool.
-    const activePools =
-      this.options.pools ?? allNonReservedPoolNames(poolConfig);
+    //     Precedence: explicit `pools` > `allPools` (incl. reserved) >
+    //     non-reserved default. BULLMQ-1 Phase 1 adds the `allPools` rung so
+    //     the standalone worker drains the reserved `events_*` bridge lanes.
+    const activePools = this.options.pools
+      ? this.options.pools
+      : this.options.allPools
+        ? allPoolNames(poolConfig)
+        : allNonReservedPoolNames(poolConfig);
 
     for (const poolName of activePools) {
       const def = poolConfig.get(poolName);
@@ -193,14 +232,20 @@ export class JobWorkerOrchestrator implements OnModuleInit, OnModuleDestroy {
       };
       const worker = this.options.workerFactory
         ? this.options.workerFactory(workerOptions)
-        : this.spawnWorker(workerOptions);
+        : backend === 'bullmq'
+          ? this.spawnBullMQWorker(poolName, def.queue, def.concurrency, poolConfig)
+          : this.spawnWorker(workerOptions);
       // `JobWorker` extends Nest's lifecycle hooks but the worker isn't
       // a Nest provider here (we manage the array ourselves). Call
-      // `onModuleInit` synchronously to start the polling loop.
-      worker.onModuleInit();
+      // `onModuleInit` to start the loop. The Drizzle/stub workers return
+      // void; `BullMQJobWorker.onModuleInit` is async (it lazily loads the
+      // optional `bullmq` package), so we `await` — awaiting a `void` is a
+      // harmless no-op for the synchronous workers.
+      await worker.onModuleInit();
       this.workers.push(worker);
       this.logger.log(
-        `JobWorker started: pool='${poolName}' (queue='${def.queue}') concurrency=${def.concurrency}`,
+        `JobWorker started: pool='${poolName}' (queue='${def.queue}') ` +
+          `concurrency=${def.concurrency} backend='${backend}'`,
       );
     }
   }
@@ -220,6 +265,20 @@ export class JobWorkerOrchestrator implements OnModuleInit, OnModuleDestroy {
       }
     }
     this.workers.length = 0;
+
+    // BULLMQ-1 — close the orchestrator's producer-side Queue/FlowProducer
+    // connections so the process can exit cleanly. The orchestrator is the
+    // BullMQ producer; workers are the consumers (closed above).
+    const orch = this.orchestrator as { closeConnections?: () => Promise<void> };
+    if (typeof orch.closeConnections === 'function') {
+      try {
+        await orch.closeConnections();
+      } catch (err) {
+        this.logger.error(
+          `BullMQ orchestrator connection close failed: ${(err as Error).message}`,
+        );
+      }
+    }
   }
 
   // ============================================================================
@@ -296,6 +355,54 @@ export class JobWorkerOrchestrator implements OnModuleInit, OnModuleDestroy {
       this.moduleRef,
     );
   }
+
+  /**
+   * BULLMQ-1 — spawn a per-pool `BullMQJobWorker`. Requires the Drizzle
+   * client (the worker drives `job_run` as the source of truth) AND the
+   * resolved BullMQ connection (bound by `JobsDomainModule` when
+   * `backend: 'bullmq'`). The queue name is derived identically to the
+   * orchestrator's `dispatch` via `resolvePoolQueueName(pool, …)` so producer
+   * and consumer agree.
+   */
+  private spawnBullMQWorker(
+    pool: string,
+    _queueAlias: string,
+    concurrency: number,
+    poolConfig: PoolConfig,
+  ): BullMQJobWorker {
+    if (!this.db) {
+      throw new Error(
+        `JobWorkerModule: BullMQ worker spawning requires the Drizzle client ` +
+          `(no DRIZZLE provider available) — job_run remains the source of truth.`,
+      );
+    }
+    if (!this.bullConnection) {
+      throw new Error(
+        `JobWorkerModule: BullMQ worker spawning requires a resolved ` +
+          `BULLMQ_CONNECTION. Ensure JobsDomainModule was booted with ` +
+          `backend: 'bullmq'.`,
+      );
+    }
+    if (!this.moduleRef) {
+      throw new Error(
+        `JobWorkerModule: ModuleRef not available — cannot construct ` +
+          `BullMQJobWorker with handler DI support.`,
+      );
+    }
+    const queueName = resolvePoolQueueName(pool, this.bullConfig, poolConfig);
+    return new BullMQJobWorker(
+      this.db,
+      this.orchestrator,
+      this.stepService,
+      {
+        pool,
+        queueName,
+        concurrency,
+        connection: this.bullConnection,
+      },
+      this.moduleRef,
+    );
+  }
 }
 
 @Module({})
@@ -314,7 +421,14 @@ export class JobWorkerModule {
         { provide: JOB_WORKER_MODULE_OPTIONS, useValue: opts },
         JobWorkerOrchestrator,
       ],
-      exports: [],
+      // BULLMQ-1 Phase 1 — export the options token so `BridgeModule`'s
+      // reserved-pool guard (`onModuleInit`) can actually inject it.
+      // Previously `exports: []` left the `@Optional()` inject resolving to
+      // `undefined` and the guard silently no-opped (a dead check). With the
+      // token exported the guard fires for real; consumers that omit the
+      // reserved pools (and don't set `allPools`) now fail fast with
+      // `BridgeReservedPoolsNotPolledError` — which is correct.
+      exports: [JOB_WORKER_MODULE_OPTIONS],
     };
   }
 }