@pattern-stack/codegen 0.8.0 → 0.9.0

package/runtime/subsystems/jobs/job-run-service.drizzle-backend.ts

@@ -7,7 +7,7 @@
  * `idx_job_run_scope`, whereas orchestrator mutates individual runs by id.
  */
 import { Inject, Injectable } from '@nestjs/common';
-import { and, asc, desc, eq, inArray, isNull, sql } from 'drizzle-orm';
+import { and, asc, desc, eq, gte, inArray, isNull, lt, or, sql } from 'drizzle-orm';
 import type { DrizzleClient } from '../../types/drizzle';
 import { DRIZZLE } from '../../constants/tokens';
 import { jobRuns, type JobRunRow } from './job-orchestration.schema';
@@ -19,7 +19,16 @@ import type {
   RescheduleForScopeOptions,
   PoolStatusCount,
   JobRunFailure,
+  ListJobRunsQuery,
+  JobRunPage,
+  JobRunSummary,
 } from './job-run-service.protocol';
+import {
+  clampLimit,
+  decodeKeysetCursor,
+  encodeKeysetCursor,
+  toJobRunSummary,
+} from './job-run-keyset-cursor';
 import type { IJobOrchestrator } from './job-orchestrator.protocol';
 import { JOB_ORCHESTRATOR, JOBS_MULTI_TENANT } from './jobs-domain.tokens';
 import { MissingTenantIdError } from './jobs-errors';
@@ -208,6 +217,55 @@ export class DrizzleJobRunService implements IJobRunService {
     }));
   }
 
+  async listJobRuns(query: ListJobRunsQuery = {}): Promise<JobRunPage> {
+    const limit = clampLimit(query.limit);
+    const conditions = [];
+
+    const tenantCond = this.tenantCondition('listJobRuns', query.tenantId);
+    if (tenantCond) conditions.push(tenantCond);
+    if (query.poolId) conditions.push(eq(jobRuns.pool, query.poolId));
+    if (query.rootRunId) conditions.push(eq(jobRuns.rootRunId, query.rootRunId));
+    if (query.status) conditions.push(eq(jobRuns.status, query.status));
+    if (query.since) conditions.push(gte(jobRuns.createdAt, query.since));
+
+    // Keyset seek: WHERE (created_at, id) < (cursorCreatedAt, cursorId),
+    // expanded into a SARGable OR so the same `created_at` index is used.
+    if (query.cursor) {
+      const keyset = decodeKeysetCursor(query.cursor);
+      if (keyset) {
+        conditions.push(
+          or(
+            lt(jobRuns.createdAt, keyset.createdAt),
+            and(
+              eq(jobRuns.createdAt, keyset.createdAt),
+              lt(jobRuns.id, keyset.id),
+            ),
+          )!,
+        );
+      }
+    }
+
+    // Fetch one extra row to determine whether a next page exists without a
+    // separate COUNT.
+    const rows = await this.db
+      .select()
+      .from(jobRuns)
+      .where(conditions.length > 0 ? and(...conditions) : undefined)
+      .orderBy(desc(jobRuns.createdAt), desc(jobRuns.id))
+      .limit(limit + 1);
+
+    const hasMore = rows.length > limit;
+    const page = hasMore ? rows.slice(0, limit) : rows;
+    const items = page.map(toJobRunSummary);
+    const last = page[page.length - 1];
+    const nextCursor =
+      hasMore && last
+        ? encodeKeysetCursor({ createdAt: last.createdAt, id: last.id })
+        : null;
+
+    return { items, nextCursor };
+  }
+
   /**
    * Internal helper used by cascade paths (not on the public protocol).
    * Exposed as a public method on the concrete class so infrastructure

package/runtime/subsystems/jobs/job-run-service.memory-backend.ts

@@ -16,7 +16,15 @@ import type {
   RescheduleForScopeOptions,
   PoolStatusCount,
   JobRunFailure,
+  ListJobRunsQuery,
+  JobRunPage,
 } from './job-run-service.protocol';
+import {
+  clampLimit,
+  decodeKeysetCursor,
+  encodeKeysetCursor,
+  toJobRunSummary,
+} from './job-run-keyset-cursor';
 import type { IJobOrchestrator } from './job-orchestrator.protocol';
 import { JOB_ORCHESTRATOR, JOBS_MULTI_TENANT } from './jobs-domain.tokens';
 import { MissingTenantIdError } from './jobs-errors';
@@ -177,6 +185,51 @@ export class MemoryJobRunService implements IJobRunService {
     }));
   }
 
+  async listJobRuns(query: ListJobRunsQuery = {}): Promise<JobRunPage> {
+    const limit = clampLimit(query.limit);
+    const tenantCheck = this.tenantPredicate('listJobRuns', query.tenantId);
+    const keyset = query.cursor ? decodeKeysetCursor(query.cursor) : null;
+
+    const matched: JobRunRow[] = [];
+    for (const r of this.store.runs.values()) {
+      if (tenantCheck && !tenantCheck(r)) continue;
+      if (query.poolId && r.pool !== query.poolId) continue;
+      if (query.rootRunId && r.rootRunId !== query.rootRunId) continue;
+      if (query.status && r.status !== query.status) continue;
+      if (query.since && r.createdAt.getTime() < query.since.getTime()) continue;
+      matched.push(r);
+    }
+
+    // Order created_at DESC, id DESC to match the Drizzle backend's keyset.
+    matched.sort((a, b) => {
+      const dt = b.createdAt.getTime() - a.createdAt.getTime();
+      if (dt !== 0) return dt;
+      return a.id < b.id ? 1 : a.id > b.id ? -1 : 0;
+    });
+
+    // Keyset seek: drop everything at/after the cursor's (created_at, id).
+    const seeked = keyset
+      ? matched.filter((r) => {
+          const ct = r.createdAt.getTime();
+          const kt = keyset.createdAt.getTime();
+          if (ct < kt) return true;
+          if (ct > kt) return false;
+          return r.id < keyset.id;
+        })
+      : matched;
+
+    const hasMore = seeked.length > limit;
+    const page = hasMore ? seeked.slice(0, limit) : seeked;
+    const items = page.map(toJobRunSummary);
+    const last = page[page.length - 1];
+    const nextCursor =
+      hasMore && last
+        ? encodeKeysetCursor({ createdAt: last.createdAt, id: last.id })
+        : null;
+
+    return { items, nextCursor };
+  }
+
   /**
    * Direct lookup. Not on the protocol — concrete-class convenience for
    * tests. Matches `DrizzleJobRunService.findByRootRunId` in spirit; both

package/runtime/subsystems/jobs/job-run-service.protocol.ts

@@ -54,6 +54,75 @@ export interface PoolStatusCount {
   count: number;
 }
 
+/**
+ * Filter + keyset-pagination input for `IJobRunService.listJobRuns`
+ * (OBS-LIST-1). The combiner's `listJobRuns` forwards this verbatim.
+ *
+ * Pagination is keyset (a.k.a. seek) on `created_at` descending: pass the
+ * previous page's `nextCursor` as `cursor` to fetch the following page.
+ * Keyset (not offset) so deep pages stay O(log n) and don't drift as new
+ * rows arrive at the head.
+ */
+export interface ListJobRunsQuery {
+  /** Filter to a single `pool`. */
+  poolId?: string;
+  /**
+   * Filter to a single run tree by `root_run_id`. Used by the correlation
+   * timeline to gather every run sharing a root.
+   */
+  rootRunId?: string;
+  /** Filter to a single status. Accepts any `JobRun['status']`. */
+  status?: JobRun['status'];
+  /** Lower bound on `created_at` (inclusive). */
+  since?: Date;
+  /**
+   * Opaque keyset cursor returned as `nextCursor` from a previous page.
+   * Encodes the `(createdAt, id)` of the last row seen.
+   */
+  cursor?: string;
+  /** Page size. Backend clamps to a sane default + max. */
+  limit?: number;
+  /**
+   * Multi-tenancy gate, same semantics as `countByPoolAndStatus`:
+   *   - `multiTenant` off → ignored.
+   *   - on + string → filters `tenant_id = :tenantId`.
+   *   - on + null   → filters `tenant_id IS NULL`.
+   *   - on + undefined → throws `MissingTenantIdError`.
+   */
+  tenantId?: string | null;
+}
+
+/**
+ * Summary row for the `job_run` list (OBS-LIST-1). A narrow projection over
+ * `JobRun` carrying the columns a runs viewer renders. `rootRunId` is
+ * included so the correlation timeline can stitch runs to events.
+ */
+export interface JobRunSummary {
+  runId: string;
+  rootRunId: string;
+  jobType: string;
+  pool: string;
+  status: JobRun['status'];
+  scopeEntityType: string | null;
+  scopeEntityId: string | null;
+  tenantId: string | null;
+  attempts: number;
+  errorMessage: string | null;
+  runAt: Date;
+  startedAt: Date | null;
+  finishedAt: Date | null;
+  createdAt: Date;
+}
+
+/**
+ * One page of `listJobRuns` results. `nextCursor` is `null` when there are
+ * no more rows; otherwise pass it back as `query.cursor` for the next page.
+ */
+export interface JobRunPage {
+  items: JobRunSummary[];
+  nextCursor: string | null;
+}
+
 /**
  * Summary row for the "recent failed runs" observability widget (OBS-2). A
  * narrow projection over `JobRun` — just the fields a dashboard needs.
@@ -122,4 +191,12 @@ export interface IJobRunService {
     limit: number,
     tenantId?: string | null,
   ): Promise<JobRunFailure[]>;
+
+  /**
+   * Paginated, filterable list of `job_run` rows for the observability runs
+   * viewer (OBS-LIST-1). Newest first (`created_at` desc, `id` desc as the
+   * keyset tie-break). Returns a `JobRunPage` with an opaque `nextCursor`
+   * for keyset pagination. Tenant gate follows `countByPoolAndStatus`.
+   */
+  listJobRuns(query?: ListJobRunsQuery): Promise<JobRunPage>;
 }

package/runtime/subsystems/jobs/job-worker.bullmq-backend.ts

@@ -0,0 +1,311 @@
+/**
+ * BullMQJobWorker — BullMQ-backed claim/dispatch worker (BULLMQ-1).
+ *
+ * Replaces the Drizzle `JobWorker` polling loop with one BullMQ `Worker` per
+ * active pool. BullMQ owns claim (its native atomic BRPOPLPUSH), concurrency
+ * (`{ concurrency }`), and retry/backoff (job opts set by the orchestrator) —
+ * so this class is thinner than the Drizzle poller: no claim query, no stale
+ * sweeper, no backoff math.
+ *
+ * The processor still drives the domain through Postgres `job_run` (the
+ * source of truth) and runs the user handler through the existing
+ * `JobHandlerBase` contract (`ctx.input` / `ctx.step` / `ctx.spawnChild`),
+ * identical to the Drizzle path — only the claim mechanism differs.
+ *
+ *   BullMQ job (runId) → load job_run → mark running → resolve handler via
+ *   ModuleRef → run(ctx) → mark completed / let BullMQ retry on throw.
+ *
+ * On a thrown handler error we rethrow so BullMQ applies the job's `attempts`/
+ * `backoff` policy; the final failure (attempts exhausted) is mirrored to
+ * `job_run.status='failed'` in the `failed` event handler.
+ */
+import { Logger } from '@nestjs/common';
+import type { ModuleRef } from '@nestjs/core';
+// `bullmq` is an OPTIONAL peer dependency — TYPE imports ONLY here. `Worker`,
+// `Job`, `ConnectionOptions` are erased at compile time and never resolve
+// `'bullmq'` at runtime. The `Worker` VALUE constructor is loaded lazily via
+// `await import('bullmq')` in `onModuleInit` (mirrors
+// `event-bus.redis-backend.ts:createRedisClient`). See BULLMQ-1 §Lazy import.
+import type { Worker, Job, ConnectionOptions } from 'bullmq';
+import { eq } from 'drizzle-orm';
+import type { DrizzleClient } from '../../types/drizzle';
+import { jobRuns, type JobRunRow } from './job-orchestration.schema';
+import type { IJobOrchestrator, JobRun } from './job-orchestrator.protocol';
+import type { IJobStepService } from './job-step-service.protocol';
+import {
+  JOB_HANDLER_REGISTRY,
+  type JobContext,
+  type JobHandlerBase,
+  type SpawnChildOptions,
+  type StepOptions,
+} from './job-handler.base';
+
+interface BullJobPayload {
+  runId: string;
+  type: string;
+  input: unknown;
+}
+
+function serialiseError(err: unknown, attempt: number, retryable: boolean) {
+  const e = err as { message?: string; stack?: string } | undefined;
+  return {
+    message: (e?.message ?? String(err)) as string,
+    stack: e?.stack,
+    retryable,
+    attempt,
+  };
+}
+
+/**
+ * Options for a single per-pool BullMQ worker.
+ */
+export interface BullMQJobWorkerOptions {
+  /** Logical pool name (matches `job_run.pool`). */
+  pool: string;
+  /** Fully-resolved BullMQ queue name to consume. */
+  queueName: string;
+  /** Max concurrent in-flight processors. */
+  concurrency: number;
+  /** ioredis-compatible connection. */
+  connection: ConnectionOptions;
+}
+
+export class BullMQJobWorker {
+  private readonly logger = new Logger(BullMQJobWorker.name);
+  private worker: Worker | null = null;
+
+  constructor(
+    private readonly db: DrizzleClient,
+    private readonly orchestrator: IJobOrchestrator,
+    private readonly stepService: IJobStepService,
+    private readonly options: BullMQJobWorkerOptions,
+    private readonly moduleRef: ModuleRef,
+  ) {}
+
+  async onModuleInit(): Promise<void> {
+    let WorkerCtor: typeof import('bullmq').Worker;
+    try {
+      const mod = await import('bullmq');
+      WorkerCtor = mod.Worker;
+    } catch {
+      throw new Error(
+        'BullMQ backend requires the "bullmq" package. Install it with: npm install bullmq',
+      );
+    }
+    this.worker = new WorkerCtor(
+      this.options.queueName,
+      (job) => this.process(job as Job<BullJobPayload>),
+      {
+        connection: this.options.connection,
+        concurrency: this.options.concurrency,
+      },
+    );
+    this.worker.on('failed', (job, err) => {
+      // BullMQ fires `failed` after EACH attempt; only mirror to job_run when
+      // attempts are exhausted (BullMQ will not retry further).
+      if (!job) return;
+      const attemptsMade = job.attemptsMade;
+      const maxAttempts = job.opts.attempts ?? 1;
+      if (attemptsMade >= maxAttempts) {
+        void this.markFailed(job.data.runId, err, attemptsMade);
+      }
+    });
+    this.logger.log(
+      `BullMQ worker started: pool='${this.options.pool}' queue='${this.options.queueName}' concurrency=${this.options.concurrency}`,
+    );
+  }
+
+  async onModuleDestroy(): Promise<void> {
+    if (this.worker) {
+      await this.worker.close();
+      this.worker = null;
+    }
+  }
+
+  /**
+   * Process one BullMQ job. Returns the handler output (stored by BullMQ as
+   * the job return value AND written to `job_run.output`). Throws on handler
+   * failure so BullMQ applies the retry policy.
+   */
+  private async process(job: Job<BullJobPayload>): Promise<unknown> {
+    const { runId } = job.data;
+    const [row] = await this.db
+      .select()
+      .from(jobRuns)
+      .where(eq(jobRuns.id, runId))
+      .limit(1);
+    if (!row) {
+      // Domain row vanished (canceled + removed). Treat as a no-op success so
+      // BullMQ doesn't retry a job whose authoritative state is gone.
+      this.logger.warn(`process: job_run ${runId} not found; skipping`);
+      return {};
+    }
+    const run = row as JobRunRow;
+
+    // Canceled in Postgres after enqueue but before claim — honour the domain
+    // decision and skip without running the handler.
+    if (run.status === 'canceled') {
+      return {};
+    }
+
+    const registryEntry = JOB_HANDLER_REGISTRY.get(run.jobType);
+    if (!registryEntry) {
+      throw new Error(
+        `No handler registered for jobType='${run.jobType}' (run ${run.id})`,
+      );
+    }
+
+    // Mark running (mirrors the Drizzle worker's claim transition).
+    await this.db
+      .update(jobRuns)
+      .set({
+        status: 'running',
+        claimedAt: new Date(),
+        startedAt: new Date(),
+        attempts: job.attemptsMade + 1,
+        updatedAt: new Date(),
+      })
+      .where(eq(jobRuns.id, run.id));
+
+    const HandlerClass = registryEntry.handlerClass;
+    const handler = this.moduleRef.get(
+      HandlerClass as unknown as new (...args: unknown[]) => unknown,
+      { strict: false },
+    ) as JobHandlerBase<unknown>;
+
+    const ctx: JobContext<unknown> = {
+      input: run.input,
+      run: run as JobRun,
+      step: this.makeStepFn(run),
+      spawnChild: this.makeSpawnFn(run),
+      logger: new Logger(`JobRun:${run.id}`),
+    };
+
+    const output = (await handler.run(ctx)) as
+      | Record<string, unknown>
+      | undefined;
+
+    await this.db
+      .update(jobRuns)
+      .set({
+        status: 'completed',
+        output: (output ?? {}) as Record<string, unknown>,
+        finishedAt: new Date(),
+        updatedAt: new Date(),
+      })
+      .where(eq(jobRuns.id, run.id));
+
+    return output ?? {};
+  }
+
+  private async markFailed(
+    runId: string,
+    err: unknown,
+    finalAttempts: number,
+  ): Promise<void> {
+    const [row] = await this.db
+      .select()
+      .from(jobRuns)
+      .where(eq(jobRuns.id, runId))
+      .limit(1);
+    if (!row) return;
+    const run = row as JobRunRow;
+    await this.db
+      .update(jobRuns)
+      .set({
+        status: 'failed',
+        attempts: finalAttempts,
+        finishedAt: new Date(),
+        error: serialiseError(err, finalAttempts, false),
+        updatedAt: new Date(),
+      })
+      .where(eq(jobRuns.id, runId));
+
+    // Parent-close-policy cascade — identical semantics to the Drizzle worker.
+    if (run.parentClosePolicy === 'terminate') {
+      try {
+        await this.orchestrator.cancel(run.id, {
+          cascade: true,
+          reason: 'parent-failed',
+          tenantId: run.tenantId,
+        });
+      } catch (cascadeErr) {
+        this.logger.warn(
+          `cascade on failed run ${run.id}: ${(cascadeErr as Error).message}`,
+        );
+      }
+    }
+  }
+
+  // ── ctx.step / ctx.spawnChild (mirror JobWorker) ──────────────────────────
+
+  private makeStepFn(run: JobRunRow) {
+    return async <TOutput>(
+      stepId: string,
+      fn: () => Promise<TOutput>,
+      _opts?: StepOptions,
+    ): Promise<TOutput> => {
+      void _opts;
+      const existing = await this.stepService.findStep(run.id, stepId);
+      if (existing?.status === 'completed') {
+        return existing.output as TOutput;
+      }
+      const nextAttempts = (existing?.attempts ?? 0) + 1;
+      const seq = nextAttempts; // BullMQ path: seq is per-step attempt index
+      await this.stepService.recordStep({
+        jobRunId: run.id,
+        stepId,
+        kind: 'task',
+        seq,
+        status: 'running',
+        startedAt: new Date(),
+        attempts: nextAttempts,
+      });
+      try {
+        const output = await fn();
+        await this.stepService.recordStep({
+          jobRunId: run.id,
+          stepId,
+          kind: 'task',
+          seq,
+          status: 'completed',
+          output: output as Record<string, unknown> | undefined,
+          finishedAt: new Date(),
+          attempts: nextAttempts,
+        });
+        return output;
+      } catch (err) {
+        await this.stepService.recordStep({
+          jobRunId: run.id,
+          stepId,
+          kind: 'task',
+          seq,
+          status: 'failed',
+          error: serialiseError(err, nextAttempts, false),
+          finishedAt: new Date(),
+          attempts: nextAttempts,
+        });
+        throw err;
+      }
+    };
+  }
+
+  private makeSpawnFn(run: JobRunRow) {
+    return async (
+      type: string,
+      input: unknown,
+      opts?: SpawnChildOptions,
+    ): Promise<JobRun> => {
+      return this.orchestrator.start(type, input, {
+        parentRunId: run.id,
+        parentClosePolicy: opts?.closePolicy,
+        runAt: opts?.runAt,
+        priority: opts?.priority,
+        tags: opts?.tags,
+        triggerSource: 'parent',
+        triggerRef: run.id,
+        tenantId: run.tenantId,
+      });
+    };
+  }
+}