@ultimat3/jobs 1.2.0 → 2.0.0

package/src/driver-pg.ts

@@ -1,31 +1,43 @@
 // The DEFAULT driver: a Postgres queue. Zero infra to start — the database you already have
 // is the queue. `SELECT ... FOR UPDATE SKIP LOCKED` lets N workers claim disjoint batches
 // without a coordinator, and the visibility timeout (`visible_at`) makes a worker crash cost
-// one lease instead of one job. The statements themselves live in `driver-pg-sql.ts`.
+// one lease instead of one job. The statements themselves live in `driver-pg-sql.ts`, the schema
+// they run against in `driver-pg-ddl.ts`, and the row-to-record decoding in `driver-pg-rows.ts`.
 
 import type { Clock } from '@ultimat3/core';
 import { systemClock, uuid } from '@ultimat3/core';
+import type { BackfillLedger } from './backfill-ledger';
 import { nowMs } from './clock';
 import type {
   ClaimedJob,
   ClaimOptions,
   EnqueueRequest,
   EnqueueResult,
+  HeartbeatOptions,
   JobDriver,
   JobFilter,
   JobIntrospection,
-  JobRecord,
   NackOptions,
   QueueStats,
 } from './driver';
 import { DEFAULT_QUEUE } from './driver';
+import type { BackfillRow, JobRow, StepRow } from './driver-pg-rows';
+import { num, toBackfillRun, toJobRecord, toStepRecord } from './driver-pg-rows';
 import {
   SQL_ACK,
   SQL_ADVISORY_UNLOCK,
+  SQL_BACKFILL_FINISH,
+  SQL_BACKFILL_LIST,
+  SQL_BACKFILL_PROGRESS,
+  SQL_BACKFILL_START,
+  SQL_CANCEL,
   SQL_CLAIM,
   SQL_ENQUEUE,
   SQL_FIND_LIVE_BY_KEY,
   SQL_HEARTBEAT,
+  SQL_LEASE_ACQUIRE,
+  SQL_LEASE_RELEASE,
+  SQL_LEASE_RENEW,
   SQL_NACK,
   SQL_STATS,
   SQL_STEP_GET,
@@ -33,9 +45,20 @@ import {
   SQL_TRY_ADVISORY_LOCK,
 } from './driver-pg-sql';
 import { DriverUnavailableError, JobDuplicateError } from './errors';
-import type { StepRecord, StepStore } from './steps';
+import type { HeldLease, LeaseStore } from './leases';
+import type { StepStore } from './steps';
 
-/** The one thing this driver needs from the DB layer. Satisfied by `Bun.sql` and by a Tx. */
+/**
+ * The one thing this driver needs from the DB layer, declared structurally so this package can
+ * depend on no database package at all.
+ *
+ * **Not satisfied by `Bun.sql`** — verified against Bun 1.3.14: `Bun.sql.query` is `undefined`.
+ * `Bun.sql` is a tagged template whose positional form is `unsafe`, so a `{ executor: Bun.sql }`
+ * would `TypeError` on the first claim. What satisfies it is a one-line adapter over a client that
+ * already speaks `(text, values)` — `@ultimat3/cli`'s `pgExecutorFor(client)` is the framework's
+ * own, wrapping `@ultimat3/db`'s `DbClient.query({ text, values })` — and a `DbTx`, which is a
+ * client on the transaction's own connection.
+ */
 export interface PgExecutor {
   query<R>(sql: string, params: readonly unknown[]): Promise<readonly R[]>;
 }
@@ -45,91 +68,14 @@ export interface PgDriverOptions {
   readonly clock?: Clock;
 }
 
-interface StepRow {
-  readonly run_id: string;
-  readonly name: string;
-  readonly status: string;
-  readonly output: unknown;
-  readonly started_at: number | string;
-  readonly completed_at: number | string | null;
-  readonly wake_at: number | string | null;
-  readonly event: string | null;
-  readonly correlation_key: string | null;
-  readonly attempts: number;
-  readonly error: string | null;
-}
-
-interface JobRow {
-  readonly id: string;
-  readonly name: string;
-  readonly queue: string;
-  readonly input: unknown;
-  readonly idempotency_key: string;
-  readonly run_id: string;
-  readonly attempt: number;
-  readonly max_attempts: number;
-  readonly state: string;
-  readonly tenant_id: string | null;
-  readonly last_error: string | null;
-  readonly claimed_by: string | null;
-  readonly run_at: number | string;
-  readonly visible_at: number | string | null;
-  readonly created_at: number | string;
-  readonly updated_at: number | string;
-}
-
-const num = (value: number | string | null | undefined): number =>
-  value === null || value === undefined ? 0 : Number(value);
-
-const optionalNum = (value: number | string | null | undefined): number | undefined =>
-  value === null || value === undefined ? undefined : Number(value);
-
-function toJobRecord(row: JobRow): JobRecord {
-  const visibleAt = optionalNum(row.visible_at);
-  return {
-    id: row.id,
-    name: row.name,
-    queue: row.queue,
-    input: row.input,
-    idempotencyKey: row.idempotency_key,
-    runId: row.run_id,
-    attempt: row.attempt,
-    maxAttempts: row.max_attempts,
-    state: row.state as JobRecord['state'],
-    runAt: num(row.run_at),
-    createdAt: num(row.created_at),
-    updatedAt: num(row.updated_at),
-    ...(row.tenant_id === null ? {} : { tenantId: row.tenant_id }),
-    ...(row.last_error === null ? {} : { lastError: row.last_error }),
-    ...(row.claimed_by === null ? {} : { claimedBy: row.claimed_by }),
-    ...(visibleAt === undefined ? {} : { visibleAt }),
-  };
-}
-
-function toStepRecord(row: StepRow): StepRecord {
-  const completedAt = optionalNum(row.completed_at);
-  const wakeAt = optionalNum(row.wake_at);
-  return {
-    runId: row.run_id,
-    name: row.name,
-    status: row.status as StepRecord['status'],
-    output: row.output,
-    startedAt: num(row.started_at),
-    attempts: row.attempts,
-    ...(completedAt === undefined ? {} : { completedAt }),
-    ...(wakeAt === undefined ? {} : { wakeAt }),
-    ...(row.event === null ? {} : { event: row.event }),
-    ...(row.correlation_key === null ? {} : { correlationKey: row.correlation_key }),
-    ...(row.error === null ? {} : { error: row.error }),
-  };
-}
-
 function resolveExecutor(injected: PgExecutor | undefined): PgExecutor {
   if (injected !== undefined) return injected;
   throw new DriverUnavailableError({
     driver: 'pg',
-    cause: 'no PgExecutor was provided and Bun.sql is not configured',
-    fix: 'set DATABASE_URL in .env then run `x db up`, or use driver: "memory" in app.config.ts',
+    // `Bun.sql` is named nowhere in this function and never was: there is no ambient fallback to
+    // be "not configured". An executor is injected by the boot or the driver has none.
+    cause: 'createPgDriver() was called with no executor, and this driver has no ambient fallback',
+    fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: setJobDriver(createMemoryDriver())',
   });
 }
 
@@ -171,6 +117,68 @@ function pgStepStore(exec: () => PgExecutor): StepStore {
   };
 }
 
+function pgBackfillLedger(exec: () => PgExecutor): BackfillLedger {
+  return {
+    async start(run) {
+      await exec().query(SQL_BACKFILL_START, [run.runId, run.name, run.checksum, run.appVersion]);
+    },
+    async progress(runId, at) {
+      await exec().query(SQL_BACKFILL_PROGRESS, [runId, at.rows, at.cursor]);
+    },
+    async finish(runId, at) {
+      await exec().query(SQL_BACKFILL_FINISH, [runId, at.status, at.rows]);
+    },
+    async list(filter = {}) {
+      const rows = await exec().query<BackfillRow>(SQL_BACKFILL_LIST, [
+        filter.name ?? null,
+        filter.status ?? null,
+        filter.runId ?? null,
+        filter.limit ?? 100,
+      ]);
+      return rows.map(toBackfillRun);
+    },
+  };
+}
+
+/**
+ * Fleet-wide slots over `x_job_leases`. Every decision is ONE statement — the `(lease_key, slot)`
+ * primary key is what serialises two workers, so nothing here reads a count and then acts on it.
+ */
+function pgLeaseStore(exec: () => PgExecutor): LeaseStore {
+  return {
+    async acquire(key, limit, ttlMs, holder) {
+      if (limit <= 0) return undefined;
+      const rows = await exec().query<{ slot: number | string }>(SQL_LEASE_ACQUIRE, [
+        key,
+        holder,
+        limit,
+        ttlMs,
+      ]);
+      const row = rows[0];
+      return row === undefined ? undefined : { key, slot: Number(row.slot), holder };
+    },
+    async renew(lease, ttlMs) {
+      const rows = await exec().query<{ slot: number | string }>(SQL_LEASE_RENEW, [
+        lease.key,
+        lease.slot,
+        lease.holder,
+        ttlMs,
+      ]);
+      return rows.length > 0;
+    },
+    async release(lease: HeldLease) {
+      await exec().query(SQL_LEASE_RELEASE, [lease.key, lease.slot, lease.holder]);
+    },
+    async held(key) {
+      const rows = await exec().query<{ n: number | string }>(
+        `select count(*)::int as n from x_job_leases where lease_key = $1 and expires_at > now()`,
+        [key],
+      );
+      return num(rows[0]?.n);
+    },
+  };
+}
+
 export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
   const clock = options.clock ?? systemClock;
   let executor: PgExecutor | undefined;
@@ -224,16 +232,23 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         throw new DriverUnavailableError({
           driver: 'pg',
           cause: `job ${jobId} does not exist`,
-          fix: 'x jobs list --state dead --json',
+          fix: 'x jobs ls --state dead --json',
         });
       }
       return toJobRecord(row);
     },
+    async cancel(jobId, reason) {
+      const rows = await exec().query<JobRow>(SQL_CANCEL, [jobId, reason ?? null]);
+      const row = rows[0];
+      return row === undefined ? undefined : toJobRecord(row);
+    },
   };
 
   return {
     name: 'pg',
     steps: pgStepStore(exec),
+    backfills: pgBackfillLedger(exec),
+    leases: pgLeaseStore(exec),
     introspect,
 
     async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
@@ -248,14 +263,19 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         request.maxAttempts,
         runAt,
         request.tenantId ?? null,
+        request.traceparent ?? null,
+        request.enqueuedBy ?? null,
       ]);
       const inserted = rows[0];
       if (inserted !== undefined) {
         return { id: inserted.id, runId: inserted.run_id, deduped: false };
       }
 
-      // `do nothing` fired: a live job already owns this idempotency key.
+      // `do nothing` fired: a live job OF THIS NAME already owns this idempotency key. The name
+      // is in the lookup because it is in the index — without it this returned whichever other
+      // job happened to derive the same natural key, and the caller's work silently never ran.
       const existing = await exec().query<{ id: string; run_id: string }>(SQL_FIND_LIVE_BY_KEY, [
+        request.name,
         request.idempotencyKey,
       ]);
       const found = existing[0];
@@ -263,7 +283,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         throw new DriverUnavailableError({
           driver: 'pg',
           cause: `enqueue of "${request.name}" was rejected but no live row holds its idempotency key`,
-          fix: 'x db check — the x_jobs_idempotency_live_idx index is missing or stale',
+          fix: 'x db migrate   # reapplies SQL_JOBS_TABLE, whose x_jobs_name_idempotency_live_idx is what this lookup reads',
         });
       }
       if (request.onConflict === 'error') {
@@ -311,8 +331,15 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
       ]);
     },
 
-    async heartbeat(jobId: string, heartbeatOptions): Promise<void> {
-      await exec().query(SQL_HEARTBEAT, [jobId, heartbeatOptions.visibilityTimeoutMs]);
+    async heartbeat(jobId: string, heartbeatOptions: HeartbeatOptions): Promise<boolean> {
+      const rows = await exec().query<{ id: string }>(SQL_HEARTBEAT, [
+        jobId,
+        heartbeatOptions.visibilityTimeoutMs,
+        heartbeatOptions.workerId ?? null,
+      ]);
+      // No row means the job is no longer ours: cancelled from outside, or re-claimed after this
+      // lease lapsed. Either way the caller has to stop running it.
+      return rows.length > 0;
     },
 
     async stats(): Promise<readonly QueueStats[]> {
@@ -338,18 +365,37 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
   };
 }
 
-/** Advisory-lock leader election, used by the `scheduler` role. */
+/**
+ * Advisory-lock leader election. **Correct only on a DEDICATED connection.**
+ *
+ * `pg_try_advisory_lock` takes a SESSION-level lock, and a session is a connection: hand this a
+ * pooled executor and the lock is released the instant that connection goes back to the pool, so
+ * every node reads itself as leader and a rolling restart double-fires every task. Postgres also
+ * refcounts the lock per acquisition, so a second `acquire()` on a session that already holds the
+ * key would need a second `release()` — hence the `held` guard below, which makes repeated
+ * `acquire()` calls (the scheduler renews every round) a no-op rather than a leak.
+ *
+ * `@ultimat3/realtime`'s `PgAdvisoryLock` is the shape that gets this right: it opens a connection
+ * of its own and *is* the lock. This package holds no wire protocol, so the executor it is handed
+ * is whatever boot built — which is a pool. **Use `createPgLeaseLeader` instead** unless you can
+ * prove the executor is a single dedicated session.
+ */
 export function createPgLeader(
   lockKey: number,
   options: PgDriverOptions = {},
 ): { acquire(): Promise<boolean>; release(): Promise<void> } {
   const exec = (): PgExecutor => resolveExecutor(options.executor);
+  let held = false;
   return {
     async acquire() {
+      if (held) return true;
       const rows = await exec().query<{ locked: boolean }>(SQL_TRY_ADVISORY_LOCK, [lockKey]);
-      return rows[0]?.locked === true;
+      held = rows[0]?.locked === true;
+      return held;
     },
     async release() {
+      if (!held) return;
+      held = false;
       await exec().query(SQL_ADVISORY_UNLOCK, [lockKey]);
     },
   };

package/src/driver-redis.ts

@@ -11,6 +11,7 @@ import type {
   ClaimOptions,
   EnqueueRequest,
   EnqueueResult,
+  HeartbeatOptions,
   JobDriver,
   NackOptions,
   QueueStats,
@@ -66,7 +67,7 @@ export function createRedisDriver(_options: RedisDriverOptions = {}): JobDriver
     nack(_jobId: string, _options: NackOptions): Promise<void> {
       return unavailable('nack');
     },
-    heartbeat(_jobId: string, _options: { readonly visibilityTimeoutMs: number }): Promise<void> {
+    heartbeat(_jobId: string, _options: HeartbeatOptions): Promise<boolean> {
       return unavailable('heartbeat');
     },
     stats(): Promise<readonly QueueStats[]> {

package/src/driver.ts

@@ -1,10 +1,31 @@
-// The queue contract. Every driver (pg, memory, redis, nats) implements exactly this, so
-// switching backends is a config line and ZERO job-code change. Six methods and no more:
-// claim/ack/nack with a visibility timeout is the smallest set that survives a worker crash.
-
+// The queue contract. Every driver implements exactly this, so a job's code never names one.
+// Six methods and no more: claim/ack/nack with a visibility timeout is the smallest set that
+// survives a worker crash.
+//
+// This header used to say "switching backends is a config line" (`As of 2026-08`). There is no
+// such config line: `JobsConfig.driver` has no reader anywhere and boot always builds
+// `createPgDriver`. `pg` and `memory` are the two that exist; `redis` and `nats` are honest
+// `X_NOT_IMPLEMENTED` stubs. What IS true is the second half — swapping the driver is
+// `setJobDriver(other)` and ZERO job-code change — and that is what the interface buys.
+
+import type { BackfillLedger } from './backfill-ledger';
+import type { LeaseStore } from './leases';
 import type { StepStore } from './steps';
 
-export type JobState = 'ready' | 'delayed' | 'running' | 'suspended' | 'done' | 'failed' | 'dead';
+/**
+ * `cancelled` is terminal and is NOT `dead`: a dead letter is work that failed and can be retried,
+ * a cancellation is work an operator stopped on purpose and `x jobs retry` must not resurrect by
+ * accident. It appears in no claim predicate, so the queue never hands a cancelled row out again.
+ */
+export type JobState =
+  | 'ready'
+  | 'delayed'
+  | 'running'
+  | 'suspended'
+  | 'done'
+  | 'failed'
+  | 'dead'
+  | 'cancelled';
 
 export interface JobRecord {
   readonly id: string;
@@ -27,6 +48,14 @@ export interface JobRecord {
   readonly lastError?: string;
   readonly claimedBy?: string;
   readonly visibleAt?: number;
+  /**
+   * W3C `traceparent` of the request that queued this job. The job's span is opened as a CHILD of
+   * it, so a checkout trace shows the HTTP span, the action span and the charge that ran two
+   * seconds later as one trace rather than three unrelated roots.
+   */
+  readonly traceparent?: string;
+  /** Actor id of whoever asked for this work. AUDIT ONLY — see `EnqueueRequest.enqueuedBy`. */
+  readonly enqueuedBy?: string;
 }
 
 export type ConflictPolicy = 'dedupe' | 'error';
@@ -43,6 +72,21 @@ export interface EnqueueRequest {
   /** Reuse an existing run id when resuming, so step history is preserved. */
   readonly runId?: string;
   readonly onConflict?: ConflictPolicy;
+  /** W3C `traceparent` of the enqueuing request. The facade stamps it; callers rarely set it. */
+  readonly traceparent?: string;
+  /**
+   * Who asked for this work — an actor id, ATTRIBUTION AND NOT AUTHORITY.
+   *
+   * The framework picks one answer to "whose permissions does a job run with" and this is it: a
+   * job body runs with SYSTEM authority and this column is an audit trail. The alternative —
+   * resolving the enqueuer at claim time and impersonating them — is worse in exactly the case
+   * that matters: a job that sleeps three days, or dead-letters and is retried next quarter, would
+   * act as somebody whose role, org membership or employment has since changed. `02-primitives.md`
+   * already calls a job server-authoritative work; this makes the row say so too. A job that must
+   * act for a user takes that user's id in its INPUT and re-authorises it in the body, where the
+   * check is visible.
+   */
+  readonly enqueuedBy?: string;
 }
 
 export interface EnqueueResult {
@@ -102,6 +146,22 @@ export interface JobIntrospection {
   deadLetters(limit?: number): Promise<readonly JobRecord[]>;
   /** Re-queue a dead/failed job. `fromStep` drops step records from that step onward. */
   requeue(jobId: string, options?: { readonly fromStep?: string }): Promise<JobRecord>;
+  /**
+   * Stop a job from outside. The only answer to a runaway pass that was otherwise "scale the
+   * worker to zero" (which stops every job) or a hand-written `UPDATE` (which the running
+   * worker's next ack overwrote). Terminal for a queued row immediately; a RUNNING one stops at
+   * its next heartbeat, which no longer matches its own row and cancels the attempt.
+   *
+   * Optional on the interface for the reason `requeue` is not: a driver may have no way to
+   * address a single row. Answers `undefined` for a job id it does not hold.
+   */
+  cancel?(jobId: string, reason?: string): Promise<JobRecord | undefined>;
+}
+
+export interface HeartbeatOptions {
+  readonly visibilityTimeoutMs: number;
+  /** Renew only if this worker is still the claimant. Omit and any claimant matches. */
+  readonly workerId?: string;
 }
 
 export interface JobDriver {
@@ -112,9 +172,33 @@ export interface JobDriver {
   claim(options: ClaimOptions): Promise<readonly ClaimedJob[]>;
   ack(jobId: string): Promise<void>;
   nack(jobId: string, options: NackOptions): Promise<void>;
-  /** Extends the lease of a long-running job so it is not double-claimed. */
-  heartbeat(jobId: string, options: { readonly visibilityTimeoutMs: number }): Promise<void>;
+  /**
+   * Extends the lease of a long-running job so it is not double-claimed.
+   *
+   * Answers whether the renewal LANDED. `false` means this process no longer owns the job — it
+   * was cancelled, or its lease lapsed and another worker re-claimed it — and the caller must
+   * stop running it. A `void` return made both indistinguishable from success, so an external
+   * cancel had nothing to reach a running job with.
+   */
+  heartbeat(jobId: string, options: HeartbeatOptions): Promise<boolean>;
   stats(): Promise<readonly QueueStats[]>;
+  /**
+   * Optional, like `introspect`: `x_backfills` records what a `backfill()` pass has already swept,
+   * and a driver without one runs backfills with no bookkeeping rather than refusing them. It
+   * hangs here for the same reason `steps` does — durable state that ships in the queue's own DDL,
+   * so one install point covers both.
+   */
+  readonly backfills?: BackfillLedger;
+  /**
+   * Optional, like `introspect` and `backfills`: fleet-wide slot counting, which is the only thing
+   * that can make `job.concurrency` mean what its docstring says. The in-process limiter is a fast
+   * path over ONE heap and is multiplied by the replica count; this is the gate that is not.
+   * A driver without one can only hold the cap per process, so `createWorker().start()` THROWS
+   * `X_JOB_CONCURRENCY_UNENFORCEABLE` (`worker.ts`) naming every registered job that declared
+   * `concurrency` — refused rather than logged, because a documented guarantee that silently does
+   * nothing is worse than either alternative.
+   */
+  readonly leases?: LeaseStore;
   readonly introspect?: JobIntrospection;
   close?(): Promise<void>;
 }