@ultimat3/jobs 6.0.0 → 8.0.0

package/CLAUDE.md

@@ -567,6 +567,25 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
   The shed also wrote `last_error = 'limited: …'`, so `x jobs show` reported a failure for a job
   that never ran — it is a `jobs.worker.shed` log field now, and `worker.ts`'s one `shed()` is
   where both sheds go. `driver-parity.test.ts` pins which bucket each lands in, in both drivers.
+- **No read returns a WHOLE row, and `driver-pg-sql.test.ts` is what enforces it** (`As of
+  2026-08-22`). `PgExecutor` is an injected seam over any client that speaks `(text, values)`, and a
+  client with no type map decodes `timestamptz` as TEXT — so `toJobRecord`/`toStepRecord` read
+  `Number('2026-01-01 00:00:00+00')` and answered `NaN`. Six statements were `select *` /
+  `returning *` and shipped that way: `pgStepStore.list`, `introspect.job`, `introspect.list`,
+  `introspect.deadLetters`, `introspect.requeue` and `SQL_CANCEL`. Every one of them feeds a
+  surface an operator reads — `x jobs ls`, `x jobs show`, `x jobs cancel` — and `SQL_CLAIM` had
+  projected epoch ms all along, so the driver disagreed with itself. The guard is a scan of every
+  production file in this directory, DISCOVERED rather than listed and with comments stripped: a
+  `select *` anywhere in them, in either case, is a failing test rather than a review note. It
+  reads the directory because the hand-kept list it replaces had already missed one
+  (`driver-pg-ddl.ts`) — a registry a new SQL source opts out of by simply not joining it is a
+  guard with the shape of a rule and none of the force.
+- **A run's acquisitions are handed back even when the wiring throws.** `worker-run.ts` starts the
+  lease heartbeat first, so every line between it and the `try` can leak an interval renewing the
+  lease of a job that never ran, with nothing left holding a reference to stop it. `context()` was
+  moved ABOVE the heartbeat for that reason; `createRunSignal` and `fleetSlots.startRenewal` — the
+  second an injected seam whose production implementation reaches a lease store — are INSIDE the
+  `try`, with `undefined` meaning "never taken" and both handbacks idempotent.
 - Step results are persisted BEFORE the step returns. Keep it that way or replay breaks.
 - All time is epoch ms from an injected `Clock`, read via `nowMs()` in `clock.ts`.
 - Drivers implement exactly the six `JobDriver` methods plus optional `introspect`, `backfills`
@@ -653,8 +672,9 @@ picture from the other side.
 | `scheduler-pg.ts` | `pgSchedulerState` (the durable watermark) + `createPgLeaseLeader` |
 | `events-pg.ts` | `createPgEventBus` — `step.waitForEvent` across processes |
 | `driver.ts` | `JobDriver` contract + wire records |
-| `driver-pg.ts` | default driver, real SQL constants, advisory-lock leader |
+| `driver-pg.ts` | default driver, real SQL constants, and `createPgLeader` — the advisory-lock election that is **not** what a scheduler uses; `scheduler-pg.ts` above owns the lease-row one boot wires |
 | `driver-pg-ddl.ts` | `SQL_JOBS_TABLE` + `SQL_OUTBOX_TABLE` — the schema the driver installs. Whichever file holds the DDL is the one whose comments may carry no `;` and no `'` |
+| `driver-pg-jobs-sql.ts` | every statement returning a whole `x_jobs` row, and the `JOB_ROW_COLUMNS` projection they share. Split off at `driver-pg-sql.ts`'s size ceiling and re-exported from it |
 | `driver-pg-rows.ts` | a Postgres row → a wire record: `JobRow`/`StepRow`/`BackfillRow` and their mappings |
 | `driver-memory.ts` | `x dev` / tests |
 | `driver-redis.ts`, `driver-nats.ts` | honest `X_NOT_IMPLEMENTED` stubs |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/jobs",
-  "version": "6.0.0",
+  "version": "8.0.0",
   "description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
   "license": "MIT",
   "type": "module",
@@ -32,9 +32,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "6.0.0",
-    "@ultimat3/entity": "6.0.0",
-    "@ultimat3/schema": "6.0.0",
-    "@ultimat3/time": "6.0.0"
+    "@ultimat3/core": "8.0.0",
+    "@ultimat3/entity": "8.0.0",
+    "@ultimat3/schema": "8.0.0",
+    "@ultimat3/time": "8.0.0"
   }
 }

package/src/driver-pg-jobs-sql.ts

@@ -0,0 +1,45 @@
+// Every statement returning a WHOLE `x_jobs` row, and the one column list they share. They ask
+// Postgres for epoch ms because `select *` left the decoding to the client's type map: one with
+// none decodes `timestamptz` as TEXT, so `toJobRecord` read `Number('2026-01-01 00:00:00+00')`
+// and `x jobs ls` / `show` / `cancel` printed `NaN` for every timestamp.
+
+/**
+ * The `JobRow` shape as a projection. Asking Postgres for epoch ms is what makes the decoding
+ * independent of the client's type map — never `select *`, whose correctness is the driver's
+ * opinion about `timestamptz` rather than this statement's.
+ */
+export const JOB_ROW_COLUMNS = `id, name, queue, input, idempotency_key, run_id, attempt,
+       max_attempts, state, tenant_id, last_error, claimed_by,
+       traceparent, enqueued_by,
+       (extract(epoch from run_at)     * 1000)::bigint as run_at,
+       (extract(epoch from visible_at) * 1000)::bigint as visible_at,
+       (extract(epoch from created_at) * 1000)::bigint as created_at,
+       (extract(epoch from updated_at) * 1000)::bigint as updated_at`;
+
+export const SQL_JOB_GET = `
+select ${JOB_ROW_COLUMNS}
+  from x_jobs where id = $1
+`.trim();
+
+export const SQL_JOB_LIST = `
+select ${JOB_ROW_COLUMNS}
+  from x_jobs
+ where ($1::text is null or queue = $1)
+   and ($2::text is null or name  = $2)
+   and ($3::text is null or state = $3)
+ order by created_at desc
+ limit $4
+`.trim();
+
+export const SQL_JOB_DEAD_LETTERS = `
+select ${JOB_ROW_COLUMNS}
+  from x_jobs where state = 'dead' order by updated_at desc limit $1
+`.trim();
+
+/** `run_at = now()` makes the requeued job due immediately; the attempt counter starts over. */
+export const SQL_JOB_REQUEUE = `
+update x_jobs
+   set state = 'ready', attempt = 0, run_at = now(), updated_at = now()
+ where id = $1
+returning ${JOB_ROW_COLUMNS}
+`.trim();

package/src/driver-pg-sql.ts

@@ -8,6 +8,19 @@
 // point keeps the ONE import path every caller already uses.
 
 export { SQL_JOBS_TABLE, SQL_OUTBOX_TABLE } from './driver-pg-ddl';
+// The whole-`x_jobs`-row reads, split off at this file's size ceiling and re-exported for the
+// same reason the DDL is. `JOB_ROW_COLUMNS` is imported as a value because `SQL_CANCEL` also
+// returns a whole row and must project it identically — two spellings of one row shape is how
+// one of them goes back to `returning *`.
+export {
+  JOB_ROW_COLUMNS,
+  SQL_JOB_DEAD_LETTERS,
+  SQL_JOB_GET,
+  SQL_JOB_LIST,
+  SQL_JOB_REQUEUE,
+} from './driver-pg-jobs-sql';
+
+import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
 
 export const SQL_ENQUEUE = `
 insert into x_jobs
@@ -112,7 +125,7 @@ update x_jobs
    set state = 'cancelled', visible_at = null, claimed_by = null,
        last_error = coalesce($2, last_error), updated_at = now()
  where id = $1 and state <> 'done'
-returning *
+returning ${JOB_ROW_COLUMNS}
 `.trim();
 
 /**
@@ -370,6 +383,22 @@ select run_id, name, status, output, attempts, error,
   from x_job_steps where run_id = $1 and name = $2
 `.trim();
 
+/**
+ * `list`'s projection is `SQL_STEP_GET`'s, minus the name predicate. It exists as its own constant
+ * because `select *` was what `list` issued: a client that decodes `timestamptz` as text — most of
+ * them, without a type map — handed `toStepRecord` a date string, `Number()` made it `NaN`, and
+ * `x jobs show` printed one per step. Every other statement in this file asks Postgres to do the
+ * conversion; this is that rule applied to the one statement that had opted out of it.
+ */
+export const SQL_STEP_LIST = `
+select run_id, name, status, output, attempts, error,
+       (extract(epoch from started_at)   * 1000)::bigint as started_at,
+       (extract(epoch from completed_at) * 1000)::bigint as completed_at,
+       (extract(epoch from wake_at)      * 1000)::bigint as wake_at,
+       event, correlation_key
+  from x_job_steps where run_id = $1 order by started_at
+`.trim();
+
 export const SQL_STEP_PUT = `
 insert into x_job_steps
   (run_id, name, status, output, started_at, completed_at, wake_at, event,

package/src/driver-pg.ts

@@ -35,12 +35,17 @@ import {
   SQL_ENQUEUE,
   SQL_FIND_LIVE_BY_KEY,
   SQL_HEARTBEAT,
+  SQL_JOB_DEAD_LETTERS,
+  SQL_JOB_GET,
+  SQL_JOB_LIST,
+  SQL_JOB_REQUEUE,
   SQL_LEASE_ACQUIRE,
   SQL_LEASE_RELEASE,
   SQL_LEASE_RENEW,
   SQL_NACK,
   SQL_STATS,
   SQL_STEP_GET,
+  SQL_STEP_LIST,
   SQL_STEP_PUT,
   SQL_TRY_ADVISORY_LOCK,
 } from './driver-pg-sql';
@@ -102,10 +107,7 @@ function pgStepStore(exec: () => PgExecutor): StepStore {
       ]);
     },
     async list(runId) {
-      const rows = await exec().query<StepRow>(
-        `select * from x_job_steps where run_id = $1 order by started_at`,
-        [runId],
-      );
+      const rows = await exec().query<StepRow>(SQL_STEP_LIST, [runId]);
       return rows.map(toStepRecord);
     },
     async del(runId, name) {
@@ -189,27 +191,21 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
 
   const introspect: JobIntrospection = {
     async job(jobId) {
-      const rows = await exec().query<JobRow>(`select * from x_jobs where id = $1`, [jobId]);
+      const rows = await exec().query<JobRow>(SQL_JOB_GET, [jobId]);
       const row = rows[0];
       return row === undefined ? undefined : toJobRecord(row);
     },
     async list(filter: JobFilter = {}) {
-      const rows = await exec().query<JobRow>(
-        `select * from x_jobs
-          where ($1::text is null or queue = $1)
-            and ($2::text is null or name  = $2)
-            and ($3::text is null or state = $3)
-          order by created_at desc
-          limit $4`,
-        [filter.queue ?? null, filter.name ?? null, filter.state ?? null, filter.limit ?? 100],
-      );
+      const rows = await exec().query<JobRow>(SQL_JOB_LIST, [
+        filter.queue ?? null,
+        filter.name ?? null,
+        filter.state ?? null,
+        filter.limit ?? 100,
+      ]);
       return rows.map(toJobRecord);
     },
     async deadLetters(limit = 100) {
-      const rows = await exec().query<JobRow>(
-        `select * from x_jobs where state = 'dead' order by updated_at desc limit $1`,
-        [limit],
-      );
+      const rows = await exec().query<JobRow>(SQL_JOB_DEAD_LETTERS, [limit]);
       return rows.map(toJobRecord);
     },
     async requeue(jobId, requeueOptions) {
@@ -222,11 +218,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
           ]);
         }
       }
-      const rows = await exec().query<JobRow>(
-        `update x_jobs set state = 'ready', attempt = 0, run_at = now(), updated_at = now()
-          where id = $1 returning *`,
-        [jobId],
-      );
+      const rows = await exec().query<JobRow>(SQL_JOB_REQUEUE, [jobId]);
       const row = rows[0];
       if (row === undefined) {
         throw new DriverUnavailableError({

package/src/scheduler.ts

@@ -1,7 +1,12 @@
 // The `scheduler` role: one node walks every registered task's cron, dispatches the occurrences
 // it owes and enqueues their jobs. The `task` primitive it reads lives in `task.ts`.
 //
-// Exactly one node dispatches per tick, enforced by a Postgres advisory lock. Two schedulers
+// Exactly one node dispatches per tick, enforced by leader election. Multi-node that is an
+// EXPIRING LEASE ROW in `x_scheduler_leader` (`createPgLeaseLeader`), never an advisory lock: an
+// advisory lock is held by the SESSION, not by this process — it outlives every transaction and is
+// released only by an explicit unlock, the pool's reset on release, or the connection dying, and
+// the next round may run on a different connection. So a node can neither renew it nor prove it
+// still holds one, and leadership passes to a second node while the first is still dispatching. Two schedulers
 // double-enqueue every task; the idempotency key would absorb it, but leader election means
 // the queue never sees the duplicate at all. One ROUND at a time is the same rule inside one
 // process: the loop re-arms on the round it just finished, and any other caller joins that
@@ -40,7 +45,7 @@ export interface LeaderElection {
 }
 
 /** Single-node default: always the leader. Multi-node uses `createPgLeaseLeader()` — never
- * `createPgLeader()`, whose advisory lock dies with the pooled connection that took it. */
+ * `createPgLeader()`, whose advisory lock is owned by a pooled session this process cannot name. */
 export function soleLeader(): LeaderElection {
   return {
     acquire: () => Promise.resolve(true),
@@ -276,10 +281,10 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     timer = undefined;
     logger.info('jobs.scheduler.draining', { reason, dispatching: round !== undefined });
     try {
-      // The round this stop races runs to the end first. Releasing the advisory lock under a
+      // The round this stop races runs to the end first. Releasing the lease under a
       // live dispatch hands the next node a task this one is still enqueueing for, and both
       // then own the same occurrence — the exact double-fire leader election exists to prevent.
-      // Settled, not awaited: a round that failed is its own caller's to see, and the lock still
+      // Settled, not awaited: a round that failed is its own caller's to see, and the lease still
       // has to go back.
       await Promise.allSettled([round]);
       if (isLeader) await leader.release();

package/src/worker-run.ts

@@ -11,6 +11,7 @@ import type { JobExecution } from './execute';
 import { executeJob } from './execute';
 import { startLeaseHeartbeat } from './heartbeat';
 import { getJob } from './job';
+import type { RunSignal } from './run-signal';
 import { createRunSignal } from './run-signal';
 import type { EventLookup } from './steps';
 import type { FleetSlots } from './worker-fleet-slots';
@@ -77,33 +78,42 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
     ...(options.clock === undefined ? {} : { clock: options.clock }),
   });
 
-  // The caller's context plus this lease's cancellation, so a job cancelled from outside — or one
-  // this worker lost the lease on — stops at the next renewal. `steps.ts` refuses every write past
-  // the signal, which is what unwinds a body that never reads it. Composed through a controller
-  // this worker owns rather than `AbortSignal.any`, for two reasons: it is handed BACK when the run
-  // settles (an app whose `context()` carries a process-lifetime signal was accumulating one
-  // composite per job), and the worker can abort it itself — which is the only way a fleet slot
-  // taken by somebody else reaches the body running under it.
-  const runSignal = createRunSignal([base.signal, heartbeat.signal]);
-  const ctx: Ctx = { ...base, signal: runSignal.signal };
+  // Everything from here on is INSIDE the `finally`, because the heartbeat is now running.
+  // `createRunSignal` and `fleetSlots.startRenewal` are ordinary calls — the second is an injected
+  // seam whose production implementation reaches a lease store — and a throw from either used to
+  // land outside the `try`, leaving an interval renewing the lease of a job that never ran with
+  // nothing holding a reference to stop it. The same defect `context()` was moved ABOVE the
+  // heartbeat for; the acquisitions after it need the `finally` rather than a reordering, since
+  // each one depends on the last. `undefined` is "never taken", and both handbacks are idempotent.
+  let runSignal: RunSignal | undefined;
+  let stopSlotRenewal: (() => void) | undefined;
 
-  // The fleet slot this job already holds, kept alive for as long as the lease is and stopped in
-  // the same `finally`: one clock for "this worker still owns the job" and "this worker still owns
-  // the slot" is one fewer way for them to disagree. A renewal answering "not yours" means another
-  // worker is already running this job under a cap of one, so it CANCELS — discarding that boolean
-  // made `job.concurrency` a number the framework prints and does not hold.
-  const stopSlotRenewal = fleetSlots.startRenewal(claimed.id, (slot) => {
-    runSignal.abort(
-      new JobSlotLostError({ job: claimed.name, jobId: claimed.id, slot: slot.slot }),
-    );
-  });
+  try {
+    // The caller's context plus this lease's cancellation, so a job cancelled from outside — or
+    // one this worker lost the lease on — stops at the next renewal. `steps.ts` refuses every write
+    // past the signal, which is what unwinds a body that never reads it. Composed through a
+    // controller this worker owns rather than `AbortSignal.any`, for two reasons: it is handed BACK
+    // when the run settles (an app whose `context()` carries a process-lifetime signal was
+    // accumulating one composite per job), and the worker can abort it itself — which is the only
+    // way a fleet slot taken by somebody else reaches the body running under it.
+    runSignal = createRunSignal([base.signal, heartbeat.signal]);
+    const signal = runSignal;
+    const ctx: Ctx = { ...base, signal: signal.signal };
 
-  // The job's span is a CHILD of the request that queued it when the row carries a trace. That
-  // link is what `04-jobs.md` promised and no column existed to hold: without it a checkout's
-  // `chargeCard` opens a fresh root two seconds later with nothing pointing back.
-  const parent = parseTraceparent(claimed.traceparent);
+    // The fleet slot this job already holds, kept alive for as long as the lease is and stopped in
+    // the same `finally`: one clock for "this worker still owns the job" and "this worker still
+    // owns the slot" is one fewer way for them to disagree. A renewal answering "not yours" means
+    // another worker is already running this job under a cap of one, so it CANCELS — discarding
+    // that boolean made `job.concurrency` a number the framework prints and does not hold.
+    stopSlotRenewal = fleetSlots.startRenewal(claimed.id, (slot) => {
+      signal.abort(new JobSlotLostError({ job: claimed.name, jobId: claimed.id, slot: slot.slot }));
+    });
+
+    // The job's span is a CHILD of the request that queued it when the row carries a trace. That
+    // link is what `04-jobs.md` promised and no column existed to hold: without it a checkout's
+    // `chargeCard` opens a fresh root two seconds later with nothing pointing back.
+    const parent = parseTraceparent(claimed.traceparent);
 
-  try {
     return await withSpan(
       `job.${handle.name}`,
       () =>
@@ -126,10 +136,10 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
       },
     );
   } finally {
-    stopSlotRenewal();
+    stopSlotRenewal?.();
     heartbeat.stop();
     // Nothing of the caller's is held past the run: `dispose` is what makes the composition above
     // reversible, and it is the whole reason this is not `AbortSignal.any`.
-    runSignal.dispose();
+    runSignal?.dispose();
   }
 }

package/src/worker.ts

@@ -31,13 +31,12 @@ const QUEUE_DEPTH_INTERVAL_MS = 15_000;
  * is deliberately unmapped: parking a run is control flow, so counting it would make every
  * `step.sleep` read as a finished job and make the failure ratio meaningless.
  */
-const JOB_OUTCOME_LABELS: Readonly<Record<JobOutcome, 'ok' | 'failed' | 'dead' | null>> =
-  Object.freeze({
-    completed: 'ok',
-    suspended: null,
-    retried: 'failed',
-    'dead-lettered': 'dead',
-  });
+const JOB_OUTCOME_LABELS = Object.freeze<Record<JobOutcome, 'ok' | 'failed' | 'dead' | null>>({
+  completed: 'ok',
+  suspended: null,
+  retried: 'failed',
+  'dead-lettered': 'dead',
+});
 
 export interface WorkerOptions {
   readonly driver: JobDriver;