@ultimat3/jobs 29.3.2 → 30.0.0

package/src/consumer-registry.ts

@@ -0,0 +1,103 @@
+// What `consumer()` declared, read back off a plain `JobHandle` — the `origin` WeakMap shape
+// `backfill-registry.ts` uses, and derived from `registeredJobs()` rather than a second registry.
+// Also the one projection of a consumer's run for `x jobs show`: key, last batch at, lag.
+
+import type { JobProgress } from './introspection';
+import type { AnyJobHandle } from './job';
+import { isJobHandle, registeredJobs } from './job';
+
+/** Everything `consumer()` knows that a `JobHandle` has no field for. */
+export interface ConsumerOrigin {
+  readonly supervisor: string;
+  readonly arm: string;
+  readonly cron: string;
+  readonly batch: number;
+  readonly lags: boolean;
+}
+
+/** One declaration as `--json` and the manifest report it. Plain JSON. */
+export interface ConsumerDeclaration {
+  readonly kind: 'consumer';
+  readonly name: string;
+  readonly queue: string;
+  /** The task that re-arms every key, and the job its tick enqueues. */
+  readonly supervisor: string;
+  readonly arm: string;
+  /** The supervisor's cron, in UTC: a re-arm interval has no wall-clock meaning. */
+  readonly rearm: string;
+  readonly batch: number;
+  /** Whether `lag` was declared, so a reader knows a `lag: null` is "not declared". */
+  readonly lags: boolean;
+}
+
+/** One consumer run, as `x jobs show <id> --json` reports it under `consumer`. */
+export interface ConsumerRow {
+  readonly key: string | null;
+  /** When the run last reported a committed batch; `null` before its first. */
+  readonly lastBatchAt: string | null;
+  /** `null` when no `lag` is declared or no batch has committed yet. */
+  readonly lag: number | null;
+}
+
+const origin = new WeakMap<object, ConsumerOrigin>();
+
+/** Called by `consumer()` and nothing else — not exported from `src/index.ts`. */
+export function stampConsumer(handle: AnyJobHandle, source: ConsumerOrigin): void {
+  origin.set(handle, source);
+}
+
+export function isConsumer(value: unknown): value is AnyJobHandle {
+  return isJobHandle(value) && origin.has(value);
+}
+
+/** Reads `handle.name` live: registration rebinds that property in place. */
+export function consumerDeclarationOf(handle: AnyJobHandle): ConsumerDeclaration | undefined {
+  const source = origin.get(handle);
+  if (source === undefined) return undefined;
+  return {
+    kind: 'consumer',
+    name: handle.name,
+    queue: handle.queue,
+    supervisor: source.supervisor,
+    arm: source.arm,
+    rearm: source.cron,
+    batch: source.batch,
+    lags: source.lags,
+  };
+}
+
+/** Every consumer this process declared, in `registeredJobs()`' code-unit order. */
+export function registeredConsumers(): readonly ConsumerDeclaration[] {
+  const declarations: ConsumerDeclaration[] = [];
+  for (const handle of registeredJobs()) {
+    const declaration = consumerDeclarationOf(handle);
+    if (declaration !== undefined) declarations.push(declaration);
+  }
+  return declarations;
+}
+
+const keyOf = (input: unknown): string | null => {
+  if (typeof input !== 'object' || input === null) return null;
+  const key = (input as { readonly key?: unknown }).key;
+  return typeof key === 'string' ? key : null;
+};
+
+/**
+ * The consumer row for one run, or `null` when `name` is no declared consumer. The loop reports
+ * `progress(items, items + lag)`, so the lag is `total - done` — and only when one is declared.
+ */
+export function consumerRow(run: {
+  readonly name: string;
+  readonly input: unknown;
+  readonly progress: JobProgress | null;
+}): ConsumerRow | null {
+  const handle = registeredJobs().find((candidate) => candidate.name === run.name);
+  const source = handle === undefined ? undefined : origin.get(handle);
+  if (source === undefined) return null;
+  const progress = run.progress;
+  return {
+    key: keyOf(run.input),
+    lastBatchAt: progress === null ? null : new Date(progress.at).toISOString(),
+    lag: !source.lags || progress === null ? null : progress.total - progress.done,
+  };
+}

package/src/consumer-types.ts

@@ -0,0 +1,81 @@
+// The `consumer()` declaration's shape and what it hands back. Apart from `consumer.ts` so the
+// factory, the turn loop (`consumer-loop.ts`) and the registry read one set of types.
+
+import type { Ctx } from '@ultimat3/core';
+import type { DurationInput } from './clock';
+import type { JobHandle } from './job';
+import type { RetryPolicy } from './retry';
+import type { TaskHandle } from './task';
+import type { JobTenant } from './tenant';
+
+/** One run per key: the payload is the key, and nothing else a queue row could drift on. */
+export interface ConsumerInput {
+  readonly key: string;
+}
+
+/**
+ * The app's own row, read and written through this seam — the reasoning `webhook-ledger.ts`
+ * states: the app owns the table. `save` runs INSIDE the batch's transaction, so a batch and the
+ * cursor past it commit together or not at all.
+ */
+export interface ConsumerCursor<C> {
+  /** Where `key` stopped. `null` before its first committed batch. */
+  load(key: string): Promise<C | null> | C | null;
+  save(key: string, next: C): Promise<void> | void;
+}
+
+export interface ConsumerBatch<Item> {
+  readonly items: readonly Item[];
+  readonly key: string;
+  readonly ctx: Ctx;
+  /** The run's cancellation — a drain aborts it. Hand it to whatever the body calls. */
+  readonly signal: AbortSignal;
+}
+
+export interface ConsumerDefinition<Item, C> {
+  /** REQUIRED: a durable queue key, the idempotency key's prefix and the supervisor's name. */
+  readonly name: string;
+  /** REQUIRED, as on `job()`: the org a key's run acts under, from the key — or `'none'`. */
+  readonly tenant: JobTenant<ConsumerInput>;
+  /**
+   * What should be running: one key, or one per account. Asked by the supervisor's arm run under
+   * `tenant: 'none'`, so a tenant-scoped read here needs the app's own `crossTenant(reason, fn)`.
+   */
+  keys(args: { readonly ctx: Ctx }): Promise<readonly string[]> | readonly string[];
+  /** The stream, from `cursor`. Any broker is the app's dependency, behind this one seam. */
+  open(args: {
+    readonly key: string;
+    readonly cursor: C | null;
+    readonly ctx: Ctx;
+    readonly signal: AbortSignal;
+  }): AsyncIterable<Item>;
+  /**
+   * One batch, inside a transaction, answering the cursor past it — which the framework hands to
+   * `cursor.save` in the SAME transaction. At least once: a drained or failed batch is delivered
+   * again, so the effect must be idempotent per item (item ids are the app's).
+   */
+  handle(batch: ConsumerBatch<Item>): Promise<C> | C;
+  readonly cursor: ConsumerCursor<C>;
+  /** Items per transaction, at most. Defaults to `DEFAULT_CONSUMER_BATCH`. */
+  readonly batch?: number;
+  readonly queue?: string;
+  readonly retry?: RetryPolicy;
+  /** How often the supervisor re-arms every key: whole minutes dividing an hour. Default `'1m'`. */
+  readonly rearm?: DurationInput;
+  /** How far behind `key` is, in the app's own unit. Reported on the run's progress. */
+  lag?(args: { readonly key: string; readonly cursor: C | null }): Promise<number> | number;
+  /** Refused (`X_CONSUMER_INVALID`): a consumer runs until its stream ends or a drain stops it. */
+  readonly timeout?: never;
+}
+
+/** What one run reports when its stream ends. Bounded, so `x jobs show` can print it. */
+export interface ConsumerReport {
+  readonly key: string;
+  readonly batches: number;
+  readonly items: number;
+}
+
+export interface ConsumerHandle extends JobHandle<ConsumerInput> {
+  /** The task re-arming every key at `rearm`. Registered at declaration under `<name>.supervisor`. */
+  readonly supervisor: TaskHandle;
+}

package/src/consumer.ts

@@ -0,0 +1,147 @@
+// `consumer()` — reading a queue, a stream or a vendor's cursor as a `job` plus a `.supervisor`
+// `task`: a factory over `job()`, never a ninth primitive. It assembles what a hand-rolled one must
+// remember and has no build error for — one body per key, a key per key, no timeout, a supervisor,
+// the drain, a cursor committed with its batch. The turn itself is `consumer-loop.ts`.
+
+import { t } from '@ultimat3/schema';
+import { finiteDurationMs } from './clock';
+import { ConsumerInvalidError } from './consumer-errors';
+import { asConsumer, refuseSelfEnqueue } from './consumer-guard';
+import { consumerRun, DEFAULT_CONSUMER_BATCH } from './consumer-loop';
+import { stampConsumer } from './consumer-registry';
+import type { ConsumerDefinition, ConsumerHandle, ConsumerInput } from './consumer-types';
+import type { EnqueueResult } from './driver';
+import type { JobHandle } from './job';
+import { job } from './job';
+import type { EnqueueManyOptions } from './job-enqueue-many';
+import type { EnqueueOptions } from './outbox';
+import { DEFAULT_RETRY } from './retry';
+import { task } from './task';
+
+const MINUTE_MS = 60_000;
+
+/** `'1m'` → `* * * * *`; whole minutes dividing an hour, so every tick is evenly spaced. */
+function rearmCron(name: string, rearm: string | number): string {
+  const ms = finiteDurationMs(rearm, `consumer "${name}"`, 'rearm');
+  const minutes = ms / MINUTE_MS;
+  if (!Number.isInteger(minutes) || minutes < 1 || 60 % minutes !== 0) {
+    throw new ConsumerInvalidError({
+      consumer: name,
+      problem: `declares rearm ${String(rearm)}, and the supervisor's cron ticks in whole minutes that divide an hour (1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30, 60)`,
+      fix: `rearm: '1m' on consumer({ name: "${name}" })`,
+    });
+  }
+  if (minutes === 60) return '0 * * * *';
+  return minutes === 1 ? '* * * * *' : `*/${minutes} * * * *`;
+}
+
+/** Every refusal a declaration can earn, before anything is registered under its name. */
+function assertDeclaration<Item, C>(definition: ConsumerDefinition<Item, C>): number {
+  const name = definition.name;
+  const cursor: unknown = definition.cursor;
+  const seam = (cursor ?? {}) as { readonly load?: unknown; readonly save?: unknown };
+  if (typeof seam.load !== 'function' || typeof seam.save !== 'function') {
+    throw new ConsumerInvalidError({
+      consumer: name,
+      problem:
+        'declares no cursor.load and cursor.save, so a restart would re-read its stream from the top and no batch could commit its position',
+      fix: `cursor: { load: (key) => repo.cursorOf(key), save: (key, next) => repo.saveCursor(key, next) } on consumer({ name: "${name}" })`,
+    });
+  }
+  if ((definition as { readonly timeout?: unknown }).timeout !== undefined) {
+    throw new ConsumerInvalidError({
+      consumer: name,
+      problem:
+        'declares a timeout, which kills a long-lived stream mid-batch on a schedule nobody chose; a run ends when its stream does, or when a drain stops it',
+      fix: `consumer({ name: "${name}", ...definition }) with the timeout field deleted`,
+    });
+  }
+  for (const field of ['keys', 'open', 'handle'] as const) {
+    if (typeof definition[field] !== 'function') {
+      throw new ConsumerInvalidError({
+        consumer: name,
+        problem: `declares no ${field}()`,
+        fix: `${field}: (args) => … on consumer({ name: "${name}" })`,
+      });
+    }
+  }
+  const size = definition.batch ?? DEFAULT_CONSUMER_BATCH;
+  if (!Number.isSafeInteger(size) || size < 1) {
+    throw new ConsumerInvalidError({
+      consumer: name,
+      problem: `declares batch ${String(size)}, and a batch is a whole number of items, at least one`,
+      fix: `consumer({ name: "${name}", batch: ${DEFAULT_CONSUMER_BATCH} })`,
+    });
+  }
+  return size;
+}
+
+export function consumer<Item, C>(definition: ConsumerDefinition<Item, C>): ConsumerHandle {
+  const size = assertDeclaration(definition);
+  const name = definition.name;
+  const cron = rearmCron(name, definition.rearm ?? '1m');
+  const queue = definition.queue === undefined ? {} : { queue: definition.queue };
+
+  const handle: JobHandle<ConsumerInput> = job<ConsumerInput>({
+    name,
+    input: t.object({ key: t.string.min(1) }),
+    tenant: definition.tenant,
+    // A key is live while its row is queued or running, so a re-arm of a running key is deduped:
+    // the supervisor is a no-op while a run is live and a restart once it is not.
+    idempotencyKey: ({ key }) => `consumer:${name}:${key}`,
+    // The backstop the key alone cannot be: two rows of one key never run at once.
+    concurrency: { key: ({ key }) => key, limit: 1, whenBusy: 'wait' },
+    retry: definition.retry ?? DEFAULT_RETRY,
+    ...queue,
+    run: (args) => asConsumer(name, () => consumerRun({ definition, size }, args)),
+  });
+
+  // The un-guarded enqueues, kept for the arm run below: it is a different job, and the only
+  // thing in the framework that enqueues a consumer.
+  const enqueue = handle.enqueue.bind(handle);
+  const enqueueMany = handle.enqueueMany.bind(handle);
+  // Guarded IN PLACE, never a wrapper object: only the object `job()` built passes `isJobHandle`.
+  Object.assign(handle, {
+    async enqueue(input: ConsumerInput, options?: EnqueueOptions): Promise<EnqueueResult> {
+      refuseSelfEnqueue(handle.name);
+      return enqueue(input, options);
+    },
+    async enqueueMany(inputs: readonly ConsumerInput[], options?: EnqueueManyOptions) {
+      refuseSelfEnqueue(handle.name);
+      return enqueueMany(inputs, options);
+    },
+  });
+
+  // `keys` may read the database, and a task's entries are synchronous — so the tick enqueues
+  // this one arm run, which asks `keys` and enqueues every key it answers.
+  const arm = job<Record<string, never>>({
+    name: `${name}.arm`,
+    input: t.object({}),
+    tenant: 'none',
+    idempotencyKey: () => `consumer:${name}:arm`,
+    retry: DEFAULT_RETRY,
+    ...queue,
+    async run({ ctx, step }) {
+      const keys = await definition.keys({ ctx });
+      // A step: an arm killed after its enqueue and before its ack replays the result rather than
+      // enqueuing the keys again behind a consumer that already finished.
+      const results = await step.run('start-keys', () => enqueueMany(keys.map((key) => ({ key }))));
+      return { keys: keys.length, started: results.filter((result) => !result.deduped).length };
+    },
+  });
+  const supervisor = task({
+    name: `${name}.supervisor`,
+    cron,
+    tz: 'UTC',
+    enqueue: () => [[arm, {}]],
+  });
+
+  stampConsumer(handle, {
+    supervisor: supervisor.name,
+    arm: arm.name,
+    cron,
+    batch: size,
+    lags: definition.lag !== undefined,
+  });
+  return Object.assign(handle, { supervisor });
+}

package/src/driver-memory-operator.ts

@@ -29,7 +29,8 @@ import {
   pageLimit,
   parseJobCursor,
 } from './introspection';
-import type { StepStore } from './steps';
+import { assertResolvable, resolvedRecord } from './step-resolve';
+import type { StepRecord, StepStore } from './steps';
 
 export interface MemoryQueueState {
   readonly jobs: Map<string, JobRecord>;
@@ -51,6 +52,8 @@ export interface MemoryOperator {
   readonly counters: MemoryCounters;
   /** The claim scan's question: is this queue held back? */
   isQueuePaused(queue: string): boolean;
+  /** Does a live registry row say this worker drains? `SQL_QUEUE_DEMAND`'s `not exists`. */
+  isWorkerDraining(workerId: string): boolean;
 }
 
 /** Newest first by `(createdAt, id)` — the keyset `SQL_JOB_LIST` orders and seeks by. */
@@ -153,6 +156,18 @@ export function memoryOperator(state: MemoryQueueState): MemoryOperator {
         : matching.slice(Math.max(0, matching.length - limit));
     },
 
+    async resolveStep(jobId, step, resolution) {
+      const before = jobs.get(jobId);
+      const record = before === undefined ? undefined : await state.steps.get(before.runId, step);
+      // Read again after the await, as `SQL_STEP_RESOLVE` reads the row it writes under: a claim
+      // that landed in between makes the job a worker's.
+      const job = jobs.get(jobId);
+      assertResolvable('memory', jobId, step, job, record);
+      const resolved = resolvedRecord(record as StepRecord, resolution, nowMs(clock));
+      await state.steps.put(resolved);
+      return resolved;
+    },
+
     async remove(jobId) {
       const record = jobs.get(jobId);
       if (record === undefined) return undefined;
@@ -320,5 +335,13 @@ export function memoryOperator(state: MemoryQueueState): MemoryOperator {
     rollupCounters: () => Promise.resolve(counters.rollup(nowMs(clock))),
   };
 
-  return { members, counters, isQueuePaused: (queue) => queues.has(queue) };
+  return {
+    members,
+    counters,
+    isQueuePaused: (queue) => queues.has(queue),
+    isWorkerDraining: (workerId) => {
+      const worker = workers.get(workerId);
+      return worker?.drainingAt !== undefined && worker.expiresAt > nowMs(clock);
+    },
+  };
 }

package/src/driver-memory.ts

@@ -19,11 +19,13 @@ import type {
   JobDriver,
   JobRecord,
   NackOptions,
+  QueueDemand,
   QueueStats,
 } from './driver';
 import {
   assertClaimBounds,
   assertClaimQueues,
+  assertEnqueueMany,
   DEFAULT_QUEUE,
   LEASE_LAPSED_FINAL_ATTEMPT,
   LIVE_STATES,
@@ -219,7 +221,7 @@ export function memoryJobDriver(options: MemoryJobDriverOptions = {}): MemoryJob
     },
   };
 
-  return {
+  const driver: MemoryJobDriver = {
     name: 'memory',
     steps,
     backfills,
@@ -275,6 +277,15 @@ export function memoryJobDriver(options: MemoryJobDriverOptions = {}): MemoryJob
       return { id: record.id, runId: record.runId, deduped: false };
     },
 
+    // A loop of this driver's own `enqueue`, which IS its semantics: in order, so a second request
+    // under a key the first just took is `deduped`, as one `SQL_ENQUEUE_MANY` statement answers.
+    async enqueueMany(requests: readonly EnqueueRequest[]): Promise<readonly EnqueueResult[]> {
+      assertEnqueueMany('memory', requests);
+      const results: EnqueueResult[] = [];
+      for (const request of requests) results.push(await driver.enqueue(request));
+      return results;
+    },
+
     // `async`, so an empty queue list REJECTS here exactly as it does on the pg driver: a
     // synchronous throw out of a method typed `Promise<…>` is a different answer to the same
     // question, which is the class of divergence this pair is checked for.
@@ -431,9 +442,32 @@ export function memoryJobDriver(options: MemoryJobDriverOptions = {}): MemoryJob
       );
     },
 
+    // `SQL_QUEUE_DEMAND` over these rows. A `running` row is never "due": it counts on its claimer.
+    demand(): Promise<readonly QueueDemand[]> {
+      const at = nowMs(clock);
+      const byQueue = new Map<string, number>();
+      for (const record of jobs.values()) {
+        const held =
+          record.state === 'running' && !operator.isWorkerDraining(record.claimedBy ?? '');
+        const due =
+          (record.state === 'ready' ||
+            record.state === 'delayed' ||
+            record.state === 'suspended') &&
+          record.runAt <= at &&
+          !operator.isQueuePaused(record.queue);
+        if (held || due) byQueue.set(record.queue, (byQueue.get(record.queue) ?? 0) + 1);
+      }
+      return Promise.resolve(
+        [...byQueue]
+          .map(([queue, demand]) => ({ queue, demand }))
+          .sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0)),
+      );
+    },
+
     close(): Promise<void> {
       jobs.clear();
       return Promise.resolve();
     },
   };
+  return driver;
 }

package/src/driver-pg-claim-sql.ts

@@ -0,0 +1,128 @@
+// The claim: the one statement every worker loop runs, and the only one whose cost is a question of
+// how far behind the queue is. Apart from `driver-pg-sql.ts`, which re-exports it, because the plan
+// it depends on takes a page to explain. `driver-pg-claim.live.test.ts` holds that plan to account.
+
+import { LEASE_LAPSED_FINAL_ATTEMPT } from './driver';
+import { COUNTER_BUCKET_MS } from './introspection';
+
+/** What both arms of the claim return: a `JobRow`, read off the updated row `j`. */
+const CLAIM_ROW_COLUMNS = `j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt,
+          j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by, j.claims,
+          j.traceparent, j.enqueued_by,
+          (extract(epoch from j.run_at)     * 1000)::bigint as run_at,
+          (extract(epoch from j.visible_at) * 1000)::bigint as visible_at,
+          (extract(epoch from j.created_at) * 1000)::bigint as created_at,
+          (extract(epoch from j.updated_at) * 1000)::bigint as updated_at`;
+
+const CLAIM_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attempt, max_attempts,
+       state, tenant_id, last_error, claimed_by, claims, traceparent, enqueued_by, run_at,
+       visible_at, created_at, updated_at`;
+
+/**
+ * The claim. SKIP LOCKED is the whole design: without it, worker 2 blocks on worker 1's row
+ * lock and throughput collapses to one worker. `visible_at` in the predicate reclaims leases
+ * abandoned by a crashed worker. `cancelled` is absent from every branch by construction — a
+ * cancelled row is terminal, so it is never handed to a worker again.
+ *
+ * TWO arms over one pick. A `running` row is picked only with a lapsed lease, and one that lapsed
+ * on its FINAL attempt (`attempt >= max_attempts` — `isFinalAttempt`, `retry.ts`) is `buried`:
+ * settled `dead` here, counted, and answered with its new state so the driver reports it instead
+ * of handing it out. Re-claimed, it was an attempt past `max_attempts` every lease, forever — a
+ * job that kills its worker took one per visibility timeout and never reached the dead letters.
+ * `claimed_by` is cleared with the lease, so the body that stalled rather than died cannot settle
+ * the row it lost. A buried row spends a slot of `$2`: the pass after it claims what it displaced.
+ * `$5` names the jobs declaring `retry.deadLetter: false`: their rows are buried `failed`.
+ *
+ * `picked` SEEKS, so a claim costs the same at any backlog. One `state in (...) or running` test
+ * over `queue = any(...)` cannot be served in `run_at` order by any index, so it read and sorted
+ * every due row, and a queue slowed down exactly when it was behind. Hence two ordered index scans
+ * per open queue, due rows and lapsed leases, then an outer sort of at most `2 x limit x queues`
+ * rows. The due scan's `state in` is `x_jobs_claim_idx`'s predicate word for word, or the planner
+ * cannot prove the index applies; the lapsed scan reads `x_jobs_depth_idx`. Measured 2026-11-02,
+ * Postgres 17, 1M `done` rows, a backlog of 100k, shared box (read the ratios): one queue
+ * 88-122 ms against 0.8 ms; three queues 130-189 ms against 0.7-0.9 ms; a noop `jobWorker` on 8
+ * slots 42-55 jobs/s against 443-493. At a backlog of 1,000 the two are level.
+ *
+ * Rows a scan locked and the outer `limit` dropped stay locked until the TRANSACTION ends, not the
+ * statement. That is correct only because a claim autocommits (`exec()` in `driver-pg.ts` is the
+ * pool), so the two coincide; a concurrent claimer may skip them for one poll, which `skip locked`
+ * already permits. A claim inside a long transaction would hold them for its whole life.
+ *
+ * `distinct` because a queue named twice is two laterals over one queue, and a row this statement
+ * already locked is not skipped: the repeat would spend `limit` on rows it had already picked.
+ */
+export const SQL_CLAIM = `
+with open as (
+  select distinct q.queue
+    from unnest($1::text[]) as q(queue)
+   where not exists (
+     select 1 from x_job_pauses p where p.kind = 'queue' and p.name = q.queue
+   )
+), picked as (
+  select id, exhausted
+    from (
+      select due.id, due.run_at, due.exhausted
+        from open
+       cross join lateral (
+         select id, run_at, false as exhausted
+           from x_jobs
+          where queue = open.queue
+            and state in ('ready', 'delayed', 'suspended')
+            and run_at <= now()
+          order by run_at
+          limit $2
+            for update skip locked
+       ) due
+      union all
+      select lapsed.id, lapsed.run_at, lapsed.exhausted
+        from open
+       cross join lateral (
+         select id, run_at, attempt >= max_attempts as exhausted
+           from x_jobs
+          where queue = open.queue
+            and state = 'running' and visible_at <= now()
+            and run_at <= now()
+          order by run_at
+          limit $2
+            for update skip locked
+       ) lapsed
+    ) candidates
+   order by run_at
+   limit $2
+), buried as (
+  update x_jobs j
+     set state            = case when j.name = any($5::text[]) then 'failed' else 'dead' end,
+         visible_at       = null,
+         claimed_by       = null,
+         last_error       = '${LEASE_LAPSED_FINAL_ATTEMPT}',
+         last_error_stack = null,
+         updated_at       = now()
+    from picked p
+   where j.id = p.id and p.exhausted
+  returning ${CLAIM_ROW_COLUMNS}
+), counted as (
+  insert into x_job_counters (job, bucket_ms, bucket_start, dead, failed, duration_ms)
+  select name, ${COUNTER_BUCKET_MS}, date_trunc('minute', now()),
+         (count(*) filter (where state = 'dead'))::int,
+         (count(*) filter (where state = 'failed'))::int, 0
+    from buried
+   group by name
+  on conflict (job, bucket_ms, bucket_start) do update
+     set dead   = x_job_counters.dead + excluded.dead,
+         failed = x_job_counters.failed + excluded.failed
+), claimed as (
+  update x_jobs j
+     set state      = 'running',
+         attempt    = j.attempt + 1,
+         claims     = j.claims + 1,
+         claimed_by = $3,
+         visible_at = now() + ($4::bigint * interval '1 millisecond'),
+         updated_at = now()
+    from picked p
+   where j.id = p.id and not p.exhausted
+  returning ${CLAIM_ROW_COLUMNS}
+)
+select ${CLAIM_ROW_NAMES} from claimed
+union all
+select ${CLAIM_ROW_NAMES} from buried
+`.trim();

package/src/driver-pg-ddl.ts

@@ -194,6 +194,11 @@ create table if not exists x_job_events (
 create index if not exists x_job_events_lookup_idx
   on x_job_events (name, published_at);
 
+-- The steps parked on an event, which is what a waking publish reads. Partial, so it holds only
+-- the open waits: without it each publish is a scan of every step ever written.
+create index if not exists x_job_steps_waiting_idx
+  on x_job_steps (event) where status = 'waiting';
+
 -- What the body last reported through progress, as one value: done, total, note and the instant.
 -- A column and not a table, because it is read with the row and dies with it. Added by alter
 -- because x_jobs shipped without it.
@@ -257,11 +262,15 @@ create table if not exists x_job_workers (
 );
 
 -- When the worker began draining (a retire, a SIGTERM, a stop) - set by its first draining
--- announce and kept by every later one, null while it serves. A queue-depth autoscaler leaves the
--- jobs such a worker still holds out of its count (SQL_SCALER_DEPTH): its replacement cannot take
+-- announce and kept by every later one, null while it serves. A queue-demand autoscaler leaves the
+-- jobs such a worker still holds out of its count (SQL_SCALER_DEMAND): its replacement cannot take
 -- them. Added by alter because x_job_workers shipped without it.
 alter table x_job_workers add column if not exists draining_at timestamptz;
 
+-- The build the worker runs, the id x-ultimate-build answers with. Null on a row a worker from
+-- before this column wrote. x deploy verify counts a queue as served only by a row of this build.
+alter table x_job_workers add column if not exists build text;
+
 -- Per-job history, in buckets. A settle adds to the one-minute bucket in the SAME statement that
 -- moves the row, and the scheduler leader folds old buckets into wider ones, so the key count per
 -- job name is fixed whatever the throughput.

package/src/driver-pg-demand-sql.ts

@@ -0,0 +1,27 @@
+// `queue_demand` on Postgres: which `x_jobs` rows a serving worker of the fleet must hold, and that
+// count per queue. One predicate, because `SQL_SCALER_DEMAND` (a scaler reading Postgres itself) and
+// the gauge answer one question. Only Postgres knows who drains, so no sum of gauges can replace it.
+
+/**
+ * A `running` row whose claimer's live registry row says `draining_at` is out: that worker claims
+ * nothing more and finishes the job itself, so no pod a scaler adds could take it. One adopter
+ * counted them and averaged 1.84 pods where one carried the load. A claimer with no live row
+ * (killed, lapsed) stays in: the queue hands that job back and somebody must pick it up.
+ */
+export const QUEUE_DEMAND_ROWS = `((j.state = 'running'
+         and not exists (select 1 from x_job_workers w
+                          where w.id = j.claimed_by
+                            and w.draining_at is not null
+                            and w.expires_at > now()))
+     or (j.state in ('ready', 'delayed', 'suspended') and j.run_at <= now()
+         and not exists (select 1 from x_job_pauses p
+                          where p.kind = 'queue' and p.name = j.queue)))`;
+
+/** Grouped, every 15 s from a worker with work about: an idle one never asks (`worker-queue-demand.ts`). */
+export const SQL_QUEUE_DEMAND = `
+select j.queue, count(*)::int as demand
+  from x_jobs j
+ where ${QUEUE_DEMAND_ROWS}
+ group by j.queue
+ order by j.queue collate "C"
+`.trim();