@ultimat3/jobs 29.4.0 → 30.1.0

package/src/driver-pg-enqueue-many.ts

@@ -0,0 +1,153 @@
+// The Postgres driver's `enqueueMany`: every request in ONE insert over `jsonb_array_elements`,
+// with `SQL_ENQUEUE`'s conflict target and its `not exists` on the id, then one read for the rows
+// that insert did not return. Split from `driver-pg.ts` at that file's size ceiling.
+
+import type { Clock, PgExecutor } from '@ultimat3/core';
+import { uuidV7 } from '@ultimat3/core';
+import { nowMs } from './clock';
+import type { EnqueueRequest, EnqueueResult } from './driver';
+import { assertEnqueueMany, DEFAULT_QUEUE } from './driver';
+import { notifyJobReady } from './driver-pg-wake-sql';
+
+/**
+ * Each request is one element of `$1`, in request order. The insert is `SQL_ENQUEUE`'s, row for
+ * row: the same conflict target spelling the index expression, and the same `not exists` on the
+ * id — without it a relayed batch repeated after a crash runs every job that already FINISHED a
+ * second time, because the partial index collapses a repeat only while its job is live.
+ *
+ * Two requests sharing a key inside the batch: `order by ord` inserts the first, and `do nothing`
+ * skips the second against the row the same statement just wrote.
+ *
+ * `runAt` crosses as `numeric` and is rounded by its `::bigint`, the instant `SQL_ENQUEUE`'s
+ * `$8::numeric::bigint` stores: JSON text such as `1767225660000.5` is no `bigint` literal, so a fractional
+ * epoch ms — `now + Math.random() * 1000` — was a raw `22P02` here and fine one row at a time.
+ *
+ * ONE wake per queue: only the earliest-due row of each queue asks `notifyJobReady`. A batch of
+ * 1,000 rows is one queue becoming claimable, not 1,000 announcements of it.
+ */
+export const SQL_ENQUEUE_MANY = `
+with req as (
+  select (r->>'id')::uuid as id, r->>'name' as name, r->>'queue' as queue, r->'input' as input,
+         r->>'idempotencyKey' as idempotency_key, (r->>'runId')::uuid as run_id,
+         (r->>'maxAttempts')::int as max_attempts,
+         to_timestamp((r->>'runAt')::numeric::bigint / 1000.0) as run_at,
+         r->>'tenantId' as tenant_id, r->>'traceparent' as traceparent,
+         r->>'enqueuedBy' as enqueued_by, t.ord
+    from jsonb_array_elements($1::jsonb) with ordinality as t(r, ord)
+), inserted as (
+  insert into x_jobs
+    (id, name, queue, input, idempotency_key, run_id, max_attempts, state, run_at, tenant_id,
+     traceparent, enqueued_by)
+  select q.id, q.name, q.queue, q.input, q.idempotency_key, q.run_id, q.max_attempts,
+         case when q.run_at > now() then 'delayed' else 'ready' end,
+         q.run_at, q.tenant_id, q.traceparent, q.enqueued_by
+    from req q
+   where not exists (select 1 from x_jobs published where published.id = q.id)
+   order by q.ord
+  on conflict (name, (coalesce(tenant_id, '')), idempotency_key)
+    where state in ('ready', 'delayed', 'running', 'suspended')
+    do nothing
+  returning id, run_id, queue, run_at
+)
+select r.id, r.run_id,
+       case when r.first then${notifyJobReady('r')}::text end as woke
+  from (select i.id, i.run_id, i.queue, i.run_at,
+               row_number() over (partition by i.queue order by i.run_at, i.id) = 1 as first
+          from inserted i) r
+`.trim();
+
+/**
+ * Who answers a request the insert skipped, one row per request: the job under its own id — a
+ * publish repeated, `pick` 0 — or the LIVE holder of its key in its name and tenant, `pick` 1,
+ * which `SQL_FIND_LIVE_BY_KEY` reads one at a time. A request neither finds is a holder that
+ * settled between the two statements, and goes back through `enqueue`, which retries it.
+ */
+export const SQL_ENQUEUE_MANY_RESOLVE = `
+select (t.r->>'ord')::int as ord, x.id, x.run_id
+  from jsonb_array_elements($1::jsonb) as t(r)
+  cross join lateral (
+    select c.id, c.run_id, c.pick from (
+      select j.id, j.run_id, 0 as pick from x_jobs j where j.id = (t.r->>'id')::uuid
+      union all
+      select j.id, j.run_id, 1 as pick from x_jobs j
+       where j.name = t.r->>'name'
+         and j.idempotency_key = t.r->>'idempotencyKey'
+         and coalesce(j.tenant_id, '') = coalesce(t.r->>'tenantId', '')
+         and j.state in ('ready', 'delayed', 'running', 'suspended')
+    ) c
+    order by c.pick
+    limit 1
+  ) x
+`.trim();
+
+interface Inserted {
+  readonly id: string;
+  readonly run_id: string;
+}
+
+interface Resolved extends Inserted {
+  readonly ord: number | string;
+}
+
+/** A request with its ids settled: the batch is read back by id, so every row needs one. */
+type Stated = EnqueueRequest & { readonly id: string; readonly runId: string };
+
+const wire = (request: Stated, at: number) => ({
+  id: request.id,
+  runId: request.runId,
+  name: request.name,
+  queue: request.queue || DEFAULT_QUEUE,
+  input: request.input ?? null,
+  idempotencyKey: request.idempotencyKey,
+  maxAttempts: request.maxAttempts,
+  runAt: request.runAt ?? at,
+  tenantId: request.tenantId,
+  traceparent: request.traceparent,
+  enqueuedBy: request.enqueuedBy,
+});
+
+/**
+ * `enqueueMany` over `exec`. `enqueueOne` is the driver's own `enqueue`, for the one request
+ * whose answer moved between the two statements. Answers in request order.
+ */
+export async function pgEnqueueMany(
+  exec: () => PgExecutor,
+  clock: Clock,
+  requests: readonly EnqueueRequest[],
+  enqueueOne: (request: EnqueueRequest) => Promise<EnqueueResult>,
+): Promise<readonly EnqueueResult[]> {
+  assertEnqueueMany('pg', requests);
+  if (requests.length === 0) return [];
+  const at = nowMs(clock);
+  const stated: Stated[] = requests.map((request) => ({
+    ...request,
+    id: request.id ?? uuidV7(),
+    runId: request.runId ?? uuidV7(),
+  }));
+  const rows = await exec().query<Inserted>(SQL_ENQUEUE_MANY, [
+    JSON.stringify(stated.map((request) => wire(request, at))),
+  ]);
+  const inserted = new Map(rows.map((row) => [row.id, row.run_id]));
+  const results: (EnqueueResult | undefined)[] = stated.map((request) => {
+    const runId = inserted.get(request.id);
+    return runId === undefined ? undefined : { id: request.id, runId, deduped: false };
+  });
+  const skipped = stated.flatMap((request, ord) =>
+    results[ord] === undefined ? [{ ...wire(request, at), ord }] : [],
+  );
+  if (skipped.length === 0) return results.filter((result) => result !== undefined);
+
+  const found = await exec().query<Resolved>(SQL_ENQUEUE_MANY_RESOLVE, [JSON.stringify(skipped)]);
+  for (const row of found) {
+    const ord = Number(row.ord);
+    if (stated[ord] === undefined) continue;
+    results[ord] = { id: row.id, runId: row.run_id, deduped: true };
+  }
+  const answered: EnqueueResult[] = [];
+  for (const [ord, result] of results.entries()) {
+    const request = stated[ord];
+    if (result !== undefined) answered.push(result);
+    else if (request !== undefined) answered.push(await enqueueOne(request));
+  }
+  return answered;
+}

package/src/driver-pg-event-wake-sql.ts

@@ -0,0 +1,51 @@
+// The publish that wakes its waiters: one statement inserts the event, makes every run parked on
+// it due, and tells each of their queues. One statement, so a rollback undoes all three, and the
+// NOTIFY is delivered at commit only. What `postgresEventBus({ wakes: true })` runs.
+
+import { JOBS_WAKE_CHANNEL } from './driver-pg-wake-sql';
+
+/**
+ * `SQL_EVENT_PUBLISH`, plus the wake. A waiter is a `waiting` step on this event whose key is this
+ * event's or absent — `SQL_EVENT_FIND`'s match, read from the step's side — over the partial
+ * `x_job_steps_waiting_idx`, so a publish with no waiter reads no step at all.
+ *
+ * Woken means `run_at` moved to now on a `suspended` row that was not already due: a running row,
+ * a finished one, or one a concurrent publish already woke (the row lock re-checks `run_at` after
+ * the first commit) is left alone. The poll stays the guarantee; this only brings it forward.
+ *
+ * One `pg_notify` per DISTINCT queue of the woken rows, and none when nothing woke: never
+ * unconditional, for the commit lock `driver-pg-wake-sql.ts` describes. The payload is the queue.
+ *
+ * $1 id, $2 name, $3 payload, $4 correlation key, $5 ttl (ms).
+ */
+export const SQL_EVENT_PUBLISH_WAKE = `
+with ev as (
+  insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at)
+  values ($1, $2, $3::jsonb, $4, statement_timestamp(),
+          statement_timestamp() + ($5::bigint * interval '1 millisecond'))
+  returning published_at, expires_at
+), waiters as (
+  select distinct s.run_id
+    from x_job_steps s
+   where s.status = 'waiting'
+     and s.event = $2
+     and (s.correlation_key is null or s.correlation_key = $4::text)
+), woken as (
+  update x_jobs j
+     set run_at     = statement_timestamp(),
+         updated_at = statement_timestamp()
+    from waiters w
+   where j.run_id = w.run_id
+     and j.state = 'suspended'
+     and j.run_at > statement_timestamp()
+  returning j.queue
+), told as (
+  select count(pg_notify('${JOBS_WAKE_CHANNEL}', q.queue))::int as told
+    from (select distinct queue from woken) q
+)
+select floor(extract(epoch from ev.published_at) * 1000)::bigint as published_at,
+       floor(extract(epoch from ev.expires_at)   * 1000)::bigint as expires_at,
+       (select count(*) from woken)::int as woken,
+       (select told from told) as told
+  from ev
+`.trim();

package/src/driver-pg-operator-sql.ts

@@ -148,19 +148,22 @@ select name, (extract(epoch from paused_at) * 1000)::bigint as paused_at
 
 /**
  * A worker's heartbeat: the whole row rewritten, expiry pushed out. $1 id, $2 host, $3 queues,
- * $4 slots, $5 in-flight ids, $6 started (ms), $7 ttl (ms), $8 draining. `draining_at` is stamped by
- * the first draining announce and KEPT by the later ones — when the drain began, not the last beat.
+ * $4 slots, $5 in-flight ids, $6 started (ms), $7 ttl (ms), $8 draining, $9 build. `draining_at` is
+ * stamped by the first draining announce and KEPT by the later ones — when the drain began, not the
+ * last beat.
  */
 export const SQL_WORKER_ANNOUNCE = `
 insert into x_job_workers
-  (id, host, queues, concurrency, in_flight, started_at, heartbeat_at, expires_at, draining_at)
+  (id, host, queues, concurrency, in_flight, started_at, heartbeat_at, expires_at, draining_at,
+   build)
 values
   ($1, $2, $3::jsonb, $4::int, $5::jsonb, to_timestamp($6::bigint / 1000.0), now(),
-   now() + ($7::bigint * interval '1 millisecond'), case when $8::boolean then now() end)
+   now() + ($7::bigint * interval '1 millisecond'), case when $8::boolean then now() end,
+   $9::text)
 on conflict (id) do update
    set host = excluded.host, queues = excluded.queues, concurrency = excluded.concurrency,
        in_flight = excluded.in_flight, heartbeat_at = excluded.heartbeat_at,
-       expires_at = excluded.expires_at,
+       expires_at = excluded.expires_at, build = excluded.build,
        draining_at = case when $8::boolean
                           then coalesce(x_job_workers.draining_at, excluded.draining_at) end
 `.trim();
@@ -177,7 +180,7 @@ export const SQL_WORKERS = `
 with lapsed as (
   delete from x_job_workers where expires_at <= now()
 )
-select id, host, queues, concurrency, in_flight,
+select id, host, queues, concurrency, in_flight, build,
        (extract(epoch from started_at)   * 1000)::bigint as started_at,
        (extract(epoch from heartbeat_at) * 1000)::bigint as heartbeat_at,
        (extract(epoch from draining_at)  * 1000)::bigint as draining_at
@@ -325,3 +328,28 @@ select (select count(*) from moved)::int as fired, q.id, q.run_id, q.idempotency
   from (select 1) as one
   left join queued q on true
 `.trim();
+
+/**
+ * `resolveStep`, ONE write: the step becomes `completed` with `$4` (ran) or `cleared` (did not),
+ * only while its job is held by no worker and the step started or failed without completing. The
+ * job is read in the same statement, so a claim that lands first wins and nothing is written.
+ *
+ * $1 job id, $2 step name, $3 'completed' or 'cleared', $4 the output (jsonb text).
+ */
+export const SQL_STEP_RESOLVE = `
+with job as (
+  select run_id from x_jobs where id = $1::uuid and state <> 'running'
+)
+update x_job_steps s
+   set status       = $3,
+       output       = $4::jsonb,
+       completed_at = case when $3 = 'completed' then statement_timestamp() else null end,
+       error        = null
+  from job
+ where s.run_id = job.run_id and s.name = $2 and s.status in ('started', 'failed')
+returning s.run_id, s.name, s.status, s.output, s.attempts, s.error,
+          (extract(epoch from s.started_at)   * 1000)::bigint as started_at,
+          (extract(epoch from s.completed_at) * 1000)::bigint as completed_at,
+          (extract(epoch from s.wake_at)      * 1000)::bigint as wake_at,
+          s.event, s.correlation_key
+`.trim();

package/src/driver-pg-operator.ts

@@ -21,16 +21,20 @@ import {
   SQL_PAUSE,
   SQL_PAUSED,
   SQL_RESUME,
+  SQL_STEP_RESOLVE,
   SQL_TASK_FIRE_RECORD,
   SQL_TASK_FIRES,
   SQL_WORKER_ANNOUNCE,
   SQL_WORKER_FORGET,
   SQL_WORKERS,
 } from './driver-pg-operator-sql';
-import type { JobRow } from './driver-pg-rows';
-import { num, toJobRecord } from './driver-pg-rows';
+import type { JobRow, StepRow } from './driver-pg-rows';
+import { num, toJobRecord, toStepRecord } from './driver-pg-rows';
+import { SQL_STEP_GET } from './driver-pg-sql';
 import { JOBS_WAKE_CHANNEL, SQL_WAKE } from './driver-pg-wake-sql';
 import { JobNotRemovableError } from './errors-operator';
+import { JobNotFoundError } from './errors-requeue';
+import { StepNotResolvableError } from './errors-saga';
 import type {
   BulkFilter,
   BulkResult,
@@ -47,6 +51,7 @@ import {
   pageLimit,
   parseJobCursor,
 } from './introspection';
+import { assertResolvable } from './step-resolve';
 
 export type PgOperatorMembers = Omit<
   JobIntrospection,
@@ -152,6 +157,32 @@ export function pgOperator(
       return rows.map(toJobRecord);
     },
 
+    async resolveStep(jobId, step, resolution) {
+      // Read first, so an id nobody queued — or no uuid at all — is `X_JOB_NOT_FOUND` as on memory.
+      const before = await job(jobId);
+      if (before === undefined) throw new JobNotFoundError({ jobId, driver: 'pg' });
+      const rows = await exec().query<StepRow>(SQL_STEP_RESOLVE, [
+        jobId,
+        step,
+        resolution.ran ? 'completed' : 'cleared',
+        resolution.ran ? JSON.stringify(resolution.output ?? null) : null,
+      ]);
+      const written = rows[0];
+      if (written !== undefined) return toStepRecord(written);
+      // Nothing written: say why, from what the rows hold now — the memory driver's refusals.
+      const current = await job(jobId);
+      const found = await exec().query<StepRow>(SQL_STEP_GET, [before.runId, step]);
+      const record = found[0] === undefined ? undefined : toStepRecord(found[0]);
+      assertResolvable('pg', jobId, step, current, record);
+      // Resolvable now and not a moment ago: a worker held the job, and has let go since.
+      throw new StepNotResolvableError({
+        jobId,
+        step,
+        state: 'running',
+        status: record?.status ?? 'missing',
+      });
+    },
+
     async remove(jobId) {
       const current = await job(jobId);
       if (current === undefined) return undefined;
@@ -252,6 +283,7 @@ export function pgOperator(
         worker.startedAt,
         finiteOption('the pg worker registry', 'ttlMs', ttlMs),
         worker.draining === true,
+        worker.build ?? null,
       ]);
     },
     async forgetWorker(workerId) {
@@ -267,6 +299,7 @@ export function pgOperator(
         started_at: Count;
         heartbeat_at: Count;
         draining_at: Count | null;
+        build: string | null;
       }>(SQL_WORKERS, []);
       return rows.map((row) => ({
         id: row.id,
@@ -277,6 +310,7 @@ export function pgOperator(
         inFlight: stringList(row.in_flight),
         heartbeatAt: num(row.heartbeat_at),
         ...(row.draining_at === null ? {} : { drainingAt: num(row.draining_at) }),
+        ...(row.build === null ? {} : { build: row.build }),
       }));
     },
 

package/src/driver-pg-outbox-sql.ts

@@ -20,6 +20,31 @@ select id,${NOTIFY_STAGED}::text as woke
   from staged
 `.trim();
 
+/**
+ * `SQL_OUTBOX_STAGE` for a whole chunk: each element of `$1` one row, in order, and the wake ONCE
+ * for the statement rather than once per row — the same `NOTIFY_STAGED` clause, on the caller's
+ * connection, so it still commits or vanishes with the business rows. `staged_at` is shared by
+ * the chunk and `id` (a UUIDv7) orders it, which is the claim's total key. Both instants cross as
+ * `numeric`, never `bigint`: a fractional epoch ms is no `bigint` literal, and `SQL_OUTBOX_STAGE`
+ * takes one — a `22P02` here failed the caller's whole transaction.
+ */
+export const SQL_OUTBOX_STAGE_MANY = `
+with staged as (
+  insert into x_outbox
+    (id, job, queue, input, idempotency_key, max_attempts, run_at, staged_at, tenant_id,
+     traceparent, enqueued_by, run_id)
+  select (r->>'id')::uuid, r->>'job', r->>'queue', r->'input', r->>'idempotencyKey',
+         (r->>'maxAttempts')::int, to_timestamp((r->>'runAt')::numeric / 1000.0),
+         to_timestamp((r->>'stagedAt')::numeric / 1000.0), r->>'tenantId', r->>'traceparent',
+         r->>'enqueuedBy', (r->>'runId')::uuid
+    from jsonb_array_elements($1::jsonb) with ordinality as t(r, ord)
+   order by t.ord
+  returning id
+)
+select count(*)::bigint as staged,${NOTIFY_STAGED}::text as woke
+  from staged
+`.trim();
+
 /**
  * The claim, and it has to be ONE statement. `for update skip locked` in a bare select holds its
  * row locks only until that statement ends — under autocommit, before `claim()` even resolves — so
@@ -97,6 +122,15 @@ update x_outbox set published_at = to_timestamp($2 / 1000.0)
  where id = $1 and published_at is null and claimed_by = $3
 `.trim();
 
+/**
+ * `SQL_OUTBOX_MARK_PUBLISHED` for a published batch: the same two fences, row for row, so a
+ * lapsed claimant's batch mark retires none of the rows a newer claimant now holds.
+ */
+export const SQL_OUTBOX_MARK_PUBLISHED_MANY = `
+update x_outbox set published_at = to_timestamp($2 / 1000.0)
+ where id = any($1::uuid[]) and published_at is null and claimed_by = $3
+`.trim();
+
 /**
  * The job a staged row's publish made, by the id the two share. Read when `SQL_ENQUEUE` inserted
  * nothing for a request that named its id: the answer is that publish, repeated.

package/src/driver-pg-sql.ts

@@ -7,7 +7,9 @@
 // it is applied once at boot and never by a driver method. Re-exported from here so the install
 // point keeps the ONE import path every caller already uses.
 
+export { SQL_CLAIM } from './driver-pg-claim-sql';
 export { SQL_JOBS_TABLE } from './driver-pg-ddl';
+export { SQL_ENQUEUE_MANY, SQL_ENQUEUE_MANY_RESOLVE } from './driver-pg-enqueue-many';
 // The whole-`x_jobs`-row reads, split off at this file's size ceiling and re-exported for the
 // same reason the DDL is. `JOB_ROW_COLUMNS` is imported as a value because `SQL_CANCEL` also
 // returns a whole row and must project it identically — two spellings of one row shape is how
@@ -20,34 +22,20 @@ export {
   SQL_JOB_LIVE_HOLDER,
   SQL_JOB_REQUEUE,
 } from './driver-pg-jobs-sql';
-
 export {
   SQL_OUTBOX_CLAIM,
   SQL_OUTBOX_MARK_PUBLISHED,
+  SQL_OUTBOX_MARK_PUBLISHED_MANY,
   SQL_OUTBOX_PENDING_COUNT,
   SQL_OUTBOX_PUBLISHED_JOB,
   SQL_OUTBOX_RELEASE,
   SQL_OUTBOX_STAGE,
+  SQL_OUTBOX_STAGE_MANY,
 } from './driver-pg-outbox-sql';
 export { SQL_ACK, SQL_NACK } from './driver-pg-settle-sql';
 
-import { LEASE_LAPSED_FINAL_ATTEMPT } from './driver';
 import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
 import { notifyJobReady } from './driver-pg-wake-sql';
-import { COUNTER_BUCKET_MS } from './introspection';
-
-/** What both arms of the claim return: a `JobRow`, read off the updated row `j`. */
-const CLAIM_ROW_COLUMNS = `j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt,
-          j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by, j.claims,
-          j.traceparent, j.enqueued_by,
-          (extract(epoch from j.run_at)     * 1000)::bigint as run_at,
-          (extract(epoch from j.visible_at) * 1000)::bigint as visible_at,
-          (extract(epoch from j.created_at) * 1000)::bigint as created_at,
-          (extract(epoch from j.updated_at) * 1000)::bigint as updated_at`;
-
-const CLAIM_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attempt, max_attempts,
-       state, tenant_id, last_error, claimed_by, claims, traceparent, enqueued_by, run_at,
-       visible_at, created_at, updated_at`;
 
 /**
  * The insert, and its wake: `woke` is the notification's own column, evaluated once per inserted
@@ -57,6 +45,10 @@ const CLAIM_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attemp
  * (`EnqueueRequest.id`), so a publish repeated after a crash meets the job the first one made —
  * in any state — and inserts nothing; the partial index below collapses a repeat only while that
  * job is still live. A select over `values`, so each parameter carries its cast.
+ *
+ * `$8::numeric::bigint`, never `$8::bigint`: a fractional epoch ms — `now + Math.random() * 1000`
+ * — binds as text `1767225660000.5` on some clients, which no `bigint` reads (`22P02`). Through
+ * `numeric` it is rounded to the ms, the instant `SQL_ENQUEUE_MANY` stores for the same request.
  */
 export const SQL_ENQUEUE = `
 with inserted as (
@@ -64,8 +56,8 @@ with inserted as (
     (id, name, queue, input, idempotency_key, run_id, max_attempts, state, run_at, tenant_id,
      traceparent, enqueued_by)
   select $1::uuid, $2::text, $3::text, $4::jsonb, $5::text, $6::uuid, $7::int,
-         case when to_timestamp($8::bigint / 1000.0) > now() then 'delayed' else 'ready' end,
-         to_timestamp($8::bigint / 1000.0), $9::text, $10::text, $11::text
+         case when to_timestamp($8::numeric::bigint / 1000.0) > now() then 'delayed' else 'ready' end,
+         to_timestamp($8::numeric::bigint / 1000.0), $9::text, $10::text, $11::text
    where not exists (select 1 from x_jobs published where published.id = $1::uuid)
   on conflict (name, (coalesce(tenant_id, '')), idempotency_key)
     where state in ('ready', 'delayed', 'running', 'suspended')
@@ -94,75 +86,6 @@ select id, run_id from x_jobs
  limit 1
 `.trim();
 
-/**
- * The claim. SKIP LOCKED is the whole design: without it, worker 2 blocks on worker 1's row
- * lock and throughput collapses to one worker. `visible_at` in the predicate reclaims leases
- * abandoned by a crashed worker. `cancelled` is absent from every branch by construction — a
- * cancelled row is terminal, so it is never handed to a worker again.
- *
- * TWO arms over one pick. A `running` row is picked only with a lapsed lease, and one that lapsed
- * on its FINAL attempt (`attempt >= max_attempts` — `isFinalAttempt`, `retry.ts`) is `buried`:
- * settled `dead` here, counted, and answered with its new state so the driver reports it instead
- * of handing it out. Re-claimed, it was an attempt past `max_attempts` every lease, forever — a
- * job that kills its worker took one per visibility timeout and never reached the dead letters.
- * `claimed_by` is cleared with the lease, so the body that stalled rather than died cannot settle
- * the row it lost. A buried row spends a slot of `$2`: the pass after it claims what it displaced.
- * `$5` names the jobs declaring `retry.deadLetter: false`: their rows are buried `failed`.
- */
-export const SQL_CLAIM = `
-with picked as (
-  select id, (state = 'running' and attempt >= max_attempts) as exhausted
-    from x_jobs
-   where queue = any($1::text[])
-     and run_at <= now()
-     and not exists (
-       select 1 from x_job_pauses p where p.kind = 'queue' and p.name = x_jobs.queue
-     )
-     and (
-       state in ('ready', 'delayed', 'suspended')
-       or (state = 'running' and visible_at <= now())
-     )
-   order by run_at
-   limit $2
-     for update skip locked
-), buried as (
-  update x_jobs j
-     set state            = case when j.name = any($5::text[]) then 'failed' else 'dead' end,
-         visible_at       = null,
-         claimed_by       = null,
-         last_error       = '${LEASE_LAPSED_FINAL_ATTEMPT}',
-         last_error_stack = null,
-         updated_at       = now()
-    from picked p
-   where j.id = p.id and p.exhausted
-  returning ${CLAIM_ROW_COLUMNS}
-), counted as (
-  insert into x_job_counters (job, bucket_ms, bucket_start, dead, failed, duration_ms)
-  select name, ${COUNTER_BUCKET_MS}, date_trunc('minute', now()),
-         (count(*) filter (where state = 'dead'))::int,
-         (count(*) filter (where state = 'failed'))::int, 0
-    from buried
-   group by name
-  on conflict (job, bucket_ms, bucket_start) do update
-     set dead   = x_job_counters.dead + excluded.dead,
-         failed = x_job_counters.failed + excluded.failed
-), claimed as (
-  update x_jobs j
-     set state      = 'running',
-         attempt    = j.attempt + 1,
-         claims     = j.claims + 1,
-         claimed_by = $3,
-         visible_at = now() + ($4::bigint * interval '1 millisecond'),
-         updated_at = now()
-    from picked p
-   where j.id = p.id and not p.exhausted
-  returning ${CLAIM_ROW_COLUMNS}
-)
-select ${CLAIM_ROW_NAMES} from claimed
-union all
-select ${CLAIM_ROW_NAMES} from buried
-`.trim();
-
 /**
  * Stop a job from outside — a LIVE one. The runaway backfill this exists for is `running`; the
  * worker holding it learns on its next heartbeat, which no longer matches `state = 'running'`.
@@ -352,16 +275,26 @@ select floor(extract(epoch from statement_timestamp()) * 1000)::bigint as now
 
 /**
  * Earliest matching event at or after `afterMs`, so a resumed step consumes events in publication
- * order rather than jumping to the newest one — the memory bus's rule, in SQL.
+ * order rather than jumping to the newest one — the memory bus's rule, in SQL. `$4` is the cursor a
+ * refused event leaves: rows at or before it in `(published_at, id)` order are passed over, and a
+ * cursor whose row is gone excludes nothing. `floor`, as `SQL_EVENT_PUBLISH` stamps: a rounding
+ * cast answered a stamp up to half a ms LATER than the row, which a floor then stepped past. The
+ * order is QUALIFIED: a bare `published_at` there names the floored output column, whose ties in
+ * one millisecond then fell to the random id.
+ *
+ * $1 name, $2 correlation key, $3 floor (ms), $4 cursor event id.
  */
 export const SQL_EVENT_FIND = `
-select payload, (extract(epoch from published_at) * 1000)::bigint as published_at
-  from x_job_events
+select id, payload, floor(extract(epoch from published_at) * 1000)::bigint as published_at
+  from x_job_events e
  where name = $1
    and expires_at > statement_timestamp()
    and published_at >= to_timestamp($3 / 1000.0)
    and ($2::text is null or correlation_key = $2)
- order by published_at
+   and not exists (
+     select 1 from x_job_events c
+      where c.id = $4::uuid and (e.published_at, e.id) <= (c.published_at, c.id))
+ order by e.published_at, e.id
  limit 1
 `.trim();
 

package/src/driver-pg.ts

@@ -20,6 +20,7 @@ import type {
   JobDriver,
   JobRecord,
   NackOptions,
+  QueueDemand,
   QueueStats,
 } from './driver';
 import {
@@ -29,6 +30,8 @@ import {
   nackState,
   REQUEUEABLE_STATES,
 } from './driver';
+import { SQL_QUEUE_DEMAND } from './driver-pg-demand-sql';
+import { pgEnqueueMany } from './driver-pg-enqueue-many';
 import { pgOperator } from './driver-pg-operator';
 import type { BackfillRow, JobRow, StepRow } from './driver-pg-rows';
 import { num, toBackfillRun, toJobRecord, toStepRecord } from './driver-pg-rows';
@@ -255,7 +258,7 @@ export function postgresJobDriver(options: PostgresJobDriverOptions = {}): JobDr
     },
   };
 
-  return {
+  const driver: JobDriver = {
     name: 'pg',
     steps: postgresStepStore(exec),
     backfills: postgresBackfillLedger(exec),
@@ -328,6 +331,10 @@ export function postgresJobDriver(options: PostgresJobDriverOptions = {}): JobDr
       });
     },
 
+    // One statement for the batch (`driver-pg-enqueue-many.ts`); `enqueue` for a row whose holder
+    // settled between that statement and the read behind it.
+    enqueueMany: (requests) => pgEnqueueMany(exec, clock, requests, (one) => driver.enqueue(one)),
+
     async claim(claimOptions: ClaimOptions): Promise<readonly ClaimedJob[]> {
       assertClaimQueues('pg', claimOptions);
       assertClaimBounds('pg', claimOptions);
@@ -402,6 +409,14 @@ export function postgresJobDriver(options: PostgresJobDriverOptions = {}): JobDr
       return rows.length > 0;
     },
 
+    async demand(): Promise<readonly QueueDemand[]> {
+      const rows = await exec().query<{ queue: string; demand: number | string }>(
+        SQL_QUEUE_DEMAND,
+        [],
+      );
+      return rows.map((row) => ({ queue: row.queue, demand: num(row.demand) }));
+    },
+
     async stats(): Promise<readonly QueueStats[]> {
       const rows = await exec().query<{
         queue: string;
@@ -425,6 +440,7 @@ export function postgresJobDriver(options: PostgresJobDriverOptions = {}): JobDr
       }));
     },
   };
+  return driver;
 }
 
 /**

package/src/driver-redis-refusal.ts

@@ -0,0 +1,25 @@
+// An unreachable Redis, as the jobs contract says it: `X_DRIVER_UNAVAILABLE`, not Bun's client
+// error. Core's `redisClient()` refuses every command at once while down, so an enqueue fails typed
+// and immediately where `Bun.redis` blocked it, then closed the queue for good.
+
+import { renderThrowable, stringField } from '@ultimat3/core';
+import type { RedisSender } from './driver-redis';
+import { DriverUnavailableError } from './errors';
+
+/** Bun's code for a command refused for want of a connection: nothing reached the server. */
+const CONNECTION_CLOSED = 'ERR_REDIS_CONNECTION_CLOSED';
+
+/** `client` with its connection refusals typed. A server's own error reply passes through. */
+export function typedRefusals(client: RedisSender): RedisSender {
+  return {
+    send: (command, args) =>
+      client.send(command, args).catch((error: unknown) => {
+        if (stringField(error, 'code') !== CONNECTION_CLOSED) throw error;
+        throw new DriverUnavailableError({
+          driver: 'redis',
+          cause: `${command} was not sent, the connection is down (${renderThrowable(error)})`,
+          fix: 'redis-cli -u "$REDIS_URL" PING — nothing was queued, so the same call is safe to repeat once it answers PONG; the client reconnects on its own',
+        });
+      }),
+  };
+}