@ultimat3/jobs 1.2.0 → 3.0.0

package/src/outbox-lease.ts

@@ -0,0 +1,29 @@
+// The claim lease's one definition and its one normalisation. Both outbox stores read it, because
+// a lease the memory store defaults and the pg store validates is two answers to "how long is a
+// claim mine for" — and the shorter of the two is a row published twice.
+
+import { assert } from '@ultimat3/core';
+
+/**
+ * How long a claimed row stays its claimant's. Long enough that no healthy pass loses a batch it
+ * is still publishing, short enough that a relay killed mid-batch does not strand one for minutes.
+ */
+export const DEFAULT_OUTBOX_CLAIM_LEASE_MS = 30_000;
+
+/**
+ * Refused where it is written, the way `concurrency: 0` and `stepTimeout: 0` are. A lease of `0`
+ * expires before `claim()` resolves, so every relay reclaims every row on every tick and the lease
+ * buys nothing; a fractional one is compared against `now()` in Postgres and against whole ms
+ * here; `Infinity` never expires, so the rows of a relay that died are stranded forever — the one
+ * failure the lease exists to bound. `X_INVARIANT` because this is a caller-argument check with no
+ * dedicated code, the generic `@ultimat3/db` already borrows for the same shape.
+ */
+export function resolveClaimLeaseMs(value: number | undefined): number {
+  if (value === undefined) return DEFAULT_OUTBOX_CLAIM_LEASE_MS;
+  assert(
+    Number.isInteger(value) && value > 0,
+    `outbox claimLeaseMs is ${String(value)}, which is not a positive whole number of milliseconds`,
+    'pass a positive whole claimLeaseMs: 30_000 — createPgOutboxStore({ executor, txExecutor, claimLeaseMs: 30_000 }) — or omit the field for the 30s default',
+  );
+  return value;
+}

package/src/outbox-pg.ts

@@ -0,0 +1,188 @@
+// The `x_outbox` implementation of `OutboxStore` — the half of the transactional outbox that
+// makes it transactional. `stage()` runs on the CALLER'S OWN connection, so the queue row is in
+// the same Postgres transaction as the business rows and commits or vanishes with them. The
+// memory store hangs rows off the `Tx` object and is correct only inside one process; this is
+// what a deployment installs.
+//
+// `txExecutor` is injected rather than resolved here for the reason `createPgDriver` takes a
+// `PgExecutor`: this package holds no `@ultimat3/db` dependency, and "which connection is this
+// `Tx` on" is a question only boot can answer. Boot has `currentTx()` — a `DbTx` IS a client on
+// the transaction's connection — so the wiring is one line there and no tier crossing here.
+
+import type { Clock } from '@ultimat3/core';
+import { uuid } from '@ultimat3/core';
+import type { Tx } from '@ultimat3/entity';
+import { nowMs } from './clock';
+import type { PgExecutor } from './driver-pg';
+import {
+  SQL_OUTBOX_CLAIM,
+  SQL_OUTBOX_MARK_PUBLISHED,
+  SQL_OUTBOX_PENDING_COUNT,
+  SQL_OUTBOX_RELEASE,
+  SQL_OUTBOX_STAGE,
+} from './driver-pg-sql';
+import type { OutboxRecord, OutboxStore } from './outbox';
+import { resolveClaimLeaseMs } from './outbox-lease';
+
+interface OutboxRow {
+  readonly id: string;
+  readonly job: string;
+  readonly queue: string;
+  readonly input: unknown;
+  readonly idempotency_key: string;
+  readonly max_attempts: number;
+  readonly run_at: number | string;
+  readonly staged_at: number | string;
+  readonly tenant_id: string | null;
+  readonly traceparent: string | null;
+  readonly enqueued_by: string | null;
+  readonly claimed_by?: string | null;
+}
+
+export interface PgOutboxOptions {
+  /**
+   * The pooled executor the RELAY uses: `claim`, `markPublished` and `pendingCount` all run after
+   * the caller's transaction is gone, so they must not be bound to it.
+   */
+  readonly executor: PgExecutor;
+  /**
+   * The executor bound to `tx`'s own connection. Boot supplies it; without it `stage()` would
+   * write on a second connection and the outbox would guarantee nothing at all.
+   */
+  readonly txExecutor: (tx: Tx) => PgExecutor;
+  readonly clock?: Clock;
+  /**
+   * How long a claimed row stays this relay's before any relay may take it again. It bounds one
+   * thing only: how long the rows of a relay that DIED mid-batch sit unpublished. A pass that is
+   * merely slow keeps its rows because it published them; a pass that failed hands them back
+   * through `release`.
+   */
+  readonly claimLeaseMs?: number;
+  /**
+   * Written to `claimed_by`, and read back as the FENCE on `release` and `markPublished` — so it
+   * must be UNIQUE PER PROCESS. Two replicas passing one literal are one claimant to Postgres, and
+   * each can then release or retire the other's live batch. Omit it: the default is
+   * `relay-<uuid>`, minted once per store, which is unique by construction. Diagnostics second —
+   * it is what an operator reads to see which relay is sitting on a batch, and a value that
+   * changed every tick would answer nobody.
+   */
+  readonly relayId?: string;
+}
+
+function toRecord(row: OutboxRow): OutboxRecord {
+  return {
+    id: row.id,
+    job: row.job,
+    queue: row.queue,
+    input: row.input,
+    idempotencyKey: row.idempotency_key,
+    maxAttempts: Number(row.max_attempts),
+    runAt: Number(row.run_at),
+    stagedAt: Number(row.staged_at),
+    ...(row.tenant_id === null ? {} : { tenantId: row.tenant_id }),
+    ...(row.traceparent === null ? {} : { traceparent: row.traceparent }),
+    ...(row.enqueued_by === null ? {} : { enqueuedBy: row.enqueued_by }),
+    ...(typeof row.claimed_by === 'string' ? { claimedBy: row.claimed_by } : {}),
+  };
+}
+
+export function createPgOutboxStore(options: PgOutboxOptions): OutboxStore {
+  // What each open transaction has staged, for `commit()`'s return value only. Never the source
+  // of truth — that is the row, and the row's fate is the transaction's. A WeakMap so a `Tx` that
+  // is neither committed nor rolled back (a process killed mid-request) leaves nothing behind.
+  const staged = new WeakMap<object, OutboxRecord[]>();
+  const key = (tx: Tx): object => tx as unknown as object;
+  // One id per store, minted here rather than per claim: `claimed_by` is read by an operator
+  // asking which relay is sitting on a batch, and a value that changed every tick answers nobody.
+  // Per-store is also the granularity the fence needs — two relays are two processes, two stores.
+  const relayId = options.relayId ?? `relay-${uuid()}`;
+  // Resolved once, at construction, so a lease this store could never honour fails where it was
+  // written instead of inside a relay tick whose only trace is a log line nobody reads.
+  const claimLeaseMs = resolveClaimLeaseMs(options.claimLeaseMs);
+
+  return {
+    async stage(tx, record) {
+      await options
+        .txExecutor(tx)
+        .query(SQL_OUTBOX_STAGE, [
+          record.id,
+          record.job,
+          record.queue,
+          JSON.stringify(record.input ?? null),
+          record.idempotencyKey,
+          record.maxAttempts,
+          record.runAt,
+          record.stagedAt,
+          record.tenantId ?? null,
+          record.traceparent ?? null,
+          record.enqueuedBy ?? null,
+        ]);
+      const bucket = staged.get(key(tx)) ?? [];
+      bucket.push(record);
+      staged.set(key(tx), bucket);
+    },
+
+    /**
+     * Nothing to do but report. The rows became visible when Postgres committed them — there is
+     * no second write here, and that absence IS the guarantee: a commit hook that had to run
+     * would be one more thing between the business rows and the job row.
+     */
+    commit(tx) {
+      const bucket = staged.get(key(tx)) ?? [];
+      staged.delete(key(tx));
+      return Promise.resolve(bucket);
+    },
+
+    /** Also nothing: the ROLLBACK already took the rows. Only the bookkeeping is ours to drop. */
+    rollback(tx) {
+      staged.delete(key(tx));
+      return Promise.resolve();
+    },
+
+    /**
+     * A CLAIM, not a read. `for update skip locked` in a bare select held its locks only for that
+     * statement — which under autocommit is over before this method resolves — so two relays
+     * polling 200ms apart got the identical batch and both published it. The idempotency key
+     * collapses that only while the first job is still live, so the repeat that lands after it
+     * finished runs the handler a second time. `SQL_OUTBOX_CLAIM` stamps `claimed_at` in the same
+     * statement that locks the row; `skip locked` still keeps two relays from serialising.
+     */
+    async claim(limit) {
+      const rows = await options.executor.query<OutboxRow>(SQL_OUTBOX_CLAIM, [
+        limit,
+        claimLeaseMs,
+        relayId,
+      ]);
+      return rows.map(toRecord);
+    },
+
+    /**
+     * Fenced on the CLAIMANT, not only on the ids. A relay that stalled past its lease wakes into
+     * a world where its batch belongs to another relay, and an unfenced release frees rows that
+     * relay is mid-publish on — a third relay claims them and publishes them again. `relayId` is
+     * the fallback because it is what this store stamped: a caller with no token is this store's
+     * own relay, and one holding somebody else's token could not have got it from here.
+     */
+    async release(ids, claimant) {
+      if (ids.length === 0) return;
+      await options.executor.query(SQL_OUTBOX_RELEASE, [ids, claimant ?? relayId]);
+    },
+
+    /** Same fence, and worse to miss: marking a row published is losing the job behind it. */
+    async markPublished(id, at, claimant) {
+      await options.executor.query(SQL_OUTBOX_MARK_PUBLISHED, [
+        id,
+        at || nowMs(options.clock),
+        claimant ?? relayId,
+      ]);
+    },
+
+    async pendingCount() {
+      const rows = await options.executor.query<{ pending: number | string }>(
+        SQL_OUTBOX_PENDING_COUNT,
+        [],
+      );
+      return Number(rows[0]?.pending ?? 0);
+    },
+  };
+}

package/src/outbox.ts

@@ -1,6 +1,5 @@
-// The transactional outbox — ON BY DEFAULT, because the alternative is a bug you cannot see
-// in review. `ctx.jobs.enqueue()` inside a request writes the job row in the SAME `tx` as the
-// business rows and a relay publishes it after commit. Without it, every enqueue is a
+// The transactional outbox. `ctx.jobs.enqueue()` inside a request writes the job row in the SAME
+// `tx` as the business rows and a relay publishes it after commit. Without it, every enqueue is a
 // distributed-transaction coin flip:
 //
 //   enqueue then rollback  -> the job runs against rows that never existed
@@ -9,15 +8,30 @@
 // Both are load-dependent, both pass every test, and both are the top source of "the email
 // went out but the order isn't in the database" tickets. Joining the transaction removes the
 // window entirely; the relay's at-least-once delivery is deduped by the job's idempotencyKey.
+//
+// **It is NOT on by default, and this header used to claim it was** (`As of 2026-08`). Three
+// things have to be true in a process for an enqueue to be transactional, and the fallback at
+// `jobsFacade()` is what happens when they are not:
+//
+//   1. `x_outbox` exists — it ships in `SQL_JOBS_TABLE` now, so applying the queue DDL is enough.
+//   2. `setJobsFacade(createJobsFacade({ store, driver }, currentTx))` ran at boot, with a store
+//      from `createPgOutboxStore` and a REAL `currentTx` accessor.
+//   3. `createOutboxRelay({ store, driver }).start()` is running somewhere.
+//
+// With none of them, `jobsFacade()` answers the fallback below, whose `currentTx` is
+// `() => undefined`: every enqueue publishes straight to the driver, outside the caller's
+// transaction, and both failure modes above are live. That fallback is deliberate — a script, a
+// test and `x dev` must enqueue with nothing wired — but it is a fallback, not the guarantee.
 
 import type { Clock } from '@ultimat3/core';
-import { logger, uuid } from '@ultimat3/core';
+import { currentSpanContext, logger, traceparent, uuid } from '@ultimat3/core';
 import type { Tx } from '@ultimat3/entity';
 import { nowMs } from './clock';
 import type { EnqueueResult, JobDriver } from './driver';
 import { DEFAULT_QUEUE, jobDriver } from './driver';
 import { DriverUnavailableError, OutboxNoTxError } from './errors';
 import type { JobHandle } from './job';
+import { resolveClaimLeaseMs } from './outbox-lease';
 
 export interface OutboxRecord {
   readonly id: string;
@@ -29,7 +43,15 @@ export interface OutboxRecord {
   readonly runAt: number;
   readonly stagedAt: number;
   readonly tenantId?: string;
+  /** The enqueuing request's trace, carried across the relay so the job's span still has a parent. */
+  readonly traceparent?: string;
+  readonly enqueuedBy?: string;
   readonly publishedAt?: number;
+  /**
+   * Stamped by `claim()`, absent on a staged row. Hand it back to `release`/`markPublished`: it is
+   * the FENCE, so a claimant whose lease lapsed cannot touch the rows a newer one is publishing.
+   */
+  readonly claimedBy?: string;
 }
 
 export interface OutboxStore {
@@ -39,22 +61,81 @@ export interface OutboxStore {
   commit(tx: Tx): Promise<readonly OutboxRecord[]>;
   /** Called by the tx runner after ROLLBACK. Staged rows vanish with the transaction. */
   rollback(tx: Tx): Promise<void>;
-  /** Unpublished, committed rows — the relay's work queue. */
+  /**
+   * CLAIM unpublished, committed rows — the relay's work queue, and a lease rather than a read.
+   * A store that hands the same rows to two relays hands the same job to two workers, and the
+   * idempotency key only collapses that while the first job is still live.
+   */
   claim(limit: number): Promise<readonly OutboxRecord[]>;
-  markPublished(id: string, at: number): Promise<void>;
+  /**
+   * Hand a claim back before its lease runs out, for the batch a failed publish stopped. OPTIONAL
+   * so a store written before the claim became a lease still compiles: without it those rows wait
+   * out the whole lease, which is slower, never wrong.
+   *
+   * `claimant` is the `claimedBy` the claim stamped. Passing it is what makes a lapsed relay's
+   * late release a no-op instead of an unclaim of somebody else's live batch.
+   */
+  release?(ids: readonly string[], claimant?: string): Promise<void>;
+  /** `claimant` fences the same way, and here it is worse to miss: this retires the row. */
+  markPublished(id: string, at: number, claimant?: string): Promise<void>;
   pendingCount(): Promise<number>;
 }
 
+/**
+ * The claim's sort key, and it is TOTAL: `id` after `stagedAt`, exactly what `SQL_OUTBOX_CLAIM`
+ * orders by. Every row staged in one transaction shares a `stagedAt`, so the key ties for the
+ * batch that most depends on order — and a tie leaves both which rows a limit takes and the order
+ * they publish in to whatever the store iterated first. Code units, never `localeCompare`, for
+ * the reason `registeredJobs()` sorts that way.
+ */
+function byClaimOrder(a: OutboxRecord, b: OutboxRecord): number {
+  if (a.stagedAt !== b.stagedAt) return a.stagedAt - b.stagedAt;
+  if (a.id === b.id) return 0;
+  return a.id < b.id ? -1 : 1;
+}
+
+export interface MemoryOutboxOptions {
+  readonly clock?: Clock;
+  readonly claimLeaseMs?: number;
+}
+
+export interface MemoryOutboxStore extends OutboxStore {
+  /**
+   * Committed rows this process is still holding. The relay's backlog and nothing else — a
+   * published row is dropped, so this is a bound, not a total. `x dev` and the tests are the
+   * only readers; a pg deployment reads `x_outbox` instead.
+   */
+  retained(): number;
+}
+
 /**
  * Default store. Staged rows hang off the `Tx` object itself in a WeakMap, so the "same
  * transaction" guarantee needs no cooperation from the DB layer and rollback is a delete.
  * The pg store swaps this for a real `x_outbox` table written by the same connection.
  */
-export function createMemoryOutboxStore(): OutboxStore {
+export function createMemoryOutboxStore(options: MemoryOutboxOptions = {}): MemoryOutboxStore {
   const staged = new WeakMap<object, OutboxRecord[]>();
   const committed = new Map<string, OutboxRecord>();
+  /** Each claimed row's lease: when it was taken and by whom. Absent is `claimed_at is null`. */
+  const claims = new Map<string, { at: number; by: string }>();
+  const leaseMs = resolveClaimLeaseMs(options.claimLeaseMs);
+  // A token per CLAIM, where the pg store stamps one per RELAY. Two relays there are two stores
+  // with two ids; here they are two `claim()` calls on one store, so the claim is the only
+  // granularity at which this store can answer "is this mutation from the current holder".
+  let claimSeq = 0;
 
   const key = (tx: Tx): object => tx as unknown as object;
+  const free = (id: string, at: number): boolean => {
+    const claim = claims.get(id);
+    return claim === undefined || at - claim.at >= leaseMs;
+  };
+  /**
+   * A mutation from a claimant that no longer holds the row is a NO-OP. `undefined` is the caller
+   * that holds no token at all — a store-level caller, or one written before the fence — and is
+   * left unfenced rather than silently dropped, the way `release` itself is optional.
+   */
+  const owns = (id: string, claimant: string | undefined): boolean =>
+    claimant === undefined || claims.get(id)?.by === claimant;
 
   return {
     stage(tx, record) {
@@ -73,16 +154,36 @@ export function createMemoryOutboxStore(): OutboxStore {
       staged.delete(key(tx));
       return Promise.resolve();
     },
+    /**
+     * The same question `SQL_OUTBOX_CLAIM` answers, and it has to stay the same one: a row this
+     * store hands back is CLAIMED for `leaseMs`, so a second relay polling the same store gets
+     * nothing, and a claim whose holder died is reclaimable once the window passes.
+     */
     claim(limit) {
+      const at = nowMs(options.clock);
+      claimSeq += 1;
+      const by = `claim-${claimSeq}`;
       const ready = [...committed.values()]
-        .filter((record) => record.publishedAt === undefined)
-        .sort((a, b) => a.stagedAt - b.stagedAt)
+        .filter((record) => record.publishedAt === undefined && free(record.id, at))
+        .sort(byClaimOrder)
         .slice(0, limit);
-      return Promise.resolve(ready);
+      for (const record of ready) claims.set(record.id, { at, by });
+      return Promise.resolve(ready.map((record) => ({ ...record, claimedBy: by })));
+    },
+    release(ids, claimant) {
+      for (const id of ids) {
+        if (owns(id, claimant)) claims.delete(id);
+      }
+      return Promise.resolve();
     },
-    markPublished(id, at) {
-      const record = committed.get(id);
-      if (record !== undefined) committed.set(id, { ...record, publishedAt: at });
+    markPublished(id, _at, claimant) {
+      if (!owns(id, claimant)) return Promise.resolve();
+      // Deleted, not stamped. A published row is out of the relay's reach either way, and the
+      // pg store's `published_at` column is a retained audit trail this map is not: rewriting
+      // it in place held every payload ever enqueued — arbitrary job input — for the life of
+      // the process, and made `claim()` and `pendingCount()` walk all of them every 200ms.
+      committed.delete(id);
+      claims.delete(id);
       return Promise.resolve();
     },
     pendingCount() {
@@ -92,6 +193,7 @@ export function createMemoryOutboxStore(): OutboxStore {
       }
       return Promise.resolve(count);
     },
+    retained: () => committed.size,
   };
 }
 
@@ -102,6 +204,29 @@ export interface EnqueueOptions {
   readonly queue?: string;
   /** Escape hatch for enqueues that must fire regardless of the caller's transaction. */
   readonly outbox?: boolean;
+  /**
+   * Who asked. AUDIT ONLY — the job body still runs with system authority. `handle.as(actor, ...)`
+   * fills it from the actor; set it directly only where there is no actor object to hand over.
+   */
+  readonly enqueuedBy?: string;
+  /** Override the ambient trace. Almost never: the facade stamps the current span for you. */
+  readonly traceparent?: string;
+}
+
+/**
+ * The W3C `traceparent` of the span this enqueue is happening inside, or `undefined` outside a
+ * trace. This is the ONE place the link is minted: `docs/idea/04-jobs.md` promises a job trace
+ * linked to the enqueuing request, and before this there was no field to carry the link, so a
+ * checkout's `chargeCard` opened a fresh root two seconds later with nothing pointing back.
+ *
+ * A context recovered from a `Ctx` has an empty `spanId` (`currentSpanContext`), which renders as
+ * an all-zero parent that every collector rejects — so that case carries no header rather than a
+ * malformed one, and the job opens a root as it did before.
+ */
+function ambientTraceparent(): string | undefined {
+  const context = currentSpanContext();
+  if (context === undefined || context.spanId === '') return undefined;
+  return traceparent(context);
 }
 
 export interface OutboxDeps {
@@ -121,6 +246,7 @@ export function enqueueInTx<I>(
   options: EnqueueOptions = {},
 ): Promise<OutboxRecord> {
   const at = nowMs(deps.clock);
+  const trace = options.traceparent ?? ambientTraceparent();
   const record: OutboxRecord = {
     id: uuid(),
     job: handle.name,
@@ -131,6 +257,10 @@ export function enqueueInTx<I>(
     runAt: options.runAt ?? at,
     stagedAt: at,
     ...(options.tenantId === undefined ? {} : { tenantId: options.tenantId }),
+    // Stamped at STAGE time and not at publish time: the relay runs after commit, in its own
+    // timer, with no request span in scope — a trace read there would be nobody's.
+    ...(trace === undefined ? {} : { traceparent: trace }),
+    ...(options.enqueuedBy === undefined ? {} : { enqueuedBy: options.enqueuedBy }),
   };
   return deps.store.stage(tx, record).then(() => record);
 }
@@ -158,6 +288,7 @@ export function createJobsFacade(deps: OutboxDeps, currentTx: () => Tx | undefin
         if (deps.mode === 'required' && options.outbox !== false) {
           throw new OutboxNoTxError({ job: handle.name });
         }
+        const trace = options.traceparent ?? ambientTraceparent();
         return deps.driver.enqueue({
           name: handle.name,
           queue: options.queue ?? handle.queue,
@@ -166,6 +297,8 @@ export function createJobsFacade(deps: OutboxDeps, currentTx: () => Tx | undefin
           maxAttempts: handle.retry.attempts,
           runAt: options.runAt ?? nowMs(deps.clock),
           ...(options.tenantId === undefined ? {} : { tenantId: options.tenantId }),
+          ...(trace === undefined ? {} : { traceparent: trace }),
+          ...(options.enqueuedBy === undefined ? {} : { enqueuedBy: options.enqueuedBy }),
         });
       }
 
@@ -233,7 +366,13 @@ export interface OutboxRelay {
   /** One pass. Returns how many rows were published. Call it directly in tests. */
   tick(): Promise<number>;
   start(): void;
-  stop(): void;
+  /**
+   * Stop polling and WAIT OUT the pass in flight, the way `worker.stop()` waits out its rounds and
+   * `scheduler.stop()` its dispatch. A pass is a publish followed by a `markPublished`, and a
+   * caller that returned between the two closed the database under the row it was about to mark:
+   * re-published next boot at best, a rejection against a closed pool at worst.
+   */
+  stop(): Promise<void>;
   pending(): Promise<number>;
 }
 
@@ -246,6 +385,8 @@ export function createOutboxRelay(options: RelayOptions): OutboxRelay {
   const intervalMs = options.intervalMs ?? 200;
   let timer: ReturnType<typeof setInterval> | undefined;
   let running = false;
+  /** The pass in flight, so `stop()` joins it instead of returning underneath it. */
+  let pass: Promise<void> | undefined;
 
   const tick = async (): Promise<number> => {
     const batch = await options.store.claim(batchSize);
@@ -260,16 +401,36 @@ export function createOutboxRelay(options: RelayOptions): OutboxRelay {
           maxAttempts: record.maxAttempts,
           runAt: record.runAt,
           ...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
+          ...(record.traceparent === undefined ? {} : { traceparent: record.traceparent }),
+          ...(record.enqueuedBy === undefined ? {} : { enqueuedBy: record.enqueuedBy }),
         });
-        await options.store.markPublished(record.id, nowMs(options.clock));
+        // The claim's own token goes back with the mark. Without it a relay whose lease lapsed
+        // mid-stall retires a row the relay that reclaimed it has not published yet — the row is
+        // gone and nothing publishes it.
+        await options.store.markPublished(record.id, nowMs(options.clock), record.claimedBy);
         published += 1;
       } catch (error) {
-        // Leave the row unpublished; the next tick retries it. Order is preserved per queue.
+        // STOP the batch. `claim()` returns rows in `staged_at` order and the loop used to log
+        // and continue, which published every LATER row past the one that failed — so an app
+        // that stages `createInvoice` then `chargeCard` in one transaction could have the charge
+        // run first. The row stays unpublished and the next tick starts again from it; a
+        // permanently poisoned row wedges its queue, which is visible in `pending()` and is the
+        // correct trade against silently reordering committed work.
         logger.warn('jobs.outbox.publish-failed', {
           job: record.job,
           id: record.id,
+          published,
+          remaining: batch.length - published,
           error: error instanceof Error ? error.message : String(error),
         });
+        // Hand the rest of the batch back rather than sit on a claim nobody is publishing. The
+        // claim is a lease now, so without this a single pool timeout parks every committed row
+        // behind it for the whole lease window instead of for one poll interval.
+        await options.store.release?.(
+          batch.slice(published).map((row) => row.id),
+          record.claimedBy,
+        );
+        break;
       }
     }
     return published;
@@ -282,55 +443,39 @@ export function createOutboxRelay(options: RelayOptions): OutboxRelay {
       timer = setInterval(() => {
         if (running) return;
         running = true;
-        void tick().finally(() => {
-          running = false;
-        });
+        // `.catch` before `.finally`, the shape every other loop in this package uses. `tick()`
+        // guards each publish but not `store.claim()` — one pool timeout during a failover
+        // rejects here unobserved, and Bun's default for an unhandled rejection is to end the
+        // process, taking every staged, unpublished row with it.
+        //
+        // Kept rather than discarded, because `stop()` awaits exactly this chain: the publish and
+        // the `markPublished` behind it are one pass, and a teardown that returned between them
+        // closed the database under the row it was about to mark. The chain carries its own
+        // `catch`, so a caller that does not await still gets no unhandled rejection.
+        pass = tick()
+          .then((): void => undefined)
+          .catch((error: unknown) => {
+            logger.error('jobs.outbox.tick-failed', {
+              error: error instanceof Error ? error.message : String(error),
+            });
+          })
+          .finally(() => {
+            running = false;
+            pass = undefined;
+          });
       }, intervalMs);
     },
-    stop() {
+    async stop() {
       if (timer !== undefined) clearInterval(timer);
       timer = undefined;
+      // Awaited AFTER the interval is cleared, so no further pass can start behind this one.
+      await pass;
     },
     pending: () => options.store.pendingCount(),
   };
 }
 
-/** SQL for the pg-backed outbox. The relay publishes rows this INSERT created. */
-export const SQL_OUTBOX_TABLE = `
-create table if not exists x_outbox (
-  id              uuid primary key,
-  job             text        not null,
-  queue           text        not null default 'default',
-  input           jsonb       not null,
-  idempotency_key text        not null,
-  max_attempts    int         not null default 3,
-  run_at          timestamptz not null default now(),
-  staged_at       timestamptz not null default now(),
-  tenant_id       text,
-  published_at    timestamptz
-);
-create index if not exists x_outbox_unpublished_idx
-  on x_outbox (staged_at) where published_at is null;
-`.trim();
-
-export const SQL_OUTBOX_STAGE = `
-insert into x_outbox
-  (id, job, queue, input, idempotency_key, max_attempts, run_at, staged_at, tenant_id)
-values ($1, $2, $3, $4::jsonb, $5, $6, to_timestamp($7 / 1000.0), to_timestamp($8 / 1000.0), $9)
-`.trim();
-
-export const SQL_OUTBOX_CLAIM = `
-select id, job, queue, input, idempotency_key, max_attempts,
-       (extract(epoch from run_at) * 1000)::bigint    as run_at,
-       (extract(epoch from staged_at) * 1000)::bigint as staged_at,
-       tenant_id
-  from x_outbox
- where published_at is null
- order by staged_at
- limit $1
-   for update skip locked
-`.trim();
-
-export const SQL_OUTBOX_MARK_PUBLISHED = `
-update x_outbox set published_at = to_timestamp($2 / 1000.0) where id = $1
-`.trim();
+// The outbox's SQL moved to `driver-pg-sql.ts`, where every statement this package runs lives —
+// and, more to the point, where `SQL_JOBS_TABLE` is: `x_outbox` was declared here, in a constant
+// no boot code applied, which is the whole reason the outbox was documented and never created.
+// `src/index.ts` still re-exports the same four names, from there.

package/src/register.ts

@@ -6,7 +6,7 @@
 
 import { type RegisteredPrimitive, registerPrimitiveRegistrar } from '@ultimat3/core';
 import { isJobHandle, registerJob } from './job';
-import { isTaskHandle, registerTask } from './scheduler';
+import { isTaskHandle, registerTask } from './task';
 
 /** `registerJobs(await import('./jobs'))` — export names become job names. */
 export function registerJobs(

package/src/renewal-timer.ts

@@ -0,0 +1,35 @@
+// A renewal loop that is TERMINAL once stopped, and the one shape two files renew against.
+// `heartbeat.ts` renews a job's lease and `worker-fleet-slots.ts` a fleet slot, and both decided a
+// LOSS from an answer that arrived after the run had already finished cleanly: `stop()` cleared
+// the interval, which does nothing to the request already on the wire. So a flag is what every
+// branch after an `await` re-reads — the shape `settleWithin`'s `decided` uses in core.
+
+export interface RenewalTimer {
+  /**
+   * True once `stop()` has been called. Read AFTER every await in the renewal body: a clean
+   * completion settles the row this renewal is fenced on, so the driver answering "not yours"
+   * past that point is a finished job, not a lost lease — and reporting it is an error-level page
+   * for a non-event, on exactly the signals that mean the queue re-delivered live work.
+   */
+  stopped(): boolean;
+  /** Stop renewing, for the pass in flight as well as the next one. Idempotent. */
+  stop(): void;
+}
+
+export function startRenewalTimer(
+  intervalMs: number,
+  renew: () => void | Promise<void>,
+): RenewalTimer {
+  let stopped = false;
+  const timer = setInterval(() => {
+    void renew();
+  }, intervalMs);
+  return {
+    stopped: () => stopped,
+    stop(): void {
+      if (stopped) return;
+      stopped = true;
+      clearInterval(timer);
+    },
+  };
+}