@ultimat3/jobs 1.2.0 → 2.0.0

package/src/outbox.ts

@@ -1,6 +1,5 @@
-// The transactional outbox — ON BY DEFAULT, because the alternative is a bug you cannot see
-// in review. `ctx.jobs.enqueue()` inside a request writes the job row in the SAME `tx` as the
-// business rows and a relay publishes it after commit. Without it, every enqueue is a
+// The transactional outbox. `ctx.jobs.enqueue()` inside a request writes the job row in the SAME
+// `tx` as the business rows and a relay publishes it after commit. Without it, every enqueue is a
 // distributed-transaction coin flip:
 //
 //   enqueue then rollback  -> the job runs against rows that never existed
@@ -9,9 +8,23 @@
 // Both are load-dependent, both pass every test, and both are the top source of "the email
 // went out but the order isn't in the database" tickets. Joining the transaction removes the
 // window entirely; the relay's at-least-once delivery is deduped by the job's idempotencyKey.
+//
+// **It is NOT on by default, and this header used to claim it was** (`As of 2026-08`). Three
+// things have to be true in a process for an enqueue to be transactional, and the fallback at
+// `jobsFacade()` is what happens when they are not:
+//
+//   1. `x_outbox` exists — it ships in `SQL_JOBS_TABLE` now, so applying the queue DDL is enough.
+//   2. `setJobsFacade(createJobsFacade({ store, driver }, currentTx))` ran at boot, with a store
+//      from `createPgOutboxStore` and a REAL `currentTx` accessor.
+//   3. `createOutboxRelay({ store, driver }).start()` is running somewhere.
+//
+// With none of them, `jobsFacade()` answers the fallback below, whose `currentTx` is
+// `() => undefined`: every enqueue publishes straight to the driver, outside the caller's
+// transaction, and both failure modes above are live. That fallback is deliberate — a script, a
+// test and `x dev` must enqueue with nothing wired — but it is a fallback, not the guarantee.
 
 import type { Clock } from '@ultimat3/core';
-import { logger, uuid } from '@ultimat3/core';
+import { currentSpanContext, logger, traceparent, uuid } from '@ultimat3/core';
 import type { Tx } from '@ultimat3/entity';
 import { nowMs } from './clock';
 import type { EnqueueResult, JobDriver } from './driver';
@@ -29,6 +42,9 @@ export interface OutboxRecord {
   readonly runAt: number;
   readonly stagedAt: number;
   readonly tenantId?: string;
+  /** The enqueuing request's trace, carried across the relay so the job's span still has a parent. */
+  readonly traceparent?: string;
+  readonly enqueuedBy?: string;
   readonly publishedAt?: number;
 }
 
@@ -45,12 +61,21 @@ export interface OutboxStore {
   pendingCount(): Promise<number>;
 }
 
+export interface MemoryOutboxStore extends OutboxStore {
+  /**
+   * Committed rows this process is still holding. The relay's backlog and nothing else — a
+   * published row is dropped, so this is a bound, not a total. `x dev` and the tests are the
+   * only readers; a pg deployment reads `x_outbox` instead.
+   */
+  retained(): number;
+}
+
 /**
  * Default store. Staged rows hang off the `Tx` object itself in a WeakMap, so the "same
  * transaction" guarantee needs no cooperation from the DB layer and rollback is a delete.
  * The pg store swaps this for a real `x_outbox` table written by the same connection.
  */
-export function createMemoryOutboxStore(): OutboxStore {
+export function createMemoryOutboxStore(): MemoryOutboxStore {
   const staged = new WeakMap<object, OutboxRecord[]>();
   const committed = new Map<string, OutboxRecord>();
 
@@ -80,9 +105,12 @@ export function createMemoryOutboxStore(): OutboxStore {
         .slice(0, limit);
       return Promise.resolve(ready);
     },
-    markPublished(id, at) {
-      const record = committed.get(id);
-      if (record !== undefined) committed.set(id, { ...record, publishedAt: at });
+    markPublished(id, _at) {
+      // Deleted, not stamped. A published row is out of the relay's reach either way, and the
+      // pg store's `published_at` column is a retained audit trail this map is not: rewriting
+      // it in place held every payload ever enqueued — arbitrary job input — for the life of
+      // the process, and made `claim()` and `pendingCount()` walk all of them every 200ms.
+      committed.delete(id);
       return Promise.resolve();
     },
     pendingCount() {
@@ -92,6 +120,7 @@ export function createMemoryOutboxStore(): OutboxStore {
       }
       return Promise.resolve(count);
     },
+    retained: () => committed.size,
   };
 }
 
@@ -102,6 +131,29 @@ export interface EnqueueOptions {
   readonly queue?: string;
   /** Escape hatch for enqueues that must fire regardless of the caller's transaction. */
   readonly outbox?: boolean;
+  /**
+   * Who asked. AUDIT ONLY — the job body still runs with system authority. `handle.as(actor, ...)`
+   * fills it from the actor; set it directly only where there is no actor object to hand over.
+   */
+  readonly enqueuedBy?: string;
+  /** Override the ambient trace. Almost never: the facade stamps the current span for you. */
+  readonly traceparent?: string;
+}
+
+/**
+ * The W3C `traceparent` of the span this enqueue is happening inside, or `undefined` outside a
+ * trace. This is the ONE place the link is minted: `docs/idea/04-jobs.md` promises a job trace
+ * linked to the enqueuing request, and before this there was no field to carry the link, so a
+ * checkout's `chargeCard` opened a fresh root two seconds later with nothing pointing back.
+ *
+ * A context recovered from a `Ctx` has an empty `spanId` (`currentSpanContext`), which renders as
+ * an all-zero parent that every collector rejects — so that case carries no header rather than a
+ * malformed one, and the job opens a root as it did before.
+ */
+function ambientTraceparent(): string | undefined {
+  const context = currentSpanContext();
+  if (context === undefined || context.spanId === '') return undefined;
+  return traceparent(context);
 }
 
 export interface OutboxDeps {
@@ -121,6 +173,7 @@ export function enqueueInTx<I>(
   options: EnqueueOptions = {},
 ): Promise<OutboxRecord> {
   const at = nowMs(deps.clock);
+  const trace = options.traceparent ?? ambientTraceparent();
   const record: OutboxRecord = {
     id: uuid(),
     job: handle.name,
@@ -131,6 +184,10 @@ export function enqueueInTx<I>(
     runAt: options.runAt ?? at,
     stagedAt: at,
     ...(options.tenantId === undefined ? {} : { tenantId: options.tenantId }),
+    // Stamped at STAGE time and not at publish time: the relay runs after commit, in its own
+    // timer, with no request span in scope — a trace read there would be nobody's.
+    ...(trace === undefined ? {} : { traceparent: trace }),
+    ...(options.enqueuedBy === undefined ? {} : { enqueuedBy: options.enqueuedBy }),
   };
   return deps.store.stage(tx, record).then(() => record);
 }
@@ -158,6 +215,7 @@ export function createJobsFacade(deps: OutboxDeps, currentTx: () => Tx | undefin
         if (deps.mode === 'required' && options.outbox !== false) {
           throw new OutboxNoTxError({ job: handle.name });
         }
+        const trace = options.traceparent ?? ambientTraceparent();
         return deps.driver.enqueue({
           name: handle.name,
           queue: options.queue ?? handle.queue,
@@ -166,6 +224,8 @@ export function createJobsFacade(deps: OutboxDeps, currentTx: () => Tx | undefin
           maxAttempts: handle.retry.attempts,
           runAt: options.runAt ?? nowMs(deps.clock),
           ...(options.tenantId === undefined ? {} : { tenantId: options.tenantId }),
+          ...(trace === undefined ? {} : { traceparent: trace }),
+          ...(options.enqueuedBy === undefined ? {} : { enqueuedBy: options.enqueuedBy }),
         });
       }
 
@@ -233,7 +293,13 @@ export interface OutboxRelay {
   /** One pass. Returns how many rows were published. Call it directly in tests. */
   tick(): Promise<number>;
   start(): void;
-  stop(): void;
+  /**
+   * Stop polling and WAIT OUT the pass in flight, the way `worker.stop()` waits out its rounds and
+   * `scheduler.stop()` its dispatch. A pass is a publish followed by a `markPublished`, and a
+   * caller that returned between the two closed the database under the row it was about to mark:
+   * re-published next boot at best, a rejection against a closed pool at worst.
+   */
+  stop(): Promise<void>;
   pending(): Promise<number>;
 }
 
@@ -246,6 +312,8 @@ export function createOutboxRelay(options: RelayOptions): OutboxRelay {
   const intervalMs = options.intervalMs ?? 200;
   let timer: ReturnType<typeof setInterval> | undefined;
   let running = false;
+  /** The pass in flight, so `stop()` joins it instead of returning underneath it. */
+  let pass: Promise<void> | undefined;
 
   const tick = async (): Promise<number> => {
     const batch = await options.store.claim(batchSize);
@@ -260,16 +328,26 @@ export function createOutboxRelay(options: RelayOptions): OutboxRelay {
           maxAttempts: record.maxAttempts,
           runAt: record.runAt,
           ...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
+          ...(record.traceparent === undefined ? {} : { traceparent: record.traceparent }),
+          ...(record.enqueuedBy === undefined ? {} : { enqueuedBy: record.enqueuedBy }),
         });
         await options.store.markPublished(record.id, nowMs(options.clock));
         published += 1;
       } catch (error) {
-        // Leave the row unpublished; the next tick retries it. Order is preserved per queue.
+        // STOP the batch. `claim()` returns rows in `staged_at` order and the loop used to log
+        // and continue, which published every LATER row past the one that failed — so an app
+        // that stages `createInvoice` then `chargeCard` in one transaction could have the charge
+        // run first. The row stays unpublished and the next tick starts again from it; a
+        // permanently poisoned row wedges its queue, which is visible in `pending()` and is the
+        // correct trade against silently reordering committed work.
         logger.warn('jobs.outbox.publish-failed', {
           job: record.job,
           id: record.id,
+          published,
+          remaining: batch.length - published,
           error: error instanceof Error ? error.message : String(error),
         });
+        break;
       }
     }
     return published;
@@ -282,55 +360,39 @@ export function createOutboxRelay(options: RelayOptions): OutboxRelay {
       timer = setInterval(() => {
         if (running) return;
         running = true;
-        void tick().finally(() => {
-          running = false;
-        });
+        // `.catch` before `.finally`, the shape every other loop in this package uses. `tick()`
+        // guards each publish but not `store.claim()` — one pool timeout during a failover
+        // rejects here unobserved, and Bun's default for an unhandled rejection is to end the
+        // process, taking every staged, unpublished row with it.
+        //
+        // Kept rather than discarded, because `stop()` awaits exactly this chain: the publish and
+        // the `markPublished` behind it are one pass, and a teardown that returned between them
+        // closed the database under the row it was about to mark. The chain carries its own
+        // `catch`, so a caller that does not await still gets no unhandled rejection.
+        pass = tick()
+          .then((): void => undefined)
+          .catch((error: unknown) => {
+            logger.error('jobs.outbox.tick-failed', {
+              error: error instanceof Error ? error.message : String(error),
+            });
+          })
+          .finally(() => {
+            running = false;
+            pass = undefined;
+          });
       }, intervalMs);
     },
-    stop() {
+    async stop() {
       if (timer !== undefined) clearInterval(timer);
       timer = undefined;
+      // Awaited AFTER the interval is cleared, so no further pass can start behind this one.
+      await pass;
     },
     pending: () => options.store.pendingCount(),
   };
 }
 
-/** SQL for the pg-backed outbox. The relay publishes rows this INSERT created. */
-export const SQL_OUTBOX_TABLE = `
-create table if not exists x_outbox (
-  id              uuid primary key,
-  job             text        not null,
-  queue           text        not null default 'default',
-  input           jsonb       not null,
-  idempotency_key text        not null,
-  max_attempts    int         not null default 3,
-  run_at          timestamptz not null default now(),
-  staged_at       timestamptz not null default now(),
-  tenant_id       text,
-  published_at    timestamptz
-);
-create index if not exists x_outbox_unpublished_idx
-  on x_outbox (staged_at) where published_at is null;
-`.trim();
-
-export const SQL_OUTBOX_STAGE = `
-insert into x_outbox
-  (id, job, queue, input, idempotency_key, max_attempts, run_at, staged_at, tenant_id)
-values ($1, $2, $3, $4::jsonb, $5, $6, to_timestamp($7 / 1000.0), to_timestamp($8 / 1000.0), $9)
-`.trim();
-
-export const SQL_OUTBOX_CLAIM = `
-select id, job, queue, input, idempotency_key, max_attempts,
-       (extract(epoch from run_at) * 1000)::bigint    as run_at,
-       (extract(epoch from staged_at) * 1000)::bigint as staged_at,
-       tenant_id
-  from x_outbox
- where published_at is null
- order by staged_at
- limit $1
-   for update skip locked
-`.trim();
-
-export const SQL_OUTBOX_MARK_PUBLISHED = `
-update x_outbox set published_at = to_timestamp($2 / 1000.0) where id = $1
-`.trim();
+// The outbox's SQL moved to `driver-pg-sql.ts`, where every statement this package runs lives —
+// and, more to the point, where `SQL_JOBS_TABLE` is: `x_outbox` was declared here, in a constant
+// no boot code applied, which is the whole reason the outbox was documented and never created.
+// `src/index.ts` still re-exports the same four names, from there.

package/src/register.ts

@@ -6,7 +6,7 @@
 
 import { type RegisteredPrimitive, registerPrimitiveRegistrar } from '@ultimat3/core';
 import { isJobHandle, registerJob } from './job';
-import { isTaskHandle, registerTask } from './scheduler';
+import { isTaskHandle, registerTask } from './task';
 
 /** `registerJobs(await import('./jobs'))` — export names become job names. */
 export function registerJobs(

package/src/retry.ts

@@ -48,7 +48,12 @@ export function backoffDelayMs(policy: RetryPolicy, attempt: number, random?: Ra
   else raw = base * 2 ** (step - 1);
 
   const capped = Math.min(raw, cap);
-  if (policy.jitter !== true) return Math.round(capped);
+  // `?? DEFAULT_RETRY.jitter`, like every other option above. It read `!== true`, so an omitted
+  // `jitter` meant OFF while the field's own doc says "Equal jitter … by default" — a burst of
+  // failures then retried in lockstep, which is the thundering herd the default exists to break.
+  // Masked for jobs declared through `job()` (it merges the defaults) and live for every direct
+  // caller of this exported function, `retrySchedule` included.
+  if ((policy.jitter ?? DEFAULT_RETRY.jitter) !== true) return Math.round(capped);
   const roll = (random ?? Math.random)();
   return Math.round(capped / 2 + (capped / 2) * roll);
 }

package/src/run-signal.ts

@@ -0,0 +1,50 @@
+// The signal ONE job run is cancelled by: the caller's `Ctx.signal`, the lease heartbeat, and
+// whatever else the worker holds over the run. `AbortSignal.any` composed the same thing and could
+// not be undone — an app wiring a process-lifetime controller into `WorkerOptions.context()` grew
+// one composite per job for the life of the worker, and nothing could abort the result either.
+
+export interface RunSignal {
+  /** Handed to the run as `ctx.signal`; dies with the run. */
+  readonly signal: AbortSignal;
+  /** Cancel this run. The first reason wins, exactly as `AbortController.abort` already does. */
+  abort(reason: unknown): void;
+  /**
+   * Stop following the sources. Idempotent, and it never aborts: a run that settled leaves its
+   * signal in whatever state it ended in, it just stops being the caller's problem.
+   */
+  dispose(): void;
+}
+
+/**
+ * One controller per run, following every source it was given. A source already aborted aborts the
+ * run at composition, carrying its own reason — the same semantics `AbortSignal.any` has, minus
+ * the part that cannot be handed back.
+ *
+ * `undefined` and a non-`AbortSignal` are both skipped rather than refused: `Ctx.signal` is
+ * non-optional in the type and still arrives missing across a cast (`@ultimat3/http`'s `asCtx`, a
+ * test's `{} as Ctx`), and a job that crashed on a missing field is worse than a job with no
+ * caller to follow.
+ */
+export function createRunSignal(sources: readonly (AbortSignal | undefined)[]): RunSignal {
+  const controller = new AbortController();
+  const detach: (() => void)[] = [];
+
+  for (const source of sources) {
+    if (!(source instanceof AbortSignal)) continue;
+    if (source.aborted) {
+      controller.abort(source.reason);
+      continue;
+    }
+    const forward = (): void => controller.abort(source.reason);
+    source.addEventListener('abort', forward, { once: true });
+    detach.push(() => source.removeEventListener('abort', forward));
+  }
+
+  return {
+    signal: controller.signal,
+    abort: (reason) => controller.abort(reason),
+    dispose: () => {
+      for (const off of detach.splice(0)) off();
+    },
+  };
+}

package/src/scheduler-pg.ts

@@ -0,0 +1,103 @@
+// The scheduler's durable halves: the watermark it decides "missed" against, and leader election
+// that survives a pooled connection. Both were `Map`s and a `() => true` in the shipped boot, and
+// both failures are silent — a redeployed pod arms to tomorrow and never reports the 03:00 run the
+// pod it replaced dropped, and two pods in a rolling update both dispatch every task.
+
+import type { Clock } from '@ultimat3/core';
+import { uuid } from '@ultimat3/core';
+import { nowMs } from './clock';
+import type { PgExecutor } from './driver-pg';
+import {
+  SQL_LEADER_ACQUIRE,
+  SQL_LEADER_RELEASE,
+  SQL_SCHEDULER_STATE_GET,
+  SQL_SCHEDULER_STATE_MARK,
+} from './driver-pg-sql';
+import type { LeaderElection, SchedulerState } from './scheduler';
+
+/**
+ * `x_scheduler_state`, one row per task. The watermark is what makes "missed" a decidable
+ * question at all: `catchUp`, `maxCatchUp` and `run-once` are all relative to it, so a scheduler
+ * whose watermark dies with its process has none of them — it takes the arming branch on every
+ * boot and drops every occurrence between the two processes with nothing logged.
+ */
+export function pgSchedulerState(executor: PgExecutor): SchedulerState {
+  return {
+    async lastFiredAt(taskName) {
+      const rows = await executor.query<{ last_fired_at: number | string | null }>(
+        SQL_SCHEDULER_STATE_GET,
+        [taskName],
+      );
+      const value = rows[0]?.last_fired_at;
+      return value === null || value === undefined ? undefined : Number(value);
+    },
+    async markFired(taskName, occurrenceMs) {
+      await executor.query(SQL_SCHEDULER_STATE_MARK, [taskName, occurrenceMs]);
+    },
+  };
+}
+
+export interface PgLeaseLeaderOptions {
+  readonly executor: PgExecutor;
+  /** One key per elected role. Default `'scheduler'`. */
+  readonly lockKey?: string;
+  /** This node's identity. Defaults to a per-process uuid — never a hostname a pod reuses. */
+  readonly holder?: string;
+  /**
+   * How long a grant survives without a renewal. Must be comfortably longer than the scheduler's
+   * tick interval, or a slow round loses the lock to a standby mid-dispatch. Default 30s against
+   * a 1s tick.
+   */
+  readonly ttlMs?: number;
+  readonly clock?: Clock;
+}
+
+export const DEFAULT_LEADER_TTL_MS = 30_000;
+
+/**
+ * Leader election as an expiring row, which is what makes it correct on the executor this package
+ * is actually handed: a POOL. `createPgLeader`'s `pg_try_advisory_lock` is session-scoped, and a
+ * session on a pool ends the moment the connection goes back — so every node reads itself as
+ * leader and a rolling update double-fires every task. `@ultimat3/realtime`'s `PgAdvisoryLock`
+ * solves the same problem by owning its connection; this package holds no wire protocol, so it
+ * solves it with a row instead.
+ *
+ * `acquire()` is also the RENEWAL, so the scheduler calling it every round both keeps the lease
+ * alive and learns the round it stops being leader. A crashed node's lease is reclaimed by expiry
+ * with nothing to clean up, which is the one property the advisory lock had and a plain
+ * `insert ... on conflict do nothing` would not.
+ */
+export function createPgLeaseLeader(options: PgLeaseLeaderOptions): LeaderElection {
+  const lockKey = options.lockKey ?? 'scheduler';
+  const holder = options.holder ?? `scheduler-${uuid()}`;
+  const ttlMs = options.ttlMs ?? DEFAULT_LEADER_TTL_MS;
+  return {
+    async acquire() {
+      const rows = await options.executor.query<{ holder: string }>(SQL_LEADER_ACQUIRE, [
+        lockKey,
+        holder,
+        ttlMs,
+      ]);
+      return rows[0]?.holder === holder;
+    },
+    async release() {
+      await options.executor.query(SQL_LEADER_RELEASE, [lockKey, holder]);
+    },
+  };
+}
+
+/** Exposed so a test — and `x jobs ls` — can say which node currently holds the lease. */
+export async function currentLeader(
+  executor: PgExecutor,
+  lockKey = 'scheduler',
+  clock?: Clock,
+): Promise<string | undefined> {
+  const rows = await executor.query<{ holder: string; expires_at: number | string }>(
+    `select holder, (extract(epoch from expires_at) * 1000)::bigint as expires_at
+       from x_scheduler_leader where lock_key = $1`,
+    [lockKey],
+  );
+  const row = rows[0];
+  if (row === undefined) return undefined;
+  return Number(row.expires_at) > nowMs(clock) ? row.holder : undefined;
+}