@ultimat3/jobs 30.9.0 → 31.0.0

package/src/driver-pg-sql.ts

@@ -196,22 +196,27 @@ on conflict (task_name) do update
  * `(lease_key, slot)` primary key serialises two workers that picked the same slot, and the
  * loser's `do update` is guarded on the slot having expired — a live slot returns no row, so the
  * grant is never doubled. Under contention this can refuse a slot that is genuinely free; a
- * refusal costs one poll interval and an over-grant costs the guarantee.
+ * refusal costs one poll interval and an over-grant costs the guarantee. `nextval` runs on the ONE
+ * candidate the inner `limit 1` kept, and a takeover takes the new number too: the fence.
  */
 export const SQL_LEASE_ACQUIRE = `
-insert into x_job_leases (lease_key, slot, holder, expires_at)
-select $1, s.slot, $2, now() + ($4::bigint * interval '1 millisecond')
-  from generate_series(0, $3::int - 1) as s(slot)
- where not exists (
-   select 1 from x_job_leases l
-    where l.lease_key = $1 and l.slot = s.slot and l.expires_at > now()
- )
- order by s.slot
- limit 1
+insert into x_job_leases (lease_key, slot, holder, expires_at, fence)
+select f.lease_key, f.slot, f.holder, f.expires_at, nextval('x_job_lease_fence')
+  from (
+    select $1::text as lease_key, s.slot, $2::text as holder,
+           now() + ($4::bigint * interval '1 millisecond') as expires_at
+      from generate_series(0, $3::int - 1) as s(slot)
+     where not exists (
+       select 1 from x_job_leases l
+        where l.lease_key = $1 and l.slot = s.slot and l.expires_at > now()
+     )
+     order by s.slot
+     limit 1
+  ) as f
 on conflict (lease_key, slot) do update
-   set holder = excluded.holder, expires_at = excluded.expires_at
+   set holder = excluded.holder, expires_at = excluded.expires_at, fence = excluded.fence
  where x_job_leases.expires_at <= now()
-returning slot
+returning slot, fence
 `.trim();
 
 /**
@@ -258,12 +263,18 @@ select holder from x_job_leases
  * on commit, or "before" a question asked while the transaction was open
  * (`events-pg.live.test.ts`). Outside a transaction the two are the same instant.
  *
+ * The stamp is TRUNCATED to the millisecond, the unit every cursor is written in, so the
+ * `(published_at, id)` order the find sorts by is exactly the order a cursor compares — a
+ * microsecond stamp put an event of the cursor's millisecond with a smaller id after it. `xact`
+ * is the publishing transaction: what a wait's horizon asks about (`SQL_EVENT_FIND`).
+ *
  * $1 id, $2 name, $3 payload, $4 correlation key, $5 ttl (ms).
  */
 export const SQL_EVENT_PUBLISH = `
-insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at)
-values ($1, $2, $3::jsonb, $4, statement_timestamp(),
-        statement_timestamp() + ($5::bigint * interval '1 millisecond'))
+insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at, xact)
+values ($1, $2, $3::jsonb, $4, date_trunc('milliseconds', statement_timestamp()),
+        statement_timestamp() + ($5::bigint * interval '1 millisecond'),
+        pg_current_xact_id())
 returning floor(extract(epoch from published_at) * 1000)::bigint as published_at,
           floor(extract(epoch from expires_at)   * 1000)::bigint as expires_at
 `.trim();
@@ -274,26 +285,56 @@ select floor(extract(epoch from statement_timestamp()) * 1000)::bigint as now
 `.trim();
 
 /**
- * Earliest matching event at or after `afterMs`, so a resumed step consumes events in publication
- * order rather than jumping to the newest one — the memory bus's rule, in SQL. `$4` is the cursor a
- * refused event leaves: rows at or before it in `(published_at, id)` order are passed over, and a
- * cursor whose row is gone excludes nothing. `floor`, as `SQL_EVENT_PUBLISH` stamps: a rounding
- * cast answered a stamp up to half a ms LATER than the row, which a floor then stepped past. The
- * order is QUALIFIED: a bare `published_at` there names the floored output column, whose ties in
- * one millisecond then fell to the random id.
+ * A new wait's starting line: the bus's clock and the snapshot this statement reads under, in one
+ * statement so no commit falls between them. Every event that snapshot cannot see — a publish
+ * still in flight, or one not yet begun — is owed to the wait whatever its stamp.
+ */
+export const SQL_EVENT_MARK = `
+select floor(extract(epoch from statement_timestamp()) * 1000)::bigint as now,
+       pg_current_snapshot()::text as horizon
+`.trim();
+
+/**
+ * Earliest matching event past the cursor, so a resumed step consumes events in publication order
+ * rather than jumping to the newest one — `events-cursor.ts`'s rule, in SQL. Past the cursor is
+ * `(published_at, id) > ($3, $4)`, the nil uuid standing in for "no id": compared, never looked
+ * up, so a cursor whose event was purged still excludes what it excluded. The order is QUALIFIED:
+ * a bare `published_at` there names the floored output column.
+ *
+ * OR invisible to `$5`, the wait's horizon (`SQL_EVENT_MARK`, or the snapshot of the search that
+ * found a refused event): stamped at INSERT, visible at COMMIT, so a publish committed after the
+ * wait began can carry a stamp before it. `xact >= xmin` is implied by "not visible" and is what
+ * lets the planner answer that half from `x_job_events_xact_idx`. A row from before `xact` was
+ * written has it null and is matched by its stamp alone. The cursor's own event never matches —
+ * a row stamped at microseconds, from before truncation, floors to a cursor it would sort past.
+ * The horizon of THIS search rides the hit, for the refusal that may follow it.
+ *
+ * `$6`/`$7` are the WAIT's own line (`EventSince`), which a refusal never moves: an event is the
+ * wait's only when stamped at or after it, or unseen by the mark's horizon. Without it, a refused
+ * owed event's cursor let one committed BEFORE the wait sort past it and be handed out.
  *
- * $1 name, $2 correlation key, $3 floor (ms), $4 cursor event id.
+ * $1 name, $2 correlation key, $3 cursor (ms), $4 cursor event id, $5 the cursor search's horizon,
+ * $6 the wait's start (ms), $7 the wait's horizon (pg_snapshot text), $8 the ids it refused behind
+ * its cursor (JSON text, never a bound array: issue #384).
  */
 export const SQL_EVENT_FIND = `
-select id, payload, floor(extract(epoch from published_at) * 1000)::bigint as published_at
+select id, payload, floor(extract(epoch from published_at) * 1000)::bigint as published_at,
+       pg_current_snapshot()::text as horizon
   from x_job_events e
  where name = $1
    and expires_at > statement_timestamp()
-   and published_at >= to_timestamp($3 / 1000.0)
    and ($2::text is null or correlation_key = $2)
+   and e.id is distinct from $4::uuid
    and not exists (
-     select 1 from x_job_events c
-      where c.id = $4::uuid and (e.published_at, e.id) <= (c.published_at, c.id))
+     select 1 from json_array_elements_text(coalesce($8::json, '[]'::json)) r
+      where r.value::uuid = e.id)
+   and (e.published_at >= to_timestamp($6 / 1000.0)
+        or (e.xact >= pg_snapshot_xmin($7::pg_snapshot)
+            and not pg_visible_in_snapshot(e.xact, $7::pg_snapshot)))
+   and ((e.published_at, e.id) >
+          (to_timestamp($3 / 1000.0), coalesce($4::uuid, '00000000-0000-0000-0000-000000000000'))
+        or (e.xact >= pg_snapshot_xmin($5::pg_snapshot)
+            and not pg_visible_in_snapshot(e.xact, $5::pg_snapshot)))
  order by e.published_at, e.id
  limit 1
 `.trim();

package/src/driver-pg.ts

@@ -5,7 +5,7 @@
 // they run against in `driver-pg-ddl.ts`, and the row-to-record decoding in `driver-pg-rows.ts`.
 
 import type { Clock, PgExecutor } from '@ultimat3/core';
-import { finiteCount, renderCauseValue, systemClock, uuidV7 } from '@ultimat3/core';
+import { finiteCount, liveClock, renderCauseValue, uuidV7 } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
 import { listedRunId } from './backfill-ledger';
 import { nowMs } from './clock';
@@ -86,7 +86,7 @@ function resolveExecutor(injected: PgExecutor | undefined): PgExecutor {
     // be "not configured". An executor is injected by the boot or the driver has none.
     cause:
       'postgresJobDriver() was called with no executor, and this driver has no ambient fallback',
-    fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: setJobDriver(memoryJobDriver())',
+    fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: export const runtime = { jobs: memoryJobDriver() } in apps/<app>/runtime.ts',
   });
 }
 
@@ -165,14 +165,15 @@ function postgresLeaseStore(exec: () => PgExecutor): LeaseStore {
   return {
     async acquire(key, limit, ttlMs, holder) {
       if (limit <= 0) return undefined;
-      const rows = await exec().query<{ slot: number | string }>(SQL_LEASE_ACQUIRE, [
-        key,
-        holder,
-        limit,
-        ttlMs,
-      ]);
+      const rows = await exec().query<{ slot: number | string; fence: number | string }>(
+        SQL_LEASE_ACQUIRE,
+        [key, holder, limit, ttlMs],
+      );
       const row = rows[0];
-      return row === undefined ? undefined : { key, slot: Number(row.slot), holder };
+      // `bigint` crosses as text; a sequence stays under 2^53 for any fleet's lifetime.
+      return row === undefined
+        ? undefined
+        : { key, slot: Number(row.slot), holder, fence: Number(row.fence) };
     },
     async renew(lease, ttlMs) {
       const rows = await exec().query<{ slot: number | string }>(SQL_LEASE_RENEW, [
@@ -201,7 +202,7 @@ function postgresLeaseStore(exec: () => PgExecutor): LeaseStore {
 }
 
 export function postgresJobDriver(options: PostgresJobDriverOptions = {}): JobDriver {
-  const clock = options.clock ?? systemClock;
+  const clock = options.clock ?? liveClock;
   let executor: PgExecutor | undefined;
   const exec = (): PgExecutor => {
     executor ??= resolveExecutor(options.executor);

package/src/driver-redis.ts

@@ -13,7 +13,7 @@
 //   steps:<runId> HASH   step name → record JSON
 
 import type { Clock } from '@ultimat3/core';
-import { finiteCount, finiteOption, systemClock, uuidV7 } from '@ultimat3/core';
+import { finiteCount, finiteOption, liveClock, uuidV7 } from '@ultimat3/core';
 import { redisClient } from '@ultimat3/core/redis';
 import { nowMs } from './clock';
 import type {
@@ -158,7 +158,7 @@ function answerOf(reply: readonly unknown[], at: number, runId: string): Enqueue
 
 export function redisJobDriver(options: RedisJobDriverOptions = {}): JobDriver {
   const client = typedRefusals(options.client ?? redisClient());
-  const clock = options.clock ?? systemClock;
+  const clock = options.clock ?? liveClock;
   const ns = `{${options.prefix ?? 'x:jobs'}}`;
   const doneTtlMs = finiteCount(
     'redisJobDriver',

package/src/driver.ts

@@ -6,10 +6,11 @@
 // stale `jobs.driver` in `app.config.ts` is refused by `defineConfig` (25.0.0; it was read by
 // nothing since 5.0.0). `postgresJobDriver`, `redisJobDriver` and `memoryJobDriver` are the three
 // that exist — 25.0.0 deleted the all-throw `redis` and `nats` stubs, and #710 built the Redis one
-// for real. Swapping the driver is `setJobDriver(other)` and ZERO job-code change, and that
-// is what the interface buys.
+// for real. Swapping the driver is `jobs: other` in `apps/<app>/runtime.ts` and ZERO job-code
+// change, and that is what the interface buys.
 
 import { assert, finiteCount, finiteOption } from '@ultimat3/core';
+import { adapterSlot } from '@ultimat3/core/adapter';
 import type { BackfillLedger } from './backfill-ledger';
 import { ClaimQueuesEmptyError } from './errors';
 import type { JobIntrospection, JobProgress } from './introspection';
@@ -381,15 +382,20 @@ export interface JobDriver {
 export const DEFAULT_QUEUE = 'default';
 export const DEFAULT_VISIBILITY_TIMEOUT_MS = 30_000;
 
-let ambient: JobDriver | undefined;
+/** The installed queue: a stack, so boots and tests release in any order (`adapterSlot`). */
+const slot = adapterSlot<JobDriver | undefined>(() => undefined);
 
-/** Set once at boot over the database binding, never a config key. One instance per process. */
-export function setJobDriver(driver: JobDriver): void {
-  ambient = driver;
+/**
+ * The boot's installer (its release removes THIS install only, in any order — `adapterSlot`), on `@ultimat3/jobs/adapter` and never the `.` barrel: an app swaps the queue
+ * with `jobs:` in `apps/<app>/runtime.ts` (`RuntimeOverrides`), and a test with `@ultimat3/testing`.
+ * One instance per process, never a config key.
+ */
+export function installJobDriver(driver: JobDriver): () => void {
+  return slot.install(driver);
 }
 
 export function jobDriver(): JobDriver | undefined {
-  return ambient;
+  return slot.current();
 }
 
 /**
@@ -398,7 +404,7 @@ export function jobDriver(): JobDriver | undefined {
  * same bun process silently enqueues where it meant to run inline.
  */
 export function resetJobDriver(): void {
-  ambient = undefined;
+  slot.reset();
 }
 
 /**

package/src/error-codes.ts

@@ -67,6 +67,14 @@ export const JOB_OWNED_ERROR_CODES = [
   'X_SUBJECT_TABLE_UNDECLARED',
   'X_SUBJECT_TREATMENT_INVALID',
   'X_SUBJECT_CYCLE',
+  'X_LOCK_NOT_ACQUIRED',
+  'X_LOCK_LOST',
+  'X_LOCK_STORE_MISSING',
+  'X_LEDGER_UNBALANCED',
+  'X_LEDGER_ENTRY_INVALID',
+  'X_PERSONAL_DATA_UNERASED',
+  'X_PERSONAL_DATA_UNREACHED',
+  'X_LEDGER_POSTING_CONFLICT',
 ] as const;
 
 /**
@@ -150,6 +158,15 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
     'a table reaches the declared subject and defineSubject gives it no treatment',
   X_SUBJECT_TREATMENT_INVALID: 'a defineSubject treatment cannot be carried out',
   X_SUBJECT_CYCLE: 'the subject erase has no deletion order',
+  X_LOCK_NOT_ACQUIRED: 'withLock could not take the lock within its wait bound',
+  X_LOCK_LOST: 'a lock withLock held expired before its work finished',
+  X_LOCK_STORE_MISSING: 'withLock found no lease store to hold the lock in',
+  X_LEDGER_UNBALANCED: 'a ledger posting does not sum to zero in every currency',
+  X_LEDGER_ENTRY_INVALID: 'a ledger entry is not a whole-minor Money amount on a named account',
+  X_PERSONAL_DATA_UNERASED:
+    'a subject erase would leave a personalData() column of a reached table',
+  X_PERSONAL_DATA_UNREACHED: 'an entity carries personalData() columns no declared subject reaches',
+  X_LEDGER_POSTING_CONFLICT: 'a ledger posting key is already stored with other entries',
 };
 
 // One unconditional call, so a second package claiming one of jobs' codes throws
@@ -230,4 +247,14 @@ registerErrorRetry({
   X_SUBJECT_TABLE_UNDECLARED: 'terminal',
   X_SUBJECT_TREATMENT_INVALID: 'terminal',
   X_SUBJECT_CYCLE: 'terminal',
+  // A posting is the job's input: the same entries are refused on every attempt.
+  X_LEDGER_UNBALANCED: 'terminal',
+  X_LEDGER_ENTRY_INVALID: 'terminal',
+  // A stored posting does not change between attempts: the repair is a new key.
+  X_LEDGER_POSTING_CONFLICT: 'terminal',
+  // No store is a wiring fact; contention (`X_LOCK_NOT_ACQUIRED`) and a lost lock stay retryable.
+  X_LOCK_STORE_MISSING: 'terminal',
+  // Read off the declarations, as the `X_SUBJECT_*` refusals are: the same every attempt.
+  X_PERSONAL_DATA_UNERASED: 'terminal',
+  X_PERSONAL_DATA_UNREACHED: 'terminal',
 });

package/src/errors-concurrency.ts

@@ -85,7 +85,7 @@ export class ConcurrencyUnenforceableError extends UltimateError {
     super({
       code: 'X_JOB_CONCURRENCY_UNENFORCEABLE',
       cause: `${input.jobs.join(', ')} declare concurrency and the "${input.driver}" jobs driver has no lease store, so the cap would hold per process and the fleet would run concurrency x replicas`,
-      fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or call setJobDriver(postgresJobDriver()) at boot — the pg driver is the one with a lease store`,
+      fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or drop the jobs: field from apps/<app>/runtime.ts so the boot installs postgresJobDriver() over DATABASE_URL — the pg driver is the one with a lease store`,
     });
   }
 }

package/src/errors.ts

@@ -295,7 +295,7 @@ export class CancelUnsupportedError extends UltimateError {
     super({
       code: 'X_JOB_NOT_CANCELLABLE',
       cause: `the "${input.driver}" jobs driver cannot cancel a single job`,
-      fix: 'call setJobDriver(postgresJobDriver()) at boot — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
+      fix: 'postgresJobDriver({ executor })   # jobs: in apps/<app>/runtime.ts, or drop that field and the boot builds it over DATABASE_URL — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
     });
   }
 }

package/src/events-cursor.ts

@@ -0,0 +1,31 @@
+// The order a wait consumes events in, and the cursor a refused event leaves: ONE rule, which the
+// memory bus runs here and `SQL_EVENT_FIND` states in SQL. An event is a `(publishedAt, id)` pair,
+// ms then id, so the cursor carries both and never needs the refused event's row to still exist.
+
+/** Where a search starts: at `afterMs`, and strictly past `(afterMs, afterId)` when there is an id. */
+export interface EventCursor {
+  readonly afterMs: number;
+  readonly afterId?: string;
+}
+
+/**
+ * Whether an event is past the cursor. Without an id, "published at or after `afterMs`"; with one,
+ * strictly after `(afterMs, afterId)` — so an event in the cursor's millisecond with a smaller id
+ * came BEFORE it, whether or not the cursor's own event has since been purged. `''` sorts below
+ * every id, which is what makes the two cases one comparison.
+ */
+export function pastCursor(
+  event: { readonly publishedAt: number; readonly id: string },
+  cursor: EventCursor,
+): boolean {
+  if (event.publishedAt !== cursor.afterMs) return event.publishedAt > cursor.afterMs;
+  return event.id > (cursor.afterId ?? '');
+}
+
+/** The consumption order: earlier `publishedAt` first, the id breaking a tie in one millisecond. */
+export function eventBefore(
+  a: { readonly publishedAt: number; readonly id: string },
+  b: { readonly publishedAt: number; readonly id: string },
+): boolean {
+  return a.publishedAt === b.publishedAt ? a.id < b.id : a.publishedAt < b.publishedAt;
+}

package/src/events-pg.ts

@@ -7,13 +7,15 @@
 // resumes at 12:00:30 must still see an event published at 12:00:10.
 
 import type { PgExecutor } from '@ultimat3/core';
-import { finiteOption, logger, uuidV7 } from '@ultimat3/core';
+import { finiteOption, logger, renderThrowable, uuidV7 } from '@ultimat3/core';
+import { currentTx, outsideTransaction } from '@ultimat3/db';
 import type { DurationInput } from './clock';
 import { finiteDurationMs } from './clock';
-import { SQL_EVENT_PUBLISH_WAKE } from './driver-pg-event-wake-sql';
+import { SQL_EVENT_COMMIT_WAKE, SQL_EVENT_PUBLISH_WAKE } from './driver-pg-event-wake-sql';
 import {
   SQL_EVENT_FIND,
   SQL_EVENT_LIST,
+  SQL_EVENT_MARK,
   SQL_EVENT_NOW,
   SQL_EVENT_PUBLISH,
   SQL_EVENT_PURGE_COUNTED,
@@ -69,6 +71,26 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
     return Number(rows[0]?.removed ?? 0);
   };
 
+  /**
+   * Published inside an open transaction: wake again once it COMMITS. The publish's own wake read
+   * the waiters parked before its statement, and a run parking while the transaction stayed open
+   * met an event it could not see — it woke nobody and sat out the 15 min backstop. Off the
+   * committed transaction (`outsideTransaction`), so the executor reads the pool. Best-effort:
+   * the poll stays the guarantee, so a failed wake is a warning.
+   */
+  const wakeOnCommit = (name: string, correlationKey: string | null): void => {
+    currentTx()?.onCommit(() => {
+      void outsideTransaction(() =>
+        exec.query(SQL_EVENT_COMMIT_WAKE, [name, correlationKey]),
+      ).catch((error: unknown) =>
+        logger.warn('jobs.event.commit-wake-failed', {
+          event: name,
+          error: renderThrowable(error),
+        }),
+      );
+    });
+  };
+
   return {
     stored: true,
     wakes,
@@ -77,6 +99,12 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
       return Number(rows[0]?.now);
     },
 
+    async mark() {
+      const rows = await exec.query<{ now: number | string; horizon: string }>(SQL_EVENT_MARK, []);
+      // One row always: a select with no `from`. Read as `now()` reads its own.
+      return { at: Number(rows[0]?.now), horizon: String(rows[0]?.horizon) };
+    },
+
     async publish(name, payload, publishOptions = {}) {
       const id = uuidV7();
       const ttlMs =
@@ -97,6 +125,7 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
         event: name,
         correlationKey: publishOptions.correlationKey ?? null,
       });
+      if (wakes) wakeOnCommit(name, publishOptions.correlationKey ?? null);
       // The stamps the statement wrote, read back: what a consumer will compare against.
       return {
         id,
@@ -110,17 +139,34 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
       };
     },
 
-    /** Earliest match at or after `afterMs` and past `afterId`, so a step consumes in order. */
-    async find(name, correlationKey, afterMs, afterId) {
+    /** Earliest match past the cursor, or unseen by `horizon`, so a step consumes in order. */
+    async find(name, correlationKey, afterMs, afterId, horizon, since) {
       const rows = await exec.query<{
         id: string;
         payload: unknown;
         published_at: number | string;
-      }>(SQL_EVENT_FIND, [name, correlationKey ?? null, afterMs, afterId ?? null]);
+        horizon: string;
+      }>(SQL_EVENT_FIND, [
+        name,
+        correlationKey ?? null,
+        afterMs,
+        afterId ?? null,
+        horizon ?? null,
+        since?.at ?? afterMs,
+        (since === undefined ? horizon : since.horizon) ?? null,
+        since?.refused === undefined || since.refused.length === 0
+          ? null
+          : JSON.stringify(since.refused),
+      ]);
       const row = rows[0];
       return row === undefined
         ? undefined
-        : { id: row.id, payload: row.payload, publishedAt: Number(row.published_at) };
+        : {
+            id: row.id,
+            payload: row.payload,
+            publishedAt: Number(row.published_at),
+            horizon: row.horizon,
+          };
     },
 
     async list(name) {