@ultimat3/jobs 30.8.0 → 31.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CLAUDE.md +72 -107
  2. package/README.md +202 -55
  3. package/package.json +8 -7
  4. package/src/adapter.ts +37 -0
  5. package/src/backfill-ledger.ts +2 -2
  6. package/src/backfill-pass.ts +2 -1
  7. package/src/backfill-rate.ts +34 -5
  8. package/src/clock.ts +2 -2
  9. package/src/drain-wait.ts +1 -0
  10. package/src/driver-memory.ts +2 -2
  11. package/src/driver-pg-ddl.ts +19 -0
  12. package/src/driver-pg-event-wake-sql.ts +43 -22
  13. package/src/driver-pg-sql.ts +68 -27
  14. package/src/driver-pg.ts +11 -10
  15. package/src/driver-redis.ts +2 -2
  16. package/src/driver.ts +14 -8
  17. package/src/error-codes.ts +27 -0
  18. package/src/errors-concurrency.ts +1 -1
  19. package/src/errors.ts +1 -1
  20. package/src/events-cursor.ts +31 -0
  21. package/src/events-pg.ts +52 -6
  22. package/src/events.ts +72 -36
  23. package/src/export.ts +2 -2
  24. package/src/gdpr-audit.ts +75 -0
  25. package/src/gdpr-coverage.ts +94 -0
  26. package/src/gdpr-errors.ts +35 -0
  27. package/src/import.ts +2 -2
  28. package/src/index.ts +45 -206
  29. package/src/inspect-operator.ts +2 -2
  30. package/src/inspect.ts +1 -1
  31. package/src/job-instrument.ts +115 -0
  32. package/src/job.ts +41 -4
  33. package/src/jobs-retention.ts +23 -6
  34. package/src/leases.ts +13 -3
  35. package/src/ledger-book.ts +168 -0
  36. package/src/ledger-errors.ts +48 -0
  37. package/src/ledger.ts +145 -0
  38. package/src/limits.ts +57 -19
  39. package/src/lock-errors.ts +54 -0
  40. package/src/lock.ts +176 -0
  41. package/src/outbox-relay.ts +4 -0
  42. package/src/outbox.ts +3 -3
  43. package/src/partitions.ts +119 -0
  44. package/src/reindex-sweep.ts +94 -0
  45. package/src/reindex.ts +147 -0
  46. package/src/retry-classification.ts +3 -2
  47. package/src/retry-jitter.ts +32 -0
  48. package/src/retry.ts +13 -16
  49. package/src/scheduler.ts +10 -6
  50. package/src/steps-api.ts +44 -5
  51. package/src/steps-suspension.ts +34 -3
  52. package/src/steps-wait.ts +52 -4
  53. package/src/steps.ts +1 -0
  54. package/src/subject-jobs.ts +12 -4
  55. package/src/webhook.ts +2 -2
  56. package/src/worker-admit.ts +8 -5
  57. package/src/worker-drain-cutoff.ts +1 -0
  58. package/src/worker-loop.ts +28 -5
  59. package/src/worker-registry.ts +7 -5
  60. package/src/worker-run.ts +53 -9
  61. package/src/worker.ts +8 -8
@@ -5,31 +5,15 @@
5
5
  import { JOBS_WAKE_CHANNEL } from './driver-pg-wake-sql';
6
6
 
7
7
  /**
8
- * `SQL_EVENT_PUBLISH`, plus the wake. A waiter is a `waiting` step on this event whose key is this
9
- * event's or absent — `SQL_EVENT_FIND`'s match, read from the step's side — over the partial
10
- * `x_job_steps_waiting_idx`, so a publish with no waiter reads no step at all.
11
- *
12
- * Woken means `run_at` moved to now on a `suspended` row that was not already due: a running row,
13
- * a finished one, or one a concurrent publish already woke (the row lock re-checks `run_at` after
14
- * the first commit) is left alone. The poll stays the guarantee; this only brings it forward.
15
- *
16
- * One `pg_notify` per DISTINCT queue of the woken rows, and none when nothing woke: never
17
- * unconditional, for the commit lock `driver-pg-wake-sql.ts` describes. The payload is the queue.
18
- *
19
- * $1 id, $2 name, $3 payload, $4 correlation key, $5 ttl (ms).
8
+ * The waiters of event `$name` keyed `$key`, made due, and their queues told: the half of a waking
9
+ * publish after its insert. Shared, so the publish and the commit-time wake are one rule.
20
10
  */
21
- export const SQL_EVENT_PUBLISH_WAKE = `
22
- with ev as (
23
- insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at)
24
- values ($1, $2, $3::jsonb, $4, statement_timestamp(),
25
- statement_timestamp() + ($5::bigint * interval '1 millisecond'))
26
- returning published_at, expires_at
27
- ), waiters as (
11
+ const wakeWaiters = (name: string, key: string): string => `waiters as (
28
12
  select distinct s.run_id
29
13
  from x_job_steps s
30
14
  where s.status = 'waiting'
31
- and s.event = $2
32
- and (s.correlation_key is null or s.correlation_key = $4::text)
15
+ and s.event = ${name}
16
+ and (s.correlation_key is null or s.correlation_key = ${key}::text)
33
17
  ), woken as (
34
18
  update x_jobs j
35
19
  set run_at = statement_timestamp(),
@@ -42,10 +26,47 @@ with ev as (
42
26
  ), told as (
43
27
  select count(pg_notify('${JOBS_WAKE_CHANNEL}', q.queue))::int as told
44
28
  from (select distinct queue from woken) q
45
- )
29
+ )`;
30
+
31
+ /**
32
+ * `SQL_EVENT_PUBLISH`, plus the wake. A waiter is a `waiting` step on this event whose key is this
33
+ * event's or absent — `SQL_EVENT_FIND`'s match, read from the step's side — over the partial
34
+ * `x_job_steps_waiting_idx`, so a publish with no waiter reads no step at all.
35
+ *
36
+ * Woken means `run_at` moved to now on a `suspended` row that was not already due: a running row,
37
+ * a finished one, or one a concurrent publish already woke (the row lock re-checks `run_at` after
38
+ * the first commit) is left alone. The poll stays the guarantee; this only brings it forward.
39
+ *
40
+ * One `pg_notify` per DISTINCT queue of the woken rows, and none when nothing woke: never
41
+ * unconditional, for the commit lock `driver-pg-wake-sql.ts` describes. The payload is the queue.
42
+ *
43
+ * $1 id, $2 name, $3 payload, $4 correlation key, $5 ttl (ms).
44
+ */
45
+ export const SQL_EVENT_PUBLISH_WAKE = `
46
+ with ev as (
47
+ insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at, xact)
48
+ values ($1, $2, $3::jsonb, $4, date_trunc('milliseconds', statement_timestamp()),
49
+ statement_timestamp() + ($5::bigint * interval '1 millisecond'),
50
+ pg_current_xact_id())
51
+ returning published_at, expires_at
52
+ ), ${wakeWaiters('$2', '$4')}
46
53
  select floor(extract(epoch from ev.published_at) * 1000)::bigint as published_at,
47
54
  floor(extract(epoch from ev.expires_at) * 1000)::bigint as expires_at,
48
55
  (select count(*) from woken)::int as woken,
49
56
  (select told from told) as told
50
57
  from ev
51
58
  `.trim();
59
+
60
+ /**
61
+ * The same wake, run on the pool once the PUBLISHING transaction has committed (`Tx.onCommit`):
62
+ * the publish's own wake reads the waiters parked before its statement, and a run that parked
63
+ * while the transaction was still open — its `find` could not see the uncommitted event — woke
64
+ * nobody and waited out the 15 min backstop. After the commit every waiter is either parked, and
65
+ * woken here, or parks later and sees the event in its second `find`.
66
+ *
67
+ * $1 name, $2 correlation key.
68
+ */
69
+ export const SQL_EVENT_COMMIT_WAKE = `
70
+ with ${wakeWaiters('$1', '$2')}
71
+ select (select count(*) from woken)::int as woken, (select told from told) as told
72
+ `.trim();
@@ -196,22 +196,27 @@ on conflict (task_name) do update
196
196
  * `(lease_key, slot)` primary key serialises two workers that picked the same slot, and the
197
197
  * loser's `do update` is guarded on the slot having expired — a live slot returns no row, so the
198
198
  * grant is never doubled. Under contention this can refuse a slot that is genuinely free; a
199
- * refusal costs one poll interval and an over-grant costs the guarantee.
199
+ * refusal costs one poll interval and an over-grant costs the guarantee. `nextval` runs on the ONE
200
+ * candidate the inner `limit 1` kept, and a takeover takes the new number too: the fence.
200
201
  */
201
202
  export const SQL_LEASE_ACQUIRE = `
202
- insert into x_job_leases (lease_key, slot, holder, expires_at)
203
- select $1, s.slot, $2, now() + ($4::bigint * interval '1 millisecond')
204
- from generate_series(0, $3::int - 1) as s(slot)
205
- where not exists (
206
- select 1 from x_job_leases l
207
- where l.lease_key = $1 and l.slot = s.slot and l.expires_at > now()
208
- )
209
- order by s.slot
210
- limit 1
203
+ insert into x_job_leases (lease_key, slot, holder, expires_at, fence)
204
+ select f.lease_key, f.slot, f.holder, f.expires_at, nextval('x_job_lease_fence')
205
+ from (
206
+ select $1::text as lease_key, s.slot, $2::text as holder,
207
+ now() + ($4::bigint * interval '1 millisecond') as expires_at
208
+ from generate_series(0, $3::int - 1) as s(slot)
209
+ where not exists (
210
+ select 1 from x_job_leases l
211
+ where l.lease_key = $1 and l.slot = s.slot and l.expires_at > now()
212
+ )
213
+ order by s.slot
214
+ limit 1
215
+ ) as f
211
216
  on conflict (lease_key, slot) do update
212
- set holder = excluded.holder, expires_at = excluded.expires_at
217
+ set holder = excluded.holder, expires_at = excluded.expires_at, fence = excluded.fence
213
218
  where x_job_leases.expires_at <= now()
214
- returning slot
219
+ returning slot, fence
215
220
  `.trim();
216
221
 
217
222
  /**
@@ -258,12 +263,18 @@ select holder from x_job_leases
258
263
  * on commit, or "before" a question asked while the transaction was open
259
264
  * (`events-pg.live.test.ts`). Outside a transaction the two are the same instant.
260
265
  *
266
+ * The stamp is TRUNCATED to the millisecond, the unit every cursor is written in, so the
267
+ * `(published_at, id)` order the find sorts by is exactly the order a cursor compares — a
268
+ * microsecond stamp put an event of the cursor's millisecond with a smaller id after it. `xact`
269
+ * is the publishing transaction: what a wait's horizon asks about (`SQL_EVENT_FIND`).
270
+ *
261
271
  * $1 id, $2 name, $3 payload, $4 correlation key, $5 ttl (ms).
262
272
  */
263
273
  export const SQL_EVENT_PUBLISH = `
264
- insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at)
265
- values ($1, $2, $3::jsonb, $4, statement_timestamp(),
266
- statement_timestamp() + ($5::bigint * interval '1 millisecond'))
274
+ insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at, xact)
275
+ values ($1, $2, $3::jsonb, $4, date_trunc('milliseconds', statement_timestamp()),
276
+ statement_timestamp() + ($5::bigint * interval '1 millisecond'),
277
+ pg_current_xact_id())
267
278
  returning floor(extract(epoch from published_at) * 1000)::bigint as published_at,
268
279
  floor(extract(epoch from expires_at) * 1000)::bigint as expires_at
269
280
  `.trim();
@@ -274,26 +285,56 @@ select floor(extract(epoch from statement_timestamp()) * 1000)::bigint as now
274
285
  `.trim();
275
286
 
276
287
  /**
277
- * Earliest matching event at or after `afterMs`, so a resumed step consumes events in publication
278
- * order rather than jumping to the newest one — the memory bus's rule, in SQL. `$4` is the cursor a
279
- * refused event leaves: rows at or before it in `(published_at, id)` order are passed over, and a
280
- * cursor whose row is gone excludes nothing. `floor`, as `SQL_EVENT_PUBLISH` stamps: a rounding
281
- * cast answered a stamp up to half a ms LATER than the row, which a floor then stepped past. The
282
- * order is QUALIFIED: a bare `published_at` there names the floored output column, whose ties in
283
- * one millisecond then fell to the random id.
288
+ * A new wait's starting line: the bus's clock and the snapshot this statement reads under, in one
289
+ * statement so no commit falls between them. Every event that snapshot cannot see — a publish
290
+ * still in flight, or one not yet begun — is owed to the wait whatever its stamp.
291
+ */
292
+ export const SQL_EVENT_MARK = `
293
+ select floor(extract(epoch from statement_timestamp()) * 1000)::bigint as now,
294
+ pg_current_snapshot()::text as horizon
295
+ `.trim();
296
+
297
+ /**
298
+ * Earliest matching event past the cursor, so a resumed step consumes events in publication order
299
+ * rather than jumping to the newest one — `events-cursor.ts`'s rule, in SQL. Past the cursor is
300
+ * `(published_at, id) > ($3, $4)`, the nil uuid standing in for "no id": compared, never looked
301
+ * up, so a cursor whose event was purged still excludes what it excluded. The order is QUALIFIED:
302
+ * a bare `published_at` there names the floored output column.
303
+ *
304
+ * OR invisible to `$5`, the wait's horizon (`SQL_EVENT_MARK`, or the snapshot of the search that
305
+ * found a refused event): stamped at INSERT, visible at COMMIT, so a publish committed after the
306
+ * wait began can carry a stamp before it. `xact >= xmin` is implied by "not visible" and is what
307
+ * lets the planner answer that half from `x_job_events_xact_idx`. A row from before `xact` was
308
+ * written has it null and is matched by its stamp alone. The cursor's own event never matches —
309
+ * a row stamped at microseconds, from before truncation, floors to a cursor it would sort past.
310
+ * The horizon of THIS search rides the hit, for the refusal that may follow it.
311
+ *
312
+ * `$6`/`$7` are the WAIT's own line (`EventSince`), which a refusal never moves: an event is the
313
+ * wait's only when stamped at or after it, or unseen by the mark's horizon. Without it, a refused
314
+ * owed event's cursor let one committed BEFORE the wait sort past it and be handed out.
284
315
  *
285
- * $1 name, $2 correlation key, $3 floor (ms), $4 cursor event id.
316
+ * $1 name, $2 correlation key, $3 cursor (ms), $4 cursor event id, $5 the cursor search's horizon,
317
+ * $6 the wait's start (ms), $7 the wait's horizon (pg_snapshot text), $8 the ids it refused behind
318
+ * its cursor (JSON text, never a bound array: issue #384).
286
319
  */
287
320
  export const SQL_EVENT_FIND = `
288
- select id, payload, floor(extract(epoch from published_at) * 1000)::bigint as published_at
321
+ select id, payload, floor(extract(epoch from published_at) * 1000)::bigint as published_at,
322
+ pg_current_snapshot()::text as horizon
289
323
  from x_job_events e
290
324
  where name = $1
291
325
  and expires_at > statement_timestamp()
292
- and published_at >= to_timestamp($3 / 1000.0)
293
326
  and ($2::text is null or correlation_key = $2)
327
+ and e.id is distinct from $4::uuid
294
328
  and not exists (
295
- select 1 from x_job_events c
296
- where c.id = $4::uuid and (e.published_at, e.id) <= (c.published_at, c.id))
329
+ select 1 from json_array_elements_text(coalesce($8::json, '[]'::json)) r
330
+ where r.value::uuid = e.id)
331
+ and (e.published_at >= to_timestamp($6 / 1000.0)
332
+ or (e.xact >= pg_snapshot_xmin($7::pg_snapshot)
333
+ and not pg_visible_in_snapshot(e.xact, $7::pg_snapshot)))
334
+ and ((e.published_at, e.id) >
335
+ (to_timestamp($3 / 1000.0), coalesce($4::uuid, '00000000-0000-0000-0000-000000000000'))
336
+ or (e.xact >= pg_snapshot_xmin($5::pg_snapshot)
337
+ and not pg_visible_in_snapshot(e.xact, $5::pg_snapshot)))
297
338
  order by e.published_at, e.id
298
339
  limit 1
299
340
  `.trim();
package/src/driver-pg.ts CHANGED
@@ -5,7 +5,7 @@
5
5
  // they run against in `driver-pg-ddl.ts`, and the row-to-record decoding in `driver-pg-rows.ts`.
6
6
 
7
7
  import type { Clock, PgExecutor } from '@ultimat3/core';
8
- import { finiteCount, renderCauseValue, systemClock, uuidV7 } from '@ultimat3/core';
8
+ import { finiteCount, liveClock, renderCauseValue, uuidV7 } from '@ultimat3/core';
9
9
  import type { BackfillLedger } from './backfill-ledger';
10
10
  import { listedRunId } from './backfill-ledger';
11
11
  import { nowMs } from './clock';
@@ -86,7 +86,7 @@ function resolveExecutor(injected: PgExecutor | undefined): PgExecutor {
86
86
  // be "not configured". An executor is injected by the boot or the driver has none.
87
87
  cause:
88
88
  'postgresJobDriver() was called with no executor, and this driver has no ambient fallback',
89
- fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: setJobDriver(memoryJobDriver())',
89
+ fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: export const runtime = { jobs: memoryJobDriver() } in apps/<app>/runtime.ts',
90
90
  });
91
91
  }
92
92
 
@@ -165,14 +165,15 @@ function postgresLeaseStore(exec: () => PgExecutor): LeaseStore {
165
165
  return {
166
166
  async acquire(key, limit, ttlMs, holder) {
167
167
  if (limit <= 0) return undefined;
168
- const rows = await exec().query<{ slot: number | string }>(SQL_LEASE_ACQUIRE, [
169
- key,
170
- holder,
171
- limit,
172
- ttlMs,
173
- ]);
168
+ const rows = await exec().query<{ slot: number | string; fence: number | string }>(
169
+ SQL_LEASE_ACQUIRE,
170
+ [key, holder, limit, ttlMs],
171
+ );
174
172
  const row = rows[0];
175
- return row === undefined ? undefined : { key, slot: Number(row.slot), holder };
173
+ // `bigint` crosses as text; a sequence stays under 2^53 for any fleet's lifetime.
174
+ return row === undefined
175
+ ? undefined
176
+ : { key, slot: Number(row.slot), holder, fence: Number(row.fence) };
176
177
  },
177
178
  async renew(lease, ttlMs) {
178
179
  const rows = await exec().query<{ slot: number | string }>(SQL_LEASE_RENEW, [
@@ -201,7 +202,7 @@ function postgresLeaseStore(exec: () => PgExecutor): LeaseStore {
201
202
  }
202
203
 
203
204
  export function postgresJobDriver(options: PostgresJobDriverOptions = {}): JobDriver {
204
- const clock = options.clock ?? systemClock;
205
+ const clock = options.clock ?? liveClock;
205
206
  let executor: PgExecutor | undefined;
206
207
  const exec = (): PgExecutor => {
207
208
  executor ??= resolveExecutor(options.executor);
@@ -13,7 +13,7 @@
13
13
  // steps:<runId> HASH step name → record JSON
14
14
 
15
15
  import type { Clock } from '@ultimat3/core';
16
- import { finiteCount, finiteOption, systemClock, uuidV7 } from '@ultimat3/core';
16
+ import { finiteCount, finiteOption, liveClock, uuidV7 } from '@ultimat3/core';
17
17
  import { redisClient } from '@ultimat3/core/redis';
18
18
  import { nowMs } from './clock';
19
19
  import type {
@@ -158,7 +158,7 @@ function answerOf(reply: readonly unknown[], at: number, runId: string): Enqueue
158
158
 
159
159
  export function redisJobDriver(options: RedisJobDriverOptions = {}): JobDriver {
160
160
  const client = typedRefusals(options.client ?? redisClient());
161
- const clock = options.clock ?? systemClock;
161
+ const clock = options.clock ?? liveClock;
162
162
  const ns = `{${options.prefix ?? 'x:jobs'}}`;
163
163
  const doneTtlMs = finiteCount(
164
164
  'redisJobDriver',
package/src/driver.ts CHANGED
@@ -6,10 +6,11 @@
6
6
  // stale `jobs.driver` in `app.config.ts` is refused by `defineConfig` (25.0.0; it was read by
7
7
  // nothing since 5.0.0). `postgresJobDriver`, `redisJobDriver` and `memoryJobDriver` are the three
8
8
  // that exist — 25.0.0 deleted the all-throw `redis` and `nats` stubs, and #710 built the Redis one
9
- // for real. Swapping the driver is `setJobDriver(other)` and ZERO job-code change, and that
10
- // is what the interface buys.
9
+ // for real. Swapping the driver is `jobs: other` in `apps/<app>/runtime.ts` and ZERO job-code
10
+ // change, and that is what the interface buys.
11
11
 
12
12
  import { assert, finiteCount, finiteOption } from '@ultimat3/core';
13
+ import { adapterSlot } from '@ultimat3/core/adapter';
13
14
  import type { BackfillLedger } from './backfill-ledger';
14
15
  import { ClaimQueuesEmptyError } from './errors';
15
16
  import type { JobIntrospection, JobProgress } from './introspection';
@@ -381,15 +382,20 @@ export interface JobDriver {
381
382
  export const DEFAULT_QUEUE = 'default';
382
383
  export const DEFAULT_VISIBILITY_TIMEOUT_MS = 30_000;
383
384
 
384
- let ambient: JobDriver | undefined;
385
+ /** The installed queue: a stack, so boots and tests release in any order (`adapterSlot`). */
386
+ const slot = adapterSlot<JobDriver | undefined>(() => undefined);
385
387
 
386
- /** Set once at boot over the database binding, never a config key. One instance per process. */
387
- export function setJobDriver(driver: JobDriver): void {
388
- ambient = driver;
388
+ /**
389
+ * The boot's installer (its release removes THIS install only, in any order — `adapterSlot`), on `@ultimat3/jobs/adapter` and never the `.` barrel: an app swaps the queue
390
+ * with `jobs:` in `apps/<app>/runtime.ts` (`RuntimeOverrides`), and a test with `@ultimat3/testing`.
391
+ * One instance per process, never a config key.
392
+ */
393
+ export function installJobDriver(driver: JobDriver): () => void {
394
+ return slot.install(driver);
389
395
  }
390
396
 
391
397
  export function jobDriver(): JobDriver | undefined {
392
- return ambient;
398
+ return slot.current();
393
399
  }
394
400
 
395
401
  /**
@@ -398,7 +404,7 @@ export function jobDriver(): JobDriver | undefined {
398
404
  * same bun process silently enqueues where it meant to run inline.
399
405
  */
400
406
  export function resetJobDriver(): void {
401
- ambient = undefined;
407
+ slot.reset();
402
408
  }
403
409
 
404
410
  /**
@@ -67,6 +67,14 @@ export const JOB_OWNED_ERROR_CODES = [
67
67
  'X_SUBJECT_TABLE_UNDECLARED',
68
68
  'X_SUBJECT_TREATMENT_INVALID',
69
69
  'X_SUBJECT_CYCLE',
70
+ 'X_LOCK_NOT_ACQUIRED',
71
+ 'X_LOCK_LOST',
72
+ 'X_LOCK_STORE_MISSING',
73
+ 'X_LEDGER_UNBALANCED',
74
+ 'X_LEDGER_ENTRY_INVALID',
75
+ 'X_PERSONAL_DATA_UNERASED',
76
+ 'X_PERSONAL_DATA_UNREACHED',
77
+ 'X_LEDGER_POSTING_CONFLICT',
70
78
  ] as const;
71
79
 
72
80
  /**
@@ -150,6 +158,15 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
150
158
  'a table reaches the declared subject and defineSubject gives it no treatment',
151
159
  X_SUBJECT_TREATMENT_INVALID: 'a defineSubject treatment cannot be carried out',
152
160
  X_SUBJECT_CYCLE: 'the subject erase has no deletion order',
161
+ X_LOCK_NOT_ACQUIRED: 'withLock could not take the lock within its wait bound',
162
+ X_LOCK_LOST: 'a lock withLock held expired before its work finished',
163
+ X_LOCK_STORE_MISSING: 'withLock found no lease store to hold the lock in',
164
+ X_LEDGER_UNBALANCED: 'a ledger posting does not sum to zero in every currency',
165
+ X_LEDGER_ENTRY_INVALID: 'a ledger entry is not a whole-minor Money amount on a named account',
166
+ X_PERSONAL_DATA_UNERASED:
167
+ 'a subject erase would leave a personalData() column of a reached table',
168
+ X_PERSONAL_DATA_UNREACHED: 'an entity carries personalData() columns no declared subject reaches',
169
+ X_LEDGER_POSTING_CONFLICT: 'a ledger posting key is already stored with other entries',
153
170
  };
154
171
 
155
172
  // One unconditional call, so a second package claiming one of jobs' codes throws
@@ -230,4 +247,14 @@ registerErrorRetry({
230
247
  X_SUBJECT_TABLE_UNDECLARED: 'terminal',
231
248
  X_SUBJECT_TREATMENT_INVALID: 'terminal',
232
249
  X_SUBJECT_CYCLE: 'terminal',
250
+ // A posting is the job's input: the same entries are refused on every attempt.
251
+ X_LEDGER_UNBALANCED: 'terminal',
252
+ X_LEDGER_ENTRY_INVALID: 'terminal',
253
+ // A stored posting does not change between attempts: the repair is a new key.
254
+ X_LEDGER_POSTING_CONFLICT: 'terminal',
255
+ // No store is a wiring fact; contention (`X_LOCK_NOT_ACQUIRED`) and a lost lock stay retryable.
256
+ X_LOCK_STORE_MISSING: 'terminal',
257
+ // Read off the declarations, as the `X_SUBJECT_*` refusals are: the same every attempt.
258
+ X_PERSONAL_DATA_UNERASED: 'terminal',
259
+ X_PERSONAL_DATA_UNREACHED: 'terminal',
233
260
  });
@@ -85,7 +85,7 @@ export class ConcurrencyUnenforceableError extends UltimateError {
85
85
  super({
86
86
  code: 'X_JOB_CONCURRENCY_UNENFORCEABLE',
87
87
  cause: `${input.jobs.join(', ')} declare concurrency and the "${input.driver}" jobs driver has no lease store, so the cap would hold per process and the fleet would run concurrency x replicas`,
88
- fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or call setJobDriver(postgresJobDriver()) at boot — the pg driver is the one with a lease store`,
88
+ fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or drop the jobs: field from apps/<app>/runtime.ts so the boot installs postgresJobDriver() over DATABASE_URL — the pg driver is the one with a lease store`,
89
89
  });
90
90
  }
91
91
  }
package/src/errors.ts CHANGED
@@ -295,7 +295,7 @@ export class CancelUnsupportedError extends UltimateError {
295
295
  super({
296
296
  code: 'X_JOB_NOT_CANCELLABLE',
297
297
  cause: `the "${input.driver}" jobs driver cannot cancel a single job`,
298
- fix: 'call setJobDriver(postgresJobDriver()) at boot — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
298
+ fix: 'postgresJobDriver({ executor }) # jobs: in apps/<app>/runtime.ts, or drop that field and the boot builds it over DATABASE_URL — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
299
299
  });
300
300
  }
301
301
  }
@@ -0,0 +1,31 @@
1
+ // The order a wait consumes events in, and the cursor a refused event leaves: ONE rule, which the
2
+ // memory bus runs here and `SQL_EVENT_FIND` states in SQL. An event is a `(publishedAt, id)` pair,
3
+ // ms then id, so the cursor carries both and never needs the refused event's row to still exist.
4
+
5
+ /** Where a search starts: at `afterMs`, and strictly past `(afterMs, afterId)` when there is an id. */
6
+ export interface EventCursor {
7
+ readonly afterMs: number;
8
+ readonly afterId?: string;
9
+ }
10
+
11
+ /**
12
+ * Whether an event is past the cursor. Without an id, "published at or after `afterMs`"; with one,
13
+ * strictly after `(afterMs, afterId)` — so an event in the cursor's millisecond with a smaller id
14
+ * came BEFORE it, whether or not the cursor's own event has since been purged. `''` sorts below
15
+ * every id, which is what makes the two cases one comparison.
16
+ */
17
+ export function pastCursor(
18
+ event: { readonly publishedAt: number; readonly id: string },
19
+ cursor: EventCursor,
20
+ ): boolean {
21
+ if (event.publishedAt !== cursor.afterMs) return event.publishedAt > cursor.afterMs;
22
+ return event.id > (cursor.afterId ?? '');
23
+ }
24
+
25
+ /** The consumption order: earlier `publishedAt` first, the id breaking a tie in one millisecond. */
26
+ export function eventBefore(
27
+ a: { readonly publishedAt: number; readonly id: string },
28
+ b: { readonly publishedAt: number; readonly id: string },
29
+ ): boolean {
30
+ return a.publishedAt === b.publishedAt ? a.id < b.id : a.publishedAt < b.publishedAt;
31
+ }
package/src/events-pg.ts CHANGED
@@ -7,13 +7,15 @@
7
7
  // resumes at 12:00:30 must still see an event published at 12:00:10.
8
8
 
9
9
  import type { PgExecutor } from '@ultimat3/core';
10
- import { finiteOption, logger, uuidV7 } from '@ultimat3/core';
10
+ import { finiteOption, logger, renderThrowable, uuidV7 } from '@ultimat3/core';
11
+ import { currentTx, outsideTransaction } from '@ultimat3/db';
11
12
  import type { DurationInput } from './clock';
12
13
  import { finiteDurationMs } from './clock';
13
- import { SQL_EVENT_PUBLISH_WAKE } from './driver-pg-event-wake-sql';
14
+ import { SQL_EVENT_COMMIT_WAKE, SQL_EVENT_PUBLISH_WAKE } from './driver-pg-event-wake-sql';
14
15
  import {
15
16
  SQL_EVENT_FIND,
16
17
  SQL_EVENT_LIST,
18
+ SQL_EVENT_MARK,
17
19
  SQL_EVENT_NOW,
18
20
  SQL_EVENT_PUBLISH,
19
21
  SQL_EVENT_PURGE_COUNTED,
@@ -69,6 +71,26 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
69
71
  return Number(rows[0]?.removed ?? 0);
70
72
  };
71
73
 
74
+ /**
75
+ * Published inside an open transaction: wake again once it COMMITS. The publish's own wake read
76
+ * the waiters parked before its statement, and a run parking while the transaction stayed open
77
+ * met an event it could not see — it woke nobody and sat out the 15 min backstop. Off the
78
+ * committed transaction (`outsideTransaction`), so the executor reads the pool. Best-effort:
79
+ * the poll stays the guarantee, so a failed wake is a warning.
80
+ */
81
+ const wakeOnCommit = (name: string, correlationKey: string | null): void => {
82
+ currentTx()?.onCommit(() => {
83
+ void outsideTransaction(() =>
84
+ exec.query(SQL_EVENT_COMMIT_WAKE, [name, correlationKey]),
85
+ ).catch((error: unknown) =>
86
+ logger.warn('jobs.event.commit-wake-failed', {
87
+ event: name,
88
+ error: renderThrowable(error),
89
+ }),
90
+ );
91
+ });
92
+ };
93
+
72
94
  return {
73
95
  stored: true,
74
96
  wakes,
@@ -77,6 +99,12 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
77
99
  return Number(rows[0]?.now);
78
100
  },
79
101
 
102
+ async mark() {
103
+ const rows = await exec.query<{ now: number | string; horizon: string }>(SQL_EVENT_MARK, []);
104
+ // One row always: a select with no `from`. Read as `now()` reads its own.
105
+ return { at: Number(rows[0]?.now), horizon: String(rows[0]?.horizon) };
106
+ },
107
+
80
108
  async publish(name, payload, publishOptions = {}) {
81
109
  const id = uuidV7();
82
110
  const ttlMs =
@@ -97,6 +125,7 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
97
125
  event: name,
98
126
  correlationKey: publishOptions.correlationKey ?? null,
99
127
  });
128
+ if (wakes) wakeOnCommit(name, publishOptions.correlationKey ?? null);
100
129
  // The stamps the statement wrote, read back: what a consumer will compare against.
101
130
  return {
102
131
  id,
@@ -110,17 +139,34 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
110
139
  };
111
140
  },
112
141
 
113
- /** Earliest match at or after `afterMs` and past `afterId`, so a step consumes in order. */
114
- async find(name, correlationKey, afterMs, afterId) {
142
+ /** Earliest match past the cursor, or unseen by `horizon`, so a step consumes in order. */
143
+ async find(name, correlationKey, afterMs, afterId, horizon, since) {
115
144
  const rows = await exec.query<{
116
145
  id: string;
117
146
  payload: unknown;
118
147
  published_at: number | string;
119
- }>(SQL_EVENT_FIND, [name, correlationKey ?? null, afterMs, afterId ?? null]);
148
+ horizon: string;
149
+ }>(SQL_EVENT_FIND, [
150
+ name,
151
+ correlationKey ?? null,
152
+ afterMs,
153
+ afterId ?? null,
154
+ horizon ?? null,
155
+ since?.at ?? afterMs,
156
+ (since === undefined ? horizon : since.horizon) ?? null,
157
+ since?.refused === undefined || since.refused.length === 0
158
+ ? null
159
+ : JSON.stringify(since.refused),
160
+ ]);
120
161
  const row = rows[0];
121
162
  return row === undefined
122
163
  ? undefined
123
- : { id: row.id, payload: row.payload, publishedAt: Number(row.published_at) };
164
+ : {
165
+ id: row.id,
166
+ payload: row.payload,
167
+ publishedAt: Number(row.published_at),
168
+ horizon: row.horizon,
169
+ };
124
170
  },
125
171
 
126
172
  async list(name) {