@ultimat3/jobs 22.15.0 → 24.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CLAUDE.md +165 -124
  2. package/README.md +360 -21
  3. package/package.json +6 -6
  4. package/src/backfill-ledger.ts +24 -2
  5. package/src/backfill-pass.ts +22 -5
  6. package/src/claim-exhausted.ts +76 -0
  7. package/src/concurrency.ts +144 -0
  8. package/src/counters.ts +105 -0
  9. package/src/describe.ts +29 -0
  10. package/src/driver-memory-operator.ts +319 -0
  11. package/src/driver-memory.ts +146 -59
  12. package/src/driver-nats.ts +7 -4
  13. package/src/driver-pg-ddl.ts +71 -0
  14. package/src/driver-pg-jobs-sql.ts +96 -25
  15. package/src/driver-pg-operator-sql.ts +323 -0
  16. package/src/driver-pg-operator.ts +316 -0
  17. package/src/driver-pg-outbox-sql.ts +110 -0
  18. package/src/driver-pg-rows.ts +34 -3
  19. package/src/driver-pg-settle-sql.ts +71 -0
  20. package/src/driver-pg-sql.ts +171 -151
  21. package/src/driver-pg-wake-sql.ts +76 -0
  22. package/src/driver-pg.ts +150 -105
  23. package/src/driver-redis.ts +7 -4
  24. package/src/driver.ts +133 -35
  25. package/src/enqueue-signal.ts +50 -0
  26. package/src/errors-concurrency.ts +91 -0
  27. package/src/errors-operator.ts +102 -0
  28. package/src/errors-requeue.ts +21 -3
  29. package/src/errors.ts +18 -30
  30. package/src/events-pg.ts +45 -45
  31. package/src/events.ts +38 -2
  32. package/src/execute.ts +161 -81
  33. package/src/export.ts +23 -4
  34. package/src/heartbeat.ts +9 -3
  35. package/src/idle-backoff.ts +74 -0
  36. package/src/index.ts +115 -16
  37. package/src/inspect-operator.ts +49 -0
  38. package/src/inspect.ts +46 -5
  39. package/src/introspection.ts +274 -0
  40. package/src/job.ts +83 -21
  41. package/src/leases.ts +49 -9
  42. package/src/limits.ts +33 -7
  43. package/src/metrics.ts +6 -1
  44. package/src/outbox-pg.ts +4 -2
  45. package/src/outbox-relay.ts +114 -39
  46. package/src/outbox.ts +59 -8
  47. package/src/progress.ts +78 -0
  48. package/src/purge.ts +1 -1
  49. package/src/queue-wake.ts +219 -0
  50. package/src/redact-input.ts +26 -0
  51. package/src/renewal-timer.ts +28 -10
  52. package/src/retry-classification.ts +13 -1
  53. package/src/retry.ts +10 -2
  54. package/src/run-deadline.ts +57 -0
  55. package/src/scheduler-leader.ts +26 -0
  56. package/src/scheduler-occurrences.ts +76 -0
  57. package/src/scheduler-pg.ts +58 -4
  58. package/src/scheduler-state.ts +73 -0
  59. package/src/scheduler.ts +150 -111
  60. package/src/settled.ts +161 -0
  61. package/src/steps-suspension.ts +46 -0
  62. package/src/steps-timeout.ts +33 -0
  63. package/src/steps.ts +55 -81
  64. package/src/worker-admit.ts +167 -0
  65. package/src/worker-fleet-slots.ts +157 -54
  66. package/src/worker-hand-back.ts +6 -1
  67. package/src/worker-key-busy.ts +107 -0
  68. package/src/worker-loop.ts +169 -0
  69. package/src/worker-registry.ts +118 -0
  70. package/src/worker-run.ts +9 -0
  71. package/src/worker-types.ts +23 -0
  72. package/src/worker.ts +142 -130
@@ -3,28 +3,42 @@
3
3
  // real claim/ack/nack paths rather than a mock that always succeeds.
4
4
 
5
5
  import type { Clock } from '@ultimat3/core';
6
- import { assert, finiteCount, systemClock, uuid } from '@ultimat3/core';
6
+ import { finiteCount, systemClock, uuid } from '@ultimat3/core';
7
7
  import type { BackfillLedger } from './backfill-ledger';
8
8
  import { createMemoryBackfillLedger } from './backfill-ledger';
9
9
  import { nowMs } from './clock';
10
+ import { nackOutcome } from './counters';
10
11
  import type {
12
+ AckOptions,
11
13
  ClaimedJob,
14
+ ClaimIdentity,
12
15
  ClaimOptions,
13
16
  EnqueueRequest,
14
17
  EnqueueResult,
15
18
  HeartbeatOptions,
16
19
  JobDriver,
17
- JobFilter,
18
- JobIntrospection,
19
20
  JobRecord,
20
21
  NackOptions,
21
22
  QueueStats,
22
23
  } from './driver';
23
- import { assertClaimBounds, assertClaimQueues, DEFAULT_QUEUE, REQUEUEABLE_STATES } from './driver';
24
- import { JobDuplicateError } from './errors';
25
- import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
24
+ import {
25
+ assertClaimBounds,
26
+ assertClaimQueues,
27
+ DEFAULT_QUEUE,
28
+ LEASE_LAPSED_FINAL_ATTEMPT,
29
+ LIVE_STATES,
30
+ nackState,
31
+ REQUEUEABLE_STATES,
32
+ } from './driver';
33
+ import { createMemoryOperator } from './driver-memory-operator';
34
+ import { signalEnqueued } from './enqueue-signal';
35
+ import { JobDuplicateError, LeaseLostError } from './errors';
36
+ import { JobNotFoundError, JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
37
+ import type { JobIntrospection } from './introspection';
38
+ import { MAX_ERROR_STACK_LENGTH } from './introspection';
26
39
  import type { LeaseStore } from './leases';
27
40
  import { createMemoryLeaseStore } from './leases';
41
+ import { isFinalAttempt } from './retry';
28
42
  import type { StepStore } from './steps';
29
43
  import { createMemoryStepStore } from './steps-memory';
30
44
 
@@ -37,8 +51,6 @@ export interface MemoryDriverOptions {
37
51
  readonly leases?: LeaseStore;
38
52
  }
39
53
 
40
- const LIVE_STATES = new Set(['ready', 'delayed', 'running', 'suspended']);
41
-
42
54
  /**
43
55
  * The in-memory driver's own type: `JobDriver` with `close` REQUIRED.
44
56
  *
@@ -52,7 +64,21 @@ export type MemoryJobDriver = JobDriver & { close(): Promise<void> };
52
64
 
53
65
  export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJobDriver {
54
66
  const clock = options.clock ?? systemClock;
55
- const steps = options.steps ?? createMemoryStepStore();
67
+ const stepStore = options.steps ?? createMemoryStepStore();
68
+ // `SQL_STEP_PUT`'s fence: a write made under a claim lands only while that claim holds the row.
69
+ const steps: StepStore = {
70
+ ...stepStore,
71
+ put(record, by) {
72
+ const row = by === undefined ? undefined : jobs.get(by.jobId);
73
+ if (
74
+ by !== undefined &&
75
+ !(row?.state === 'running' && row.claimedBy === by.workerId && row.claim === by.claim)
76
+ ) {
77
+ return Promise.reject(new LeaseLostError({ job: by.job, jobId: by.jobId }));
78
+ }
79
+ return stepStore.put(record);
80
+ },
81
+ };
56
82
  const backfills = options.backfills ?? createMemoryBackfillLedger(clock);
57
83
  const leases =
58
84
  options.leases ?? createMemoryLeaseStore(options.clock === undefined ? {} : { clock });
@@ -93,32 +119,47 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
93
119
  * lease deadline — which `x jobs show` prints, and which is the very pair the claim scan's
94
120
  * lease-expiry branch reads to decide a row was abandoned.
95
121
  */
96
- const settle = (id: string, patch: Partial<JobRecord>): void => {
122
+ const settle = (id: string, patch: Partial<JobRecord>, drop?: 'stack'): void => {
97
123
  const existing = jobs.get(id);
98
124
  if (existing === undefined) return;
99
125
  const { visibleAt: _visibleAt, claimedBy: _claimedBy, ...released } = existing;
100
- jobs.set(id, { ...released, ...patch, updatedAt: nowMs(clock) });
126
+ // `drop: 'stack'` is `SQL_NACK`'s `case when $5 is null then last_error_stack else $7 end`
127
+ // with no stack bound: a NEW failure replaces the old one's stack even with nothing — the key
128
+ // is removed, never set to `undefined`, as a null column is absent from a pg record.
129
+ const { lastErrorStack: _stack, ...unstacked } = released;
130
+ jobs.set(id, {
131
+ ...(drop === 'stack' ? unstacked : released),
132
+ ...patch,
133
+ updatedAt: nowMs(clock),
134
+ });
135
+ };
136
+
137
+ const operator = createMemoryOperator({
138
+ jobs,
139
+ steps,
140
+ clock,
141
+ settle,
142
+ liveHolder: (record) => {
143
+ const holder = liveByKey(record.name, record.idempotencyKey, record.tenantId);
144
+ return holder === undefined || holder.id === record.id ? undefined : holder;
145
+ },
146
+ });
147
+
148
+ /** The row, when this worker still holds it `running` — the fence both settles share. */
149
+ const heldBy = (jobId: string, by: ClaimIdentity): JobRecord | undefined => {
150
+ const record = jobs.get(jobId);
151
+ return record?.state === 'running' &&
152
+ record.claimedBy === by.workerId &&
153
+ record.claim === by.claim
154
+ ? record
155
+ : undefined;
101
156
  };
102
157
 
103
158
  const introspect: JobIntrospection = {
104
159
  job(jobId) {
105
160
  return Promise.resolve(jobs.get(jobId));
106
161
  },
107
- // `async` for the reason `claim` is: a refused bound must REJECT here exactly as it does on the
108
- // pg driver, and a synchronous throw out of a method typed `Promise<…>` is a second answer to
109
- // one question.
110
- async list(filter: JobFilter = {}) {
111
- const rows = [...jobs.values()]
112
- .filter((record) => filter.queue === undefined || record.queue === filter.queue)
113
- .filter((record) => filter.name === undefined || record.name === filter.name)
114
- .filter((record) => filter.state === undefined || record.state === filter.state)
115
- // NEWEST first, as `createPgDriver`'s `order by created_at desc` is. Ascending here meant
116
- // `x jobs ls` answered one thing against `x dev` and the opposite in production — and,
117
- // because the limit is applied after the sort, a default page of the hundred OLDEST rows.
118
- .sort((a, b) => b.createdAt - a.createdAt)
119
- .slice(0, finiteCount('the memory driver list', 'limit', filter.limit ?? 100));
120
- return rows;
121
- },
162
+ ...operator.members,
122
163
  async deadLetters(limit = 100) {
123
164
  const rows = [...jobs.values()]
124
165
  .filter((record) => record.state === 'dead')
@@ -127,13 +168,9 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
127
168
  return rows;
128
169
  },
129
170
  async requeue(jobId, requeueOptions) {
130
- const existing = jobs.get(jobId);
131
- assert(
132
- existing !== undefined,
133
- `no job ${jobId} in the memory driver`,
134
- 'requeue a job id returned by enqueue() or stats() — the memory driver holds no state across processes, so an id from another run will not resolve',
135
- );
136
- const record = existing;
171
+ const record = jobs.get(jobId);
172
+ // The one answer both drivers give an id nobody queued — it was `X_INVARIANT` here.
173
+ if (record === undefined) throw new JobNotFoundError({ jobId, driver: 'memory' });
137
174
  // The two refusals `SQL_JOB_REQUEUE` and `SQL_JOB_LIVE_HOLDER` give, before anything moves.
138
175
  if (!REQUEUEABLE_STATES.has(record.state)) {
139
176
  throw new JobNotRequeueableError({ jobId, state: record.state });
@@ -143,7 +180,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
143
180
  throw requeueKeyTaken({ ...record, holderId: holder.id });
144
181
  }
145
182
  if (requeueOptions?.fromStep !== undefined) {
146
- // The target step and every step that started AFTER it — `SQL_STEPS_FROM`, whose header
183
+ // The target step and every step that started AFTER it — `SQL_JOB_REQUEUE`, whose header
147
184
  // says why a tie is kept. Earlier steps stay memoized; a later one replaying the old run's
148
185
  // result is what this prevents.
149
186
  const all = await steps.list(record.runId);
@@ -158,14 +195,18 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
158
195
  }
159
196
  // `settle` releases the claim — `SQL_JOB_REQUEUE` writes `claimed_by = null` too.
160
197
  settle(jobId, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
198
+ signalEnqueued(record.queue);
161
199
  const next = jobs.get(jobId);
162
200
  return next ?? record;
163
201
  },
164
202
  cancel(jobId, reason) {
165
203
  const existing = jobs.get(jobId);
166
- // `state !== 'done'`, mirroring `SQL_CANCEL`: a job that already finished has nothing to
167
- // stop, and cancelling it would rewrite a terminal row an operator is reading as success.
168
- if (existing === undefined || existing.state === 'done') return Promise.resolve(undefined);
204
+ // LIVE rows only, mirroring `SQL_CANCEL`: a job that already finished has nothing to stop.
205
+ // The fence was `state !== 'done'`, which let a cancel REWRITE a dead letter — the record of
206
+ // a failure, and its `lastError`, became the record of an operator's typo.
207
+ if (existing === undefined || !LIVE_STATES.has(existing.state)) {
208
+ return Promise.resolve(undefined);
209
+ }
169
210
  // `settle`, not `update`: a cancellation RELEASES the claim, and `SQL_CANCEL` writes
170
211
  // `visible_at = null, claimed_by = null` with the state. Left stamped, a cancelled row named
171
212
  // the worker still holding it and carried that attempt's lease deadline — the pair
@@ -190,6 +231,12 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
190
231
  // `Promise<…>` is a second answer to one question — caught by different code, and an
191
232
  // unhandled exception rather than a settled promise wherever the caller holds the promise.
192
233
  async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
234
+ // A caller-allocated id that already names a row is that row's publish, repeated: the same
235
+ // job, whatever state it has reached — `SQL_ENQUEUE`'s `not exists` is the pg half.
236
+ const published = request.id === undefined ? undefined : jobs.get(request.id);
237
+ if (published !== undefined) {
238
+ return { id: published.id, runId: published.runId, deduped: true };
239
+ }
193
240
  const existing = liveByKey(request.name, request.idempotencyKey, request.tenantId);
194
241
  if (existing !== undefined) {
195
242
  if (request.onConflict === 'error') {
@@ -205,10 +252,13 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
205
252
  const at = nowMs(clock);
206
253
  const runAt = request.runAt ?? at;
207
254
  const record: JobRecord = {
208
- id: uuid(),
255
+ id: request.id ?? uuid(),
209
256
  name: request.name,
210
257
  queue: request.queue || DEFAULT_QUEUE,
211
- input: request.input,
258
+ // Through JSON, exactly as the pg driver binds it (`JSON.stringify(request.input ?? null)`):
259
+ // a live reference kept a `Date`, an `undefined` member and a non-enumerable property the
260
+ // queue in production never stores, so a job that read one passed here and failed there.
261
+ input: JSON.parse(JSON.stringify(request.input ?? null)) as unknown,
212
262
  idempotencyKey: request.idempotencyKey,
213
263
  runId: request.runId ?? uuid(),
214
264
  attempt: 0,
@@ -235,6 +285,8 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
235
285
  const wanted = new Set(claimOptions.queues);
236
286
  const claimable = [...jobs.values()]
237
287
  .filter((record) => wanted.has(record.queue))
288
+ // A paused queue is never claimed — `SQL_CLAIM`'s `not exists` over `x_job_pauses`.
289
+ .filter((record) => !operator.isQueuePaused(record.queue))
238
290
  .filter((record) => {
239
291
  if (record.runAt > at) return false;
240
292
  if (record.state === 'ready' || record.state === 'delayed') return true;
@@ -246,12 +298,30 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
246
298
  .slice(0, claimOptions.limit);
247
299
 
248
300
  const out: ClaimedJob[] = [];
301
+ const buried: JobRecord[] = [];
249
302
  for (const record of claimable) {
303
+ // `SQL_CLAIM`'s second arm. A `running` row reaches this loop only with a lapsed lease, and
304
+ // one that lapsed on its FINAL attempt has nothing left to be re-delivered for: handing
305
+ // it out was an attempt past `maxAttempts`, every lease, forever — a job that kills its
306
+ // worker took one per visibility timeout and never reached the dead-letter queue.
307
+ if (
308
+ record.state === 'running' &&
309
+ isFinalAttempt({ attempts: record.maxAttempts }, record.attempt)
310
+ ) {
311
+ // `$5` of the statement: a job declaring `retry.deadLetter: false` is dropped.
312
+ const state = claimOptions.dropExhausted?.includes(record.name) ? 'failed' : 'dead';
313
+ settle(record.id, { state, lastError: LEASE_LAPSED_FINAL_ATTEMPT }, 'stack');
314
+ operator.counters.add(record.name, state, 0, at);
315
+ const dead = jobs.get(record.id);
316
+ if (dead !== undefined) buried.push(dead);
317
+ continue;
318
+ }
250
319
  const claimed: ClaimedJob = {
251
320
  ...record,
252
321
  state: 'running',
253
322
  attempt: record.attempt + 1,
254
323
  claimedBy: claimOptions.workerId,
324
+ claim: (record.claim ?? 0) + 1,
255
325
  claimedAt: at,
256
326
  visibleAt: at + claimOptions.visibilityTimeoutMs,
257
327
  updatedAt: at,
@@ -259,32 +329,33 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
259
329
  jobs.set(record.id, claimed);
260
330
  out.push(claimed);
261
331
  }
262
- return Promise.resolve(out);
332
+ if (buried.length > 0) claimOptions.onExhausted?.(buried);
333
+ return out;
263
334
  },
264
335
 
265
- // Both settlements are FENCED on `running`, as `SQL_ACK`/`SQL_NACK` are: an ack from a worker
266
- // whose job was cancelled — or whose lease lapsed and whose job another worker re-claimed —
267
- // would otherwise overwrite a row it no longer owns.
268
- ack(jobId: string): Promise<void> {
269
- if (jobs.get(jobId)?.state !== 'running') return Promise.resolve();
336
+ // Both settlements are FENCED on `running` AND on the claimer, as `SQL_ACK`/`SQL_NACK` are: an
337
+ // ack from a worker whose job was cancelled — or whose lease lapsed and whose job another
338
+ // worker re-claimed — would otherwise overwrite a row it no longer owns. The counter moves in
339
+ // the same step as the row, so the two cannot disagree.
340
+ ack(jobId: string, by: AckOptions): Promise<boolean> {
341
+ const record = heldBy(jobId, by);
342
+ if (record === undefined) return Promise.resolve(false);
270
343
  settle(jobId, { state: 'done' });
271
- return Promise.resolve();
344
+ if (by.counted !== false) {
345
+ operator.counters.add(record.name, 'done', by.durationMs ?? 0, nowMs(clock));
346
+ }
347
+ return Promise.resolve(true);
272
348
  },
273
349
 
274
- nack(jobId: string, nackOptions: NackOptions): Promise<void> {
275
- const record = jobs.get(jobId);
276
- if (record === undefined || record.state !== 'running') return Promise.resolve();
350
+ nack(jobId: string, nackOptions: NackOptions): Promise<boolean> {
351
+ const record = heldBy(jobId, nackOptions);
352
+ if (record === undefined) return Promise.resolve(false);
277
353
  const at = nowMs(clock);
278
354
  const counts = nackOptions.countsAsAttempt !== false;
279
355
  const patch: Partial<JobRecord> = {
280
356
  // `park`, never `counts`: parking is what leaves the ready bucket, and burning an attempt
281
357
  // is a separate fact. A shed sets neither and stays `ready`, which is what it is.
282
- state:
283
- nackOptions.deadLetter === true
284
- ? 'dead'
285
- : nackOptions.park === true
286
- ? 'suspended'
287
- : 'ready',
358
+ state: nackState(nackOptions),
288
359
  runAt: at + nackOptions.delayMs,
289
360
  // A suspension must not burn an attempt, or a 3-day sleep dead-letters the run. Floored
290
361
  // where `SQL_NACK` floors it (`greatest(attempt - 1, 0)`): the fence above is what keeps
@@ -292,9 +363,17 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
292
363
  // being read as the only one.
293
364
  attempt: counts ? record.attempt : Math.max(0, record.attempt - 1),
294
365
  ...(nackOptions.error === undefined ? {} : { lastError: nackOptions.error }),
366
+ // The stack belongs to the ERROR beside it, as `SQL_NACK` binds it: no error, no stack.
367
+ ...(nackOptions.error === undefined || nackOptions.stack === undefined
368
+ ? {}
369
+ : { lastErrorStack: nackOptions.stack.slice(0, MAX_ERROR_STACK_LENGTH) }),
295
370
  };
296
- settle(jobId, patch);
297
- return Promise.resolve();
371
+ settle(jobId, patch, nackOptions.error === undefined ? undefined : 'stack');
372
+ const outcome = nackOutcome(nackOptions);
373
+ if (outcome !== undefined) {
374
+ operator.counters.add(record.name, outcome, nackOptions.durationMs ?? 0, at);
375
+ }
376
+ return Promise.resolve(true);
298
377
  },
299
378
 
300
379
  heartbeat(jobId: string, heartbeatOptions: HeartbeatOptions): Promise<boolean> {
@@ -304,7 +383,9 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
304
383
  if (
305
384
  record === undefined ||
306
385
  record.state !== 'running' ||
307
- (heartbeatOptions.workerId !== undefined && record.claimedBy !== heartbeatOptions.workerId)
386
+ (heartbeatOptions.workerId !== undefined &&
387
+ record.claimedBy !== heartbeatOptions.workerId) ||
388
+ (heartbeatOptions.claim !== undefined && record.claim !== heartbeatOptions.claim)
308
389
  ) {
309
390
  return Promise.resolve(false);
310
391
  }
@@ -322,6 +403,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
322
403
  delayed: 0,
323
404
  running: 0,
324
405
  suspended: 0,
406
+ failed: 0,
325
407
  dead: 0,
326
408
  oldestReadyMs: 0,
327
409
  };
@@ -337,10 +419,15 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
337
419
  } else if (waiting) next.delayed += 1;
338
420
  else if (record.state === 'running') next.running += 1;
339
421
  else if (record.state === 'suspended') next.suspended += 1;
422
+ else if (record.state === 'failed') next.failed += 1;
340
423
  else if (record.state === 'dead') next.dead += 1;
341
424
  byQueue.set(record.queue, next);
342
425
  }
343
- return Promise.resolve([...byQueue.values()].sort((a, b) => a.queue.localeCompare(b.queue)));
426
+ // Code units, as `collate "C"` orders them in `SQL_STATS` — never `localeCompare`, which
427
+ // answers by the runtime's locale and so differently from the database on the same names.
428
+ return Promise.resolve(
429
+ [...byQueue.values()].sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0)),
430
+ );
344
431
  },
345
432
 
346
433
  close(): Promise<void> {
@@ -3,6 +3,7 @@
3
3
  // a durable pull consumer per worker pool (`fetch` == claim, `ack`/`nak` map 1:1),
4
4
  // `ack_wait` as the visibility timeout, and a KV bucket for step records.
5
5
 
6
+ import { NotImplementedError } from '@ultimat3/core';
6
7
  import type {
7
8
  ClaimedJob,
8
9
  ClaimOptions,
@@ -13,7 +14,6 @@ import type {
13
14
  NackOptions,
14
15
  QueueStats,
15
16
  } from './driver';
16
- import { JobsNotImplementedError } from './errors';
17
17
  import type { StepRecord, StepStore } from './steps';
18
18
 
19
19
  // Names the seam that actually replaces the stub, and NOTHING ELSE — the two other repairs this
@@ -34,7 +34,10 @@ const FIX =
34
34
  'call setJobDriver(createPgDriver()) at boot instead of this driver; nothing needs moving first, because enqueue here refuses too, so no job was ever written to it';
35
35
 
36
36
  const unavailable = (method: string): never => {
37
- throw new JobsNotImplementedError({ feature: `nats jobs driver (${method})`, fix: FIX });
37
+ throw new NotImplementedError({
38
+ cause: `nats jobs driver (${method}) is not implemented`,
39
+ fix: FIX,
40
+ });
38
41
  };
39
42
 
40
43
  const natsStepStore = (): StepStore => ({
@@ -70,10 +73,10 @@ export function createNatsDriver(_options: NatsDriverOptions = {}): JobDriver {
70
73
  claim(_options: ClaimOptions): Promise<readonly ClaimedJob[]> {
71
74
  return unavailable('claim');
72
75
  },
73
- ack(_jobId: string): Promise<void> {
76
+ ack(_jobId: string): Promise<boolean> {
74
77
  return unavailable('ack');
75
78
  },
76
- nack(_jobId: string, _options: NackOptions): Promise<void> {
79
+ nack(_jobId: string, _options: NackOptions): Promise<boolean> {
77
80
  return unavailable('nack');
78
81
  },
79
82
  heartbeat(_jobId: string, _options: HeartbeatOptions): Promise<boolean> {
@@ -132,6 +132,12 @@ alter table x_outbox add column if not exists claimed_at timestamptz;
132
132
 
133
133
  alter table x_outbox add column if not exists claimed_by text;
134
134
 
135
+ -- The run the published job carries, allocated when the row is STAGED so the enqueue that staged
136
+ -- it can answer which run it started. The row id is the published job id for the same reason.
137
+ -- Added by alter because x_outbox shipped without it. A row staged before this column existed is
138
+ -- given a run id by the default, which is what the driver would have minted at its publish.
139
+ alter table x_outbox add column if not exists run_id uuid not null default gen_random_uuid();
140
+
135
141
  -- The scheduler watermark. Without a durable one a redeployed scheduler has no idea what the
136
142
  -- pod it replaced already fired, so runRound takes the arming branch and every occurrence
137
143
  -- between the two processes is dropped with nothing logged.
@@ -141,6 +147,13 @@ create table if not exists x_scheduler_state (
141
147
  updated_at timestamptz not null default now()
142
148
  );
143
149
 
150
+ -- The last occurrence that DISPATCHED, and when. Apart from the watermark, which arming a task or
151
+ -- skipping missed occurrences also moves: an operator asking when a task last ran is asking about
152
+ -- jobs that were queued. Added by alter because x_scheduler_state shipped without them.
153
+ alter table x_scheduler_state add column if not exists fired_occurrence_at timestamptz;
154
+
155
+ alter table x_scheduler_state add column if not exists fired_at timestamptz;
156
+
144
157
  -- Leader election as an EXPIRING LEASE rather than a session advisory lock: the executor this
145
158
  -- package is handed is a pool, and a session-level pg_try_advisory_lock is released the moment
146
159
  -- that connection goes back to it. A row with an expiry needs no connection affinity at all.
@@ -176,4 +189,62 @@ create table if not exists x_job_events (
176
189
 
177
190
  create index if not exists x_job_events_lookup_idx
178
191
  on x_job_events (name, published_at);
192
+
193
+ -- What the body last reported through progress, as one value: done, total, note and the instant.
194
+ -- A column and not a table, because it is read with the row and dies with it. Added by alter
195
+ -- because x_jobs shipped without it.
196
+ alter table x_jobs add column if not exists progress jsonb;
197
+
198
+ -- The stack of the thrown value last_error describes, bounded by the driver before it is bound.
199
+ alter table x_jobs add column if not exists last_error_stack text;
200
+
201
+ -- How many times the row has been claimed, moved by the claim itself and never reset. A settle is
202
+ -- fenced on it as well as on claimed_by: a worker that takes back its own lapsed job has the same
203
+ -- id as the body still unwinding from the first claim, and only the ordinal tells the two apart.
204
+ alter table x_jobs add column if not exists claims int not null default 0;
205
+
206
+ -- The keyset a paged listing seeks by. Newest first, id as the tiebreak, so a page is an index
207
+ -- range and never a sort of the whole table.
208
+ create index if not exists x_jobs_created_idx on x_jobs (created_at desc, id desc);
209
+
210
+ -- A paused queue or task, one row each. A queue pause is a fact about the QUEUE and not a column
211
+ -- on every one of its rows: the claim reads this table with not exists, and a pause that had to
212
+ -- rewrite a million rows would not take effect within one poll interval.
213
+ create table if not exists x_job_pauses (
214
+ kind text not null,
215
+ name text not null,
216
+ paused_at timestamptz not null default now(),
217
+ primary key (kind, name)
218
+ );
219
+
220
+ -- The worker registry. One row per live worker, rewritten on its heartbeat and expired by time
221
+ -- like a lease: a killed worker announces nothing and its row stops being read. Never cleaned up
222
+ -- by anything that has to be running.
223
+ create table if not exists x_job_workers (
224
+ id text primary key,
225
+ host text not null,
226
+ queues jsonb not null,
227
+ concurrency int not null,
228
+ in_flight jsonb not null,
229
+ started_at timestamptz not null,
230
+ heartbeat_at timestamptz not null default now(),
231
+ expires_at timestamptz not null
232
+ );
233
+
234
+ -- Per-job history, in buckets. A settle adds to the one-minute bucket in the SAME statement that
235
+ -- moves the row, and the scheduler leader folds old buckets into wider ones, so the key count per
236
+ -- job name is fixed whatever the throughput.
237
+ create table if not exists x_job_counters (
238
+ job text not null,
239
+ bucket_ms int not null,
240
+ bucket_start timestamptz not null,
241
+ done bigint not null default 0,
242
+ retried bigint not null default 0,
243
+ failed bigint not null default 0,
244
+ dead bigint not null default 0,
245
+ duration_ms bigint not null default 0,
246
+ primary key (job, bucket_ms, bucket_start)
247
+ );
248
+
249
+ create index if not exists x_job_counters_age_idx on x_job_counters (bucket_ms, bucket_start);
179
250
  `.trim();
@@ -3,34 +3,95 @@
3
3
  // none decodes `timestamptz` as TEXT, so `toJobRecord` read `Number('2026-01-01 00:00:00+00')`
4
4
  // and `x jobs ls` / `show` / `cancel` printed `NaN` for every timestamp.
5
5
 
6
+ import { JOBS_WAKE_CHANNEL } from './driver-pg-wake-sql';
7
+
6
8
  /**
7
9
  * The `JobRow` shape as a projection. Asking Postgres for epoch ms is what makes the decoding
8
10
  * independent of the client's type map — never `select *`, whose correctness is the driver's
9
11
  * opinion about `timestamptz` rather than this statement's.
10
12
  */
11
13
  export const JOB_ROW_COLUMNS = `id, name, queue, input, idempotency_key, run_id, attempt,
12
- max_attempts, state, tenant_id, last_error, claimed_by,
13
- traceparent, enqueued_by,
14
+ max_attempts, state, tenant_id, last_error, last_error_stack, claimed_by, claims,
15
+ traceparent, enqueued_by, progress,
14
16
  (extract(epoch from run_at) * 1000)::bigint as run_at,
15
17
  (extract(epoch from visible_at) * 1000)::bigint as visible_at,
16
18
  (extract(epoch from created_at) * 1000)::bigint as created_at,
17
19
  (extract(epoch from updated_at) * 1000)::bigint as updated_at`;
18
20
 
21
+ /** `JOB_ROW_COLUMNS` by their output names — what a statement reading a derived table selects. */
22
+ const JOB_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attempt, max_attempts,
23
+ state, tenant_id, last_error, last_error_stack, claimed_by, claims, traceparent,
24
+ enqueued_by, progress, run_at, visible_at, created_at, updated_at`;
25
+
19
26
  export const SQL_JOB_GET = `
20
27
  select ${JOB_ROW_COLUMNS}
21
28
  from x_jobs where id = $1
22
29
  `.trim();
23
30
 
31
+ /**
32
+ * The predicates a page shares, whichever way it reads. $1 queue, $2 name, $3 state, $5 id prefix,
33
+ * $6 created from (ms), $7 created to (ms), $10 tenant. The prefix is compared, never `like`d: a
34
+ * `_` or `%` in it is a character, as `startsWith` reads it in the memory driver.
35
+ */
36
+ const LIST_MATCH = `($1::text is null or queue = $1)
37
+ and ($2::text is null or name = $2)
38
+ and ($3::text is null or state = $3)
39
+ and ($5::text is null or left(id::text, length($5::text)) = $5::text)
40
+ and ($6::bigint is null or created_at >= to_timestamp($6::bigint / 1000.0))
41
+ and ($7::bigint is null or created_at < to_timestamp($7::bigint / 1000.0))
42
+ and ($10::text is null or tenant_id = $10)`;
43
+
44
+ /**
45
+ * One page, newest first, sought by KEYSET. `(created_at, id) < (cursor)` is a row comparison the
46
+ * `x_jobs_created_idx` range answers directly; an offset would re-read every row before it and
47
+ * shift under an insert.
48
+ *
49
+ * The cursor carries the last row's `createdAt` in epoch MS, and the column holds microseconds —
50
+ * so the seek reads the cursor row's own `created_at` by id rather than trusting a rounded value,
51
+ * which would skip or repeat its millisecond-mates. A cursor row deleted mid-walk falls back to
52
+ * the millisecond after it: rows of that one millisecond may then repeat, and none is skipped.
53
+ *
54
+ * `LIST_MATCH`'s parameters, plus $4 limit, $8 cursor createdAt (ms), $9 cursor id.
55
+ */
24
56
  export const SQL_JOB_LIST = `
25
57
  select ${JOB_ROW_COLUMNS}
26
58
  from x_jobs
27
- where ($1::text is null or queue = $1)
28
- and ($2::text is null or name = $2)
29
- and ($3::text is null or state = $3)
30
- order by created_at desc
59
+ where ${LIST_MATCH}
60
+ and ($9::uuid is null or (created_at, id) < (
61
+ coalesce(
62
+ (select c.created_at from x_jobs c where c.id = $9::uuid),
63
+ to_timestamp(($8::bigint + 1) / 1000.0)
64
+ ),
65
+ $9::uuid
66
+ ))
67
+ order by created_at desc, id desc
31
68
  limit $4
32
69
  `.trim();
33
70
 
71
+ /**
72
+ * The page BEFORE a cursor: the `limit` rows nearest it on the newer side — read oldest first so
73
+ * `limit` keeps the nearest, then turned newest first like every page. The same parameters as
74
+ * `SQL_JOB_LIST`, $8/$9 being the first row of the page the reader is leaving. A cursor row
75
+ * deleted meanwhile falls back to its millisecond: a row of it may repeat, none is skipped.
76
+ */
77
+ export const SQL_JOB_LIST_BEFORE = `
78
+ select ${JOB_ROW_NAMES} from (
79
+ select ${JOB_ROW_COLUMNS}
80
+ from x_jobs
81
+ where ${LIST_MATCH}
82
+ and (created_at, id) > (
83
+ coalesce(
84
+ (select c.created_at from x_jobs c where c.id = $9::uuid),
85
+ to_timestamp($8::bigint / 1000.0)
86
+ ),
87
+ $9::uuid
88
+ )
89
+ order by created_at asc, id asc
90
+ limit $4
91
+ ) page
92
+ order by created_at desc, id desc
93
+ `.trim();
94
+
34
95
  export const SQL_JOB_DEAD_LETTERS = `
35
96
  select ${JOB_ROW_COLUMNS}
36
97
  from x_jobs where state = 'dead' order by updated_at desc limit $1
@@ -41,14 +102,37 @@ select ${JOB_ROW_COLUMNS}
41
102
  * to FINISHED states and releasing the claim: requeueing a running row left `claimed_by` set with
42
103
  * state `ready`, and a second worker claimed it — the job ran twice. `REQUEUEABLE_STATES`
43
104
  * (`driver.ts`) is the same set; a row this matches nothing for was refused before it was sent,
44
- * or raced to live in between.
105
+ * or raced to live in between. `woke` announces the row in the statement that made it claimable
106
+ * (`driver-pg-wake-sql.ts`): `x jobs retry` is a job starting now, not at an idle worker's poll.
107
+ *
108
+ * $2 is `fromStep`, or null — and the steps are dropped HERE, off the row this statement moved.
109
+ * They were a statement of their own, sent first: a requeue that then matched nothing (the row
110
+ * raced to live) had already deleted a running job's steps, and a crash between the two left a
111
+ * dead letter with its steps gone and no retry queued.
112
+ *
113
+ * "From that step onward": the step `fromStep` names and every step that started AFTER it.
114
+ * Deleting the named step alone left later steps replaying results computed from the old run.
115
+ * Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
116
+ * later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
45
117
  */
46
118
  export const SQL_JOB_REQUEUE = `
47
- update x_jobs
48
- set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
49
- claimed_by = null, visible_at = null
50
- where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
51
- returning ${JOB_ROW_COLUMNS}
119
+ with requeued as (
120
+ update x_jobs
121
+ set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
122
+ claimed_by = null, visible_at = null
123
+ where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
124
+ returning ${JOB_ROW_COLUMNS},
125
+ pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
126
+ ), dropped as (
127
+ delete from x_job_steps s
128
+ using requeued r
129
+ where $2::text is not null
130
+ and s.run_id = r.run_id
131
+ and (s.name = $2::text
132
+ or s.started_at > (select t.started_at from x_job_steps t
133
+ where t.run_id = r.run_id and t.name = $2::text))
134
+ )
135
+ select ${JOB_ROW_NAMES}, woke from requeued
52
136
  `.trim();
53
137
 
54
138
  /**
@@ -61,16 +145,3 @@ select id from x_jobs
61
145
  and id <> $4 and state in ('ready', 'delayed', 'running', 'suspended')
62
146
  limit 1
63
147
  `.trim();
64
-
65
- /**
66
- * "From that step onward": the step `fromStep` names and every step that started AFTER it.
67
- * Deleting the named step alone left later steps replaying results computed from the old run.
68
- * Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
69
- * later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
70
- */
71
- export const SQL_STEPS_FROM = `
72
- delete from x_job_steps
73
- where run_id = $1
74
- and (name = $2
75
- or started_at > (select started_at from x_job_steps where run_id = $1 and name = $2))
76
- `.trim();