@ultimat3/jobs 22.14.0 → 23.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CLAUDE.md +129 -87
  2. package/README.md +329 -19
  3. package/package.json +6 -6
  4. package/src/concurrency.ts +144 -0
  5. package/src/counters.ts +105 -0
  6. package/src/describe.ts +29 -0
  7. package/src/driver-memory-operator.ts +297 -0
  8. package/src/driver-memory.ts +79 -37
  9. package/src/driver-nats.ts +2 -2
  10. package/src/driver-pg-ddl.ts +71 -0
  11. package/src/driver-pg-jobs-sql.ts +71 -8
  12. package/src/driver-pg-operator-sql.ts +303 -0
  13. package/src/driver-pg-operator.ts +300 -0
  14. package/src/driver-pg-outbox-sql.ts +110 -0
  15. package/src/driver-pg-rows.ts +31 -0
  16. package/src/driver-pg-settle-sql.ts +71 -0
  17. package/src/driver-pg-sql.ts +80 -129
  18. package/src/driver-pg-wake-sql.ts +76 -0
  19. package/src/driver-pg.ts +74 -34
  20. package/src/driver-redis.ts +2 -2
  21. package/src/driver.ts +96 -34
  22. package/src/enqueue-signal.ts +50 -0
  23. package/src/errors-concurrency.ts +91 -0
  24. package/src/errors-operator.ts +94 -0
  25. package/src/errors.ts +13 -17
  26. package/src/events-pg.ts +38 -28
  27. package/src/events.ts +25 -0
  28. package/src/execute.ts +160 -81
  29. package/src/export.ts +23 -4
  30. package/src/heartbeat.ts +9 -3
  31. package/src/idle-backoff.ts +74 -0
  32. package/src/index.ts +105 -12
  33. package/src/inspect-operator.ts +49 -0
  34. package/src/inspect.ts +42 -2
  35. package/src/introspection.ts +262 -0
  36. package/src/job.ts +78 -15
  37. package/src/leases.ts +49 -9
  38. package/src/metrics.ts +6 -1
  39. package/src/outbox-pg.ts +3 -0
  40. package/src/outbox-relay.ts +114 -39
  41. package/src/outbox.ts +29 -6
  42. package/src/progress.ts +78 -0
  43. package/src/queue-wake.ts +220 -0
  44. package/src/redact-input.ts +26 -0
  45. package/src/renewal-timer.ts +28 -10
  46. package/src/retry-classification.ts +13 -1
  47. package/src/retry.ts +10 -2
  48. package/src/run-deadline.ts +57 -0
  49. package/src/scheduler-leader.ts +26 -0
  50. package/src/scheduler-occurrences.ts +76 -0
  51. package/src/scheduler-pg.ts +55 -2
  52. package/src/scheduler-state.ts +66 -0
  53. package/src/scheduler.ts +143 -103
  54. package/src/settled.ts +161 -0
  55. package/src/worker-admit.ts +163 -0
  56. package/src/worker-fleet-slots.ts +107 -38
  57. package/src/worker-hand-back.ts +6 -1
  58. package/src/worker-key-busy.ts +79 -0
  59. package/src/worker-loop.ts +145 -0
  60. package/src/worker-registry.ts +103 -0
  61. package/src/worker-run.ts +9 -0
  62. package/src/worker-types.ts +23 -0
  63. package/src/worker.ts +103 -134
@@ -22,20 +22,44 @@ export {
22
22
  SQL_STEPS_FROM,
23
23
  } from './driver-pg-jobs-sql';
24
24
 
25
+ export {
26
+ SQL_OUTBOX_CLAIM,
27
+ SQL_OUTBOX_MARK_PUBLISHED,
28
+ SQL_OUTBOX_PENDING_COUNT,
29
+ SQL_OUTBOX_PUBLISHED_JOB,
30
+ SQL_OUTBOX_RELEASE,
31
+ SQL_OUTBOX_STAGE,
32
+ } from './driver-pg-outbox-sql';
33
+ export { SQL_ACK, SQL_NACK } from './driver-pg-settle-sql';
34
+
25
35
  import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
36
+ import { notifyJobReady } from './driver-pg-wake-sql';
26
37
 
38
+ /**
39
+ * The insert, and its wake: `woke` is the notification's own column, evaluated once per inserted
40
+ * row and absent when the conflict fired. `driver-pg-wake-sql.ts` says when it stays silent.
41
+ *
42
+ * `where not exists` is the outbox's half. A staged row is published under ITS OWN id
43
+ * (`EnqueueRequest.id`), so a publish repeated after a crash meets the job the first one made —
44
+ * in any state — and inserts nothing; the partial index below collapses a repeat only while that
45
+ * job is still live. A select over `values`, so each parameter carries its cast.
46
+ */
27
47
  export const SQL_ENQUEUE = `
28
- insert into x_jobs
29
- (id, name, queue, input, idempotency_key, run_id, max_attempts, state, run_at, tenant_id,
30
- traceparent, enqueued_by)
31
- values
32
- ($1, $2, $3, $4::jsonb, $5, $6, $7,
33
- case when to_timestamp($8 / 1000.0) > now() then 'delayed' else 'ready' end,
34
- to_timestamp($8 / 1000.0), $9, $10, $11)
35
- on conflict (name, (coalesce(tenant_id, '')), idempotency_key)
36
- where state in ('ready', 'delayed', 'running', 'suspended')
37
- do nothing
38
- returning id, run_id
48
+ with inserted as (
49
+ insert into x_jobs
50
+ (id, name, queue, input, idempotency_key, run_id, max_attempts, state, run_at, tenant_id,
51
+ traceparent, enqueued_by)
52
+ select $1::uuid, $2::text, $3::text, $4::jsonb, $5::text, $6::uuid, $7::int,
53
+ case when to_timestamp($8::bigint / 1000.0) > now() then 'delayed' else 'ready' end,
54
+ to_timestamp($8::bigint / 1000.0), $9::text, $10::text, $11::text
55
+ where not exists (select 1 from x_jobs published where published.id = $1::uuid)
56
+ on conflict (name, (coalesce(tenant_id, '')), idempotency_key)
57
+ where state in ('ready', 'delayed', 'running', 'suspended')
58
+ do nothing
59
+ returning id, run_id, queue, run_at
60
+ )
61
+ select i.id, i.run_id,${notifyJobReady('i')}::text as woke
62
+ from inserted i
39
63
  `.trim();
40
64
 
41
65
  /**
@@ -68,6 +92,9 @@ with claimed as (
68
92
  from x_jobs
69
93
  where queue = any($1::text[])
70
94
  and run_at <= now()
95
+ and not exists (
96
+ select 1 from x_job_pauses p where p.kind = 'queue' and p.name = x_jobs.queue
97
+ )
71
98
  and (
72
99
  state in ('ready', 'delayed', 'suspended')
73
100
  or (state = 'running' and visible_at <= now())
@@ -79,13 +106,14 @@ with claimed as (
79
106
  update x_jobs j
80
107
  set state = 'running',
81
108
  attempt = j.attempt + 1,
109
+ claims = j.claims + 1,
82
110
  claimed_by = $3,
83
111
  visible_at = now() + ($4::bigint * interval '1 millisecond'),
84
112
  updated_at = now()
85
113
  from claimed c
86
114
  where j.id = c.id
87
115
  returning j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt,
88
- j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by,
116
+ j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by, j.claims,
89
117
  j.traceparent, j.enqueued_by,
90
118
  (extract(epoch from j.run_at) * 1000)::bigint as run_at,
91
119
  (extract(epoch from j.visible_at) * 1000)::bigint as visible_at,
@@ -93,30 +121,6 @@ returning j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt
93
121
  (extract(epoch from j.updated_at) * 1000)::bigint as updated_at
94
122
  `.trim();
95
123
 
96
- /**
97
- * `and state = 'running'` is a FENCE, not a filter. Without it an ack from the worker that was
98
- * cancelled — or from one whose lease lapsed and whose job another worker already re-claimed —
99
- * overwrites the row it no longer owns: `x jobs cancel` would be undone by the next settle, and
100
- * a re-delivered job would be marked done by the attempt that lost it.
101
- */
102
- export const SQL_ACK = `
103
- update x_jobs
104
- set state = 'done', visible_at = null, claimed_by = null, updated_at = now()
105
- where id = $1 and state = 'running'
106
- `.trim();
107
-
108
- export const SQL_NACK = `
109
- update x_jobs
110
- set state = $2,
111
- attempt = case when $3::boolean then attempt else greatest(attempt - 1, 0) end,
112
- run_at = now() + ($4::bigint * interval '1 millisecond'),
113
- visible_at = null,
114
- claimed_by = null,
115
- last_error = coalesce($5, last_error),
116
- updated_at = now()
117
- where id = $1 and state = 'running'
118
- `.trim();
119
-
120
124
  /**
121
125
  * Stop a job from outside. `state <> 'done'` and not `state = 'ready'`: the runaway backfill this
122
126
  * exists for is `running`, and a job that already finished has nothing to stop. The worker holding
@@ -140,11 +144,12 @@ export const SQL_HEARTBEAT = `
140
144
  update x_jobs
141
145
  set visible_at = now() + ($2::bigint * interval '1 millisecond'), updated_at = now()
142
146
  where id = $1 and state = 'running' and ($3::text is null or claimed_by = $3)
147
+ and ($4::int is null or claims = $4)
143
148
  returning id
144
149
  `.trim();
145
150
 
146
151
  /**
147
- * Five buckets a queue depth is SUMMED from, so a row may land in exactly one. `run_at > now()`
152
+ * Six buckets a queue depth is SUMMED from, so a row may land in exactly one. `run_at > now()`
148
153
  * unqualified was every state's future row: a `step.sleep` job counted as `suspended` AND
149
154
  * `delayed`, a delayed dead-letter as `dead` AND `delayed`. The memory driver's `if/else if` chain
150
155
  * has always been exclusive, so `x jobs` reported one depth under `x dev` and a larger one in
@@ -156,6 +161,7 @@ select queue,
156
161
  count(*) filter (where state in ('ready', 'delayed') and run_at > now()) as delayed,
157
162
  count(*) filter (where state = 'running') as running,
158
163
  count(*) filter (where state = 'suspended') as suspended,
164
+ count(*) filter (where state = 'failed') as failed,
159
165
  count(*) filter (where state = 'dead') as dead,
160
166
  coalesce(max(extract(epoch from now() - run_at)) filter
161
167
  (where state in ('ready', 'delayed') and run_at <= now()), 0) * 1000 as oldest_ready_ms
@@ -254,9 +260,43 @@ export const SQL_LEASE_RELEASE = `
254
260
  delete from x_job_leases where lease_key = $1 and slot = $2::int and holder = $3
255
261
  `.trim();
256
262
 
263
+ /**
264
+ * Who holds the LIVE slots of one key, in slot order. Read only after an acquire was refused, to
265
+ * tell a key another run holds from one this run's own previous claim still does — the acquire
266
+ * itself stays the one atomic decision, and this never grants anything.
267
+ */
268
+ export const SQL_LEASE_HOLDERS = `
269
+ select holder from x_job_leases
270
+ where lease_key = $1 and expires_at > now()
271
+ order by slot
272
+ `.trim();
273
+
274
+ /**
275
+ * Both instants are the DATABASE's: `published_at` is `statement_timestamp()` and the expiry is
276
+ * counted from it.
277
+ * They were bound from the publisher's process clock, and every reader compares them with
278
+ * something else — `now()` in `SQL_EVENT_FIND`, a consumer's "after" — so a pod whose clock was
279
+ * off wrote an event that had expired before it was inserted, or that came "before" a question
280
+ * asked a minute earlier. `floor`, never a rounding cast: a stamp is never later than the row.
281
+ *
282
+ * The STATEMENT's time, never `now()`, in every event statement: `now()` is when the transaction
283
+ * began, so a bus handed a long transaction's connection stamped an event at its start — expired
284
+ * on commit, or "before" a question asked while the transaction was open
285
+ * (`events-pg.live.test.ts`). Outside a transaction the two are the same instant.
286
+ *
287
+ * $1 id, $2 name, $3 payload, $4 correlation key, $5 ttl (ms).
288
+ */
257
289
  export const SQL_EVENT_PUBLISH = `
258
290
  insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at)
259
- values ($1, $2, $3::jsonb, $4, to_timestamp($5 / 1000.0), to_timestamp($6 / 1000.0))
291
+ values ($1, $2, $3::jsonb, $4, statement_timestamp(),
292
+ statement_timestamp() + ($5::bigint * interval '1 millisecond'))
293
+ returning floor(extract(epoch from published_at) * 1000)::bigint as published_at,
294
+ floor(extract(epoch from expires_at) * 1000)::bigint as expires_at
295
+ `.trim();
296
+
297
+ /** The bus's clock — the instant a publish issued now would be stamped with. */
298
+ export const SQL_EVENT_NOW = `
299
+ select floor(extract(epoch from statement_timestamp()) * 1000)::bigint as now
260
300
  `.trim();
261
301
 
262
302
  /**
@@ -267,7 +307,7 @@ export const SQL_EVENT_FIND = `
267
307
  select payload, (extract(epoch from published_at) * 1000)::bigint as published_at
268
308
  from x_job_events
269
309
  where name = $1
270
- and expires_at > now()
310
+ and expires_at > statement_timestamp()
271
311
  and published_at >= to_timestamp($3 / 1000.0)
272
312
  and ($2::text is null or correlation_key = $2)
273
313
  order by published_at
@@ -284,96 +324,7 @@ select id, name, payload, correlation_key,
284
324
  limit $2
285
325
  `.trim();
286
326
 
287
- export const SQL_EVENT_PURGE = `delete from x_job_events where expires_at <= now()`;
288
-
289
- export const SQL_OUTBOX_STAGE = `
290
- insert into x_outbox
291
- (id, job, queue, input, idempotency_key, max_attempts, run_at, staged_at, tenant_id,
292
- traceparent, enqueued_by)
293
- values ($1, $2, $3, $4::jsonb, $5, $6, to_timestamp($7 / 1000.0), to_timestamp($8 / 1000.0),
294
- $9, $10, $11)
295
- `.trim();
296
-
297
- /**
298
- * The claim, and it has to be ONE statement. `for update skip locked` in a bare select holds its
299
- * row locks only until that statement ends — under autocommit, before `claim()` even resolves — so
300
- * two relays polling 200ms apart read the same unpublished rows and both hand them to `enqueue`.
301
- * `SQL_ENQUEUE` collapses the repeat only while the first job is still LIVE: its conflict target
302
- * is a partial index over the live states, so a second publish landing after that job reached a
303
- * terminal state inserts a second row and the handler runs again. (The mechanism is Postgres
304
- * semantics; how often the two orderings line up in a deployment was never measured.)
305
- *
306
- * So the lock and the claim commit together, the CTE shape `SQL_CLAIM` already uses, and
307
- * `claimed_at` is a LEASE: `$2` is the window after which a row a dead relay was holding is
308
- * claimable again, because a claim nothing can expire strands its rows forever.
309
- *
310
- * The outer `select ... order by staged_at, id` is not cosmetic. `update ... returning` has no
311
- * defined row order and the relay publishes in the order it is handed rows, so an app staging
312
- * `createInvoice` then `chargeCard` in one transaction depends on this line.
313
- *
314
- * `, id` is what makes that key TOTAL, and the CTE needs it as much as the projection does: every
315
- * row staged in one transaction shares a `staged_at`, so `staged_at` alone leaves the tie to the
316
- * planner — which rows a `limit` takes, and in which order they publish, then differ between two
317
- * relays and between two runs of one relay. No column was added for it: `id` is a UUIDv7 minted by
318
- * `uuid()`, monotonic and already the primary key, so the tiebreak IS stage order.
319
- */
320
- export const SQL_OUTBOX_CLAIM = `
321
- with claimable as (
322
- select id
323
- from x_outbox
324
- where published_at is null
325
- and (claimed_at is null
326
- or claimed_at <= now() - ($2::bigint * interval '1 millisecond'))
327
- order by staged_at, id
328
- limit $1
329
- for update skip locked
330
- ), claimed as (
331
- update x_outbox o
332
- set claimed_at = now(), claimed_by = $3
333
- from claimable c
334
- where o.id = c.id
335
- returning o.id, o.job, o.queue, o.input, o.idempotency_key, o.max_attempts, o.tenant_id,
336
- o.traceparent, o.enqueued_by, o.claimed_by, o.run_at, o.staged_at
337
- )
338
- select id, job, queue, input, idempotency_key, max_attempts, tenant_id,
339
- traceparent, enqueued_by, claimed_by,
340
- (extract(epoch from run_at) * 1000)::bigint as run_at,
341
- (extract(epoch from staged_at) * 1000)::bigint as staged_at
342
- from claimed
343
- order by staged_at, id
344
- `.trim();
345
-
346
- /**
347
- * Hand a claim back early. The relay stops its batch on the first publish that fails, and without
348
- * this the rows behind it would wait out the whole lease before any relay could retry them — a
349
- * pool blip during a failover becoming tens of seconds of unpublished, committed work.
350
- *
351
- * Fenced on `published_at is null` so it can never unclaim a row some other pass already
352
- * published, AND on `claimed_by` so it can never unclaim one a NEWER claimant now holds: a relay
353
- * whose lease lapsed while it stalled wakes into a world where its batch is another relay's, and
354
- * an unfenced release frees rows that relay is mid-publish on — a third relay claims them and
355
- * publishes them again, which is the duplicate the lease exists to prevent.
356
- */
357
- export const SQL_OUTBOX_RELEASE = `
358
- update x_outbox
359
- set claimed_at = null, claimed_by = null
360
- where id = any($1::uuid[]) and published_at is null and claimed_by = $2
361
- `.trim();
362
-
363
- /**
364
- * Same fence, and here it is the more expensive one to miss: marking a row published is LOSING it,
365
- * so a lapsed claimant stamping a row the current one has not published yet drops that job with
366
- * nothing to notice. `published_at is null` makes the stamp first-writer-wins rather than a
367
- * rewrite of an audit timestamp.
368
- */
369
- export const SQL_OUTBOX_MARK_PUBLISHED = `
370
- update x_outbox set published_at = to_timestamp($2 / 1000.0)
371
- where id = $1 and published_at is null and claimed_by = $3
372
- `.trim();
373
-
374
- export const SQL_OUTBOX_PENDING_COUNT = `
375
- select count(*)::bigint as pending from x_outbox where published_at is null
376
- `.trim();
327
+ export const SQL_EVENT_PURGE = `delete from x_job_events where expires_at <= statement_timestamp()`;
377
328
 
378
329
  export const SQL_STEP_GET = `
379
330
  select run_id, name, status, output, attempts, error,
@@ -0,0 +1,76 @@
1
+ // The cross-process wake, as SQL: the two channels a worker pod LISTENs on, and the fragments that
2
+ // decide whether a statement announces the row it wrote. A NOTIFY is part of its transaction — it
3
+ // is delivered when that commits and never when it rolls back — which is what lets the enqueue
4
+ // and the outbox stage carry one. The payload is the QUEUE NAME and nothing else: never an id, an
5
+ // input or a tenant, because every session listening reads it.
6
+ //
7
+ // Why a statement does not always notify: Postgres serialises the commit of every transaction
8
+ // that issued a NOTIFY behind one lock, held through that commit's WAL flush. A notification per
9
+ // enqueue would cap an app's enqueuing commits at one flush at a time. So each statement asks
10
+ // first whether the loops it would wake are already awake.
11
+
12
+ /** A job became claimable on the queue the payload names. An empty payload means any queue. */
13
+ export const JOBS_WAKE_CHANNEL = 'x_jobs_wake';
14
+ /** A row was committed to `x_outbox`. The payload is always empty: a relay serves every queue. */
15
+ export const OUTBOX_WAKE_CHANNEL = 'x_outbox_wake';
16
+
17
+ /**
18
+ * One notification per queue per slot, whatever the enqueue rate — save the first commits of a
19
+ * slot racing each other, which may each send one (`notifyJobReady`). It equals the worker's
20
+ * default poll floor on purpose: a woken worker polls again one floor later, so a row
21
+ * that stayed silent because its slot had already spoken is found by that pass. Enqueue latency
22
+ * is therefore never worse than the fixed 250 ms poll this replaced.
23
+ */
24
+ export const WAKE_SLOT_MS = 250;
25
+
26
+ /**
27
+ * A row due this soon still notifies. `run_at` is stamped from the ENQUEUER's clock, so a pod
28
+ * running a few hundred ms ahead of the database writes "immediate" jobs that are due a moment
29
+ * from now — and the passes a woken worker makes (0, 250, 750 ms) are what pick those up.
30
+ */
31
+ export const WAKE_DUE_WITHIN_MS = 1_000;
32
+
33
+ const SLOTS_PER_SECOND = 1_000 / WAKE_SLOT_MS;
34
+
35
+ /** A row due this soon is one that notified when it was written — the test both sides share. */
36
+ const dueSoon = (alias: string): string =>
37
+ `${alias}.run_at <= now() + interval '${WAKE_DUE_WITHIN_MS} milliseconds'`;
38
+
39
+ /**
40
+ * `pg_notify` for the row `alias` names, or nothing: the row must be (about to be) due, and no
41
+ * OTHER row of its queue that was itself due soon may have been created in the current slot. Only
42
+ * such a row sent a wake — a far-future one sent none, so it silences nothing. The statement's own
43
+ * insert is invisible to this read — a CTE's writes are not in its statement's snapshot — so the
44
+ * row never silences itself. Read over `x_jobs_created_idx`, newest first, stopping at the slot.
45
+ *
46
+ * A read, not a reservation: two first-of-slot enqueues committing together can both notify. A
47
+ * spare wake costs one commit behind the NOTIFY lock; a slot row to reserve would put EVERY enqueue
48
+ * behind one row lock, which is the serialisation this exists to avoid.
49
+ */
50
+ export const notifyJobReady = (alias: string): string => `
51
+ (select pg_notify('${JOBS_WAKE_CHANNEL}', ${alias}.queue)
52
+ where ${dueSoon(alias)}
53
+ and not exists (
54
+ select 1 from x_jobs o
55
+ where o.queue = ${alias}.queue
56
+ and o.created_at >= to_timestamp(
57
+ floor(extract(epoch from now()) * ${SLOTS_PER_SECOND}) / ${SLOTS_PER_SECOND})
58
+ and ${dueSoon('o')}
59
+ ))`;
60
+
61
+ /**
62
+ * `pg_notify` for a staged row, unless a committed row is already waiting unclaimed: the relay
63
+ * that row woke is about to pass, or is passing at its floor, and finds this one too. A row a
64
+ * relay HOLDS does not count — its holder may be dead, and its lease is not a wake.
65
+ */
66
+ export const NOTIFY_STAGED = `
67
+ (select pg_notify('${OUTBOX_WAKE_CHANNEL}', '')
68
+ where not exists (
69
+ select 1 from x_outbox o where o.published_at is null and o.claimed_at is null
70
+ ))`;
71
+
72
+ /**
73
+ * An unconditional wake: an operator's requeue, promote or resume, and the listener's own probe.
74
+ * $1 channel, $2 payload.
75
+ */
76
+ export const SQL_WAKE = 'select pg_notify($1, $2)';
package/src/driver-pg.ts CHANGED
@@ -8,19 +8,27 @@ import type { Clock } from '@ultimat3/core';
8
8
  import { finiteCount, systemClock, uuid } from '@ultimat3/core';
9
9
  import type { BackfillLedger } from './backfill-ledger';
10
10
  import { nowMs } from './clock';
11
+ import { nackOutcome } from './counters';
11
12
  import type {
13
+ AckOptions,
12
14
  ClaimedJob,
13
15
  ClaimOptions,
14
16
  EnqueueRequest,
15
17
  EnqueueResult,
16
18
  HeartbeatOptions,
17
19
  JobDriver,
18
- JobFilter,
19
- JobIntrospection,
20
+ JobRecord,
20
21
  NackOptions,
21
22
  QueueStats,
22
23
  } from './driver';
23
- import { assertClaimBounds, assertClaimQueues, DEFAULT_QUEUE, REQUEUEABLE_STATES } from './driver';
24
+ import {
25
+ assertClaimBounds,
26
+ assertClaimQueues,
27
+ DEFAULT_QUEUE,
28
+ nackState,
29
+ REQUEUEABLE_STATES,
30
+ } from './driver';
31
+ import { pgOperator } from './driver-pg-operator';
24
32
  import type { BackfillRow, JobRow, StepRow } from './driver-pg-rows';
25
33
  import { num, toBackfillRun, toJobRecord, toStepRecord } from './driver-pg-rows';
26
34
  import {
@@ -37,13 +45,14 @@ import {
37
45
  SQL_HEARTBEAT,
38
46
  SQL_JOB_DEAD_LETTERS,
39
47
  SQL_JOB_GET,
40
- SQL_JOB_LIST,
41
48
  SQL_JOB_LIVE_HOLDER,
42
49
  SQL_JOB_REQUEUE,
43
50
  SQL_LEASE_ACQUIRE,
51
+ SQL_LEASE_HOLDERS,
44
52
  SQL_LEASE_RELEASE,
45
53
  SQL_LEASE_RENEW,
46
54
  SQL_NACK,
55
+ SQL_OUTBOX_PUBLISHED_JOB,
47
56
  SQL_STATS,
48
57
  SQL_STEP_GET,
49
58
  SQL_STEP_LIST,
@@ -53,6 +62,8 @@ import {
53
62
  } from './driver-pg-sql';
54
63
  import { DriverUnavailableError, JobDuplicateError } from './errors';
55
64
  import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
65
+ import type { JobIntrospection } from './introspection';
66
+ import { MAX_ERROR_STACK_LENGTH } from './introspection';
56
67
  import type { HeldLease, LeaseStore } from './leases';
57
68
  import type { StepStore } from './steps';
58
69
 
@@ -183,6 +194,10 @@ function pgLeaseStore(exec: () => PgExecutor): LeaseStore {
183
194
  );
184
195
  return num(rows[0]?.n);
185
196
  },
197
+ async holders(key) {
198
+ const rows = await exec().query<{ holder: string }>(SQL_LEASE_HOLDERS, [key]);
199
+ return rows.map((row) => row.holder);
200
+ },
186
201
  };
187
202
  }
188
203
 
@@ -194,21 +209,15 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
194
209
  return executor;
195
210
  };
196
211
 
212
+ const job = async (jobId: string): Promise<JobRecord | undefined> => {
213
+ const rows = await exec().query<JobRow>(SQL_JOB_GET, [jobId]);
214
+ const row = rows[0];
215
+ return row === undefined ? undefined : toJobRecord(row);
216
+ };
217
+
197
218
  const introspect: JobIntrospection = {
198
- async job(jobId) {
199
- const rows = await exec().query<JobRow>(SQL_JOB_GET, [jobId]);
200
- const row = rows[0];
201
- return row === undefined ? undefined : toJobRecord(row);
202
- },
203
- async list(filter: JobFilter = {}) {
204
- const rows = await exec().query<JobRow>(SQL_JOB_LIST, [
205
- filter.queue ?? null,
206
- filter.name ?? null,
207
- filter.state ?? null,
208
- finiteCount('the pg driver list', 'limit', filter.limit ?? 100),
209
- ]);
210
- return rows.map(toJobRecord);
211
- },
219
+ job,
220
+ ...pgOperator(exec, job),
212
221
  async deadLetters(limit = 100) {
213
222
  const rows = await exec().query<JobRow>(SQL_JOB_DEAD_LETTERS, [
214
223
  finiteCount('the pg driver dead letters', 'limit', limit),
@@ -216,7 +225,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
216
225
  return rows.map(toJobRecord);
217
226
  },
218
227
  async requeue(jobId, requeueOptions) {
219
- const current = await this.job(jobId);
228
+ const current = await job(jobId);
220
229
  if (current === undefined) {
221
230
  throw new DriverUnavailableError({
222
231
  driver: 'pg',
@@ -262,7 +271,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
262
271
  async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
263
272
  const runAt = request.runAt ?? nowMs(clock);
264
273
  const rows = await exec().query<{ id: string; run_id: string }>(SQL_ENQUEUE, [
265
- uuid(),
274
+ request.id ?? uuid(),
266
275
  request.name,
267
276
  request.queue || DEFAULT_QUEUE,
268
277
  JSON.stringify(request.input ?? null),
@@ -279,6 +288,17 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
279
288
  return { id: inserted.id, runId: inserted.run_id, deduped: false };
280
289
  }
281
290
 
291
+ // Nothing inserted under an id the CALLER allocated: the outbox publishing one staged row a
292
+ // second time. The job its first publish made is the answer, whatever state it has reached.
293
+ if (request.id !== undefined) {
294
+ const published = await exec().query<{ id: string; run_id: string }>(
295
+ SQL_OUTBOX_PUBLISHED_JOB,
296
+ [request.id],
297
+ );
298
+ const made = published[0];
299
+ if (made !== undefined) return { id: made.id, runId: made.run_id, deduped: true };
300
+ }
301
+
282
302
  // `do nothing` fired: a live job OF THIS NAME, IN THIS TENANT, already owns this idempotency
283
303
  // key. Both are in the lookup because both are in the index — without the name this returned
284
304
  // whichever other job derived the same natural key; without the tenant it returned another
@@ -321,33 +341,48 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
321
341
  const record = toJobRecord(row);
322
342
  return {
323
343
  ...record,
344
+ claimedBy: claimOptions.workerId,
345
+ claim: record.claim ?? 0,
324
346
  claimedAt: at,
325
347
  visibleAt: record.visibleAt ?? at + claimOptions.visibilityTimeoutMs,
326
348
  };
327
349
  });
328
350
  },
329
351
 
330
- async ack(jobId: string): Promise<void> {
331
- await exec().query(SQL_ACK, [jobId]);
352
+ // Both settles answer whether they LANDED: no row back is a settle from a worker that no
353
+ // longer owns the job (`SQL_ACK` says why both fences exist), and the caller logs it.
354
+ async ack(jobId: string, by: AckOptions): Promise<boolean> {
355
+ const rows = await exec().query<{ name: string }>(SQL_ACK, [
356
+ jobId,
357
+ by.workerId,
358
+ Math.round(by.durationMs ?? 0),
359
+ by.claim,
360
+ by.counted !== false,
361
+ ]);
362
+ return rows.length > 0;
332
363
  },
333
364
 
334
- async nack(jobId: string, nackOptions: NackOptions): Promise<void> {
365
+ async nack(jobId: string, nackOptions: NackOptions): Promise<boolean> {
335
366
  const counts = nackOptions.countsAsAttempt !== false;
336
- // The same three-way the memory driver takes, and it reads `park` rather than `counts`: the
337
- // attempt counter and the ready bucket are two facts, and a shed only ever meant the first.
338
- const state =
339
- nackOptions.deadLetter === true
340
- ? 'dead'
341
- : nackOptions.park === true
342
- ? 'suspended'
343
- : 'ready';
344
- await exec().query(SQL_NACK, [
367
+ const outcome = nackOutcome(nackOptions);
368
+ // The reading the memory driver takes — `nackState` is the one definition — and it reads
369
+ // `park` rather than `counts`: the attempt counter and the ready bucket are two facts, and a
370
+ // shed only ever meant the first.
371
+ const rows = await exec().query<{ name: string }>(SQL_NACK, [
345
372
  jobId,
346
- state,
373
+ nackState(nackOptions),
347
374
  counts,
348
375
  nackOptions.delayMs,
349
376
  nackOptions.error ?? null,
377
+ nackOptions.workerId,
378
+ nackOptions.stack?.slice(0, MAX_ERROR_STACK_LENGTH) ?? null,
379
+ outcome === 'retried' ? 1 : 0,
380
+ outcome === 'failed' ? 1 : 0,
381
+ outcome === 'dead' ? 1 : 0,
382
+ Math.round(nackOptions.durationMs ?? 0),
383
+ nackOptions.claim,
350
384
  ]);
385
+ return rows.length > 0;
351
386
  },
352
387
 
353
388
  async heartbeat(jobId: string, heartbeatOptions: HeartbeatOptions): Promise<boolean> {
@@ -355,6 +390,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
355
390
  jobId,
356
391
  heartbeatOptions.visibilityTimeoutMs,
357
392
  heartbeatOptions.workerId ?? null,
393
+ heartbeatOptions.claim ?? null,
358
394
  ]);
359
395
  // No row means the job is no longer ours: cancelled from outside, or re-claimed after this
360
396
  // lease lapsed. Either way the caller has to stop running it.
@@ -368,6 +404,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
368
404
  delayed: number | string;
369
405
  running: number | string;
370
406
  suspended: number | string;
407
+ failed: number | string;
371
408
  dead: number | string;
372
409
  oldest_ready_ms: number | string;
373
410
  }>(SQL_STATS, []);
@@ -377,6 +414,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
377
414
  delayed: num(row.delayed),
378
415
  running: num(row.running),
379
416
  suspended: num(row.suspended),
417
+ failed: num(row.failed),
380
418
  dead: num(row.dead),
381
419
  oldestReadyMs: Math.round(num(row.oldest_ready_ms)),
382
420
  }));
@@ -402,10 +440,12 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
402
440
  export function createPgLeader(
403
441
  lockKey: number,
404
442
  options: PgDriverOptions = {},
405
- ): { acquire(): Promise<boolean>; release(): Promise<void> } {
443
+ ): { acquire(): Promise<boolean>; release(): Promise<void>; readonly renewEveryMs: number } {
406
444
  const exec = (): PgExecutor => resolveExecutor(options.executor);
407
445
  let held = false;
408
446
  return {
447
+ // Asked every time: a held advisory lock answers from the flag below, at no cost.
448
+ renewEveryMs: 0,
409
449
  async acquire() {
410
450
  if (held) return true;
411
451
  const rows = await exec().query<{ locked: boolean }>(SQL_TRY_ADVISORY_LOCK, [lockKey]);
@@ -73,10 +73,10 @@ export function createRedisDriver(_options: RedisDriverOptions = {}): JobDriver
73
73
  claim(_options: ClaimOptions): Promise<readonly ClaimedJob[]> {
74
74
  return unavailable('claim');
75
75
  },
76
- ack(_jobId: string): Promise<void> {
76
+ ack(_jobId: string): Promise<boolean> {
77
77
  return unavailable('ack');
78
78
  },
79
- nack(_jobId: string, _options: NackOptions): Promise<void> {
79
+ nack(_jobId: string, _options: NackOptions): Promise<boolean> {
80
80
  return unavailable('nack');
81
81
  },
82
82
  heartbeat(_jobId: string, _options: HeartbeatOptions): Promise<boolean> {