@ultimat3/jobs 22.14.0 → 23.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +129 -87
- package/README.md +329 -19
- package/package.json +6 -6
- package/src/concurrency.ts +144 -0
- package/src/counters.ts +105 -0
- package/src/describe.ts +29 -0
- package/src/driver-memory-operator.ts +297 -0
- package/src/driver-memory.ts +79 -37
- package/src/driver-nats.ts +2 -2
- package/src/driver-pg-ddl.ts +71 -0
- package/src/driver-pg-jobs-sql.ts +71 -8
- package/src/driver-pg-operator-sql.ts +303 -0
- package/src/driver-pg-operator.ts +300 -0
- package/src/driver-pg-outbox-sql.ts +110 -0
- package/src/driver-pg-rows.ts +31 -0
- package/src/driver-pg-settle-sql.ts +71 -0
- package/src/driver-pg-sql.ts +80 -129
- package/src/driver-pg-wake-sql.ts +76 -0
- package/src/driver-pg.ts +74 -34
- package/src/driver-redis.ts +2 -2
- package/src/driver.ts +96 -34
- package/src/enqueue-signal.ts +50 -0
- package/src/errors-concurrency.ts +91 -0
- package/src/errors-operator.ts +94 -0
- package/src/errors.ts +13 -17
- package/src/events-pg.ts +38 -28
- package/src/events.ts +25 -0
- package/src/execute.ts +160 -81
- package/src/export.ts +23 -4
- package/src/heartbeat.ts +9 -3
- package/src/idle-backoff.ts +74 -0
- package/src/index.ts +105 -12
- package/src/inspect-operator.ts +49 -0
- package/src/inspect.ts +42 -2
- package/src/introspection.ts +262 -0
- package/src/job.ts +78 -15
- package/src/leases.ts +49 -9
- package/src/metrics.ts +6 -1
- package/src/outbox-pg.ts +3 -0
- package/src/outbox-relay.ts +114 -39
- package/src/outbox.ts +29 -6
- package/src/progress.ts +78 -0
- package/src/queue-wake.ts +220 -0
- package/src/redact-input.ts +26 -0
- package/src/renewal-timer.ts +28 -10
- package/src/retry-classification.ts +13 -1
- package/src/retry.ts +10 -2
- package/src/run-deadline.ts +57 -0
- package/src/scheduler-leader.ts +26 -0
- package/src/scheduler-occurrences.ts +76 -0
- package/src/scheduler-pg.ts +55 -2
- package/src/scheduler-state.ts +66 -0
- package/src/scheduler.ts +143 -103
- package/src/settled.ts +161 -0
- package/src/worker-admit.ts +163 -0
- package/src/worker-fleet-slots.ts +107 -38
- package/src/worker-hand-back.ts +6 -1
- package/src/worker-key-busy.ts +79 -0
- package/src/worker-loop.ts +145 -0
- package/src/worker-registry.ts +103 -0
- package/src/worker-run.ts +9 -0
- package/src/worker-types.ts +23 -0
- package/src/worker.ts +103 -134
package/src/driver-pg-sql.ts
CHANGED
|
@@ -22,20 +22,44 @@ export {
|
|
|
22
22
|
SQL_STEPS_FROM,
|
|
23
23
|
} from './driver-pg-jobs-sql';
|
|
24
24
|
|
|
25
|
+
export {
|
|
26
|
+
SQL_OUTBOX_CLAIM,
|
|
27
|
+
SQL_OUTBOX_MARK_PUBLISHED,
|
|
28
|
+
SQL_OUTBOX_PENDING_COUNT,
|
|
29
|
+
SQL_OUTBOX_PUBLISHED_JOB,
|
|
30
|
+
SQL_OUTBOX_RELEASE,
|
|
31
|
+
SQL_OUTBOX_STAGE,
|
|
32
|
+
} from './driver-pg-outbox-sql';
|
|
33
|
+
export { SQL_ACK, SQL_NACK } from './driver-pg-settle-sql';
|
|
34
|
+
|
|
25
35
|
import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
|
|
36
|
+
import { notifyJobReady } from './driver-pg-wake-sql';
|
|
26
37
|
|
|
38
|
+
/**
|
|
39
|
+
* The insert, and its wake: `woke` is the notification's own column, evaluated once per inserted
|
|
40
|
+
* row and absent when the conflict fired. `driver-pg-wake-sql.ts` says when it stays silent.
|
|
41
|
+
*
|
|
42
|
+
* `where not exists` is the outbox's half. A staged row is published under ITS OWN id
|
|
43
|
+
* (`EnqueueRequest.id`), so a publish repeated after a crash meets the job the first one made —
|
|
44
|
+
* in any state — and inserts nothing; the partial index below collapses a repeat only while that
|
|
45
|
+
* job is still live. A select over `values`, so each parameter carries its cast.
|
|
46
|
+
*/
|
|
27
47
|
export const SQL_ENQUEUE = `
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
48
|
+
with inserted as (
|
|
49
|
+
insert into x_jobs
|
|
50
|
+
(id, name, queue, input, idempotency_key, run_id, max_attempts, state, run_at, tenant_id,
|
|
51
|
+
traceparent, enqueued_by)
|
|
52
|
+
select $1::uuid, $2::text, $3::text, $4::jsonb, $5::text, $6::uuid, $7::int,
|
|
53
|
+
case when to_timestamp($8::bigint / 1000.0) > now() then 'delayed' else 'ready' end,
|
|
54
|
+
to_timestamp($8::bigint / 1000.0), $9::text, $10::text, $11::text
|
|
55
|
+
where not exists (select 1 from x_jobs published where published.id = $1::uuid)
|
|
56
|
+
on conflict (name, (coalesce(tenant_id, '')), idempotency_key)
|
|
57
|
+
where state in ('ready', 'delayed', 'running', 'suspended')
|
|
58
|
+
do nothing
|
|
59
|
+
returning id, run_id, queue, run_at
|
|
60
|
+
)
|
|
61
|
+
select i.id, i.run_id,${notifyJobReady('i')}::text as woke
|
|
62
|
+
from inserted i
|
|
39
63
|
`.trim();
|
|
40
64
|
|
|
41
65
|
/**
|
|
@@ -68,6 +92,9 @@ with claimed as (
|
|
|
68
92
|
from x_jobs
|
|
69
93
|
where queue = any($1::text[])
|
|
70
94
|
and run_at <= now()
|
|
95
|
+
and not exists (
|
|
96
|
+
select 1 from x_job_pauses p where p.kind = 'queue' and p.name = x_jobs.queue
|
|
97
|
+
)
|
|
71
98
|
and (
|
|
72
99
|
state in ('ready', 'delayed', 'suspended')
|
|
73
100
|
or (state = 'running' and visible_at <= now())
|
|
@@ -79,13 +106,14 @@ with claimed as (
|
|
|
79
106
|
update x_jobs j
|
|
80
107
|
set state = 'running',
|
|
81
108
|
attempt = j.attempt + 1,
|
|
109
|
+
claims = j.claims + 1,
|
|
82
110
|
claimed_by = $3,
|
|
83
111
|
visible_at = now() + ($4::bigint * interval '1 millisecond'),
|
|
84
112
|
updated_at = now()
|
|
85
113
|
from claimed c
|
|
86
114
|
where j.id = c.id
|
|
87
115
|
returning j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt,
|
|
88
|
-
j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by,
|
|
116
|
+
j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by, j.claims,
|
|
89
117
|
j.traceparent, j.enqueued_by,
|
|
90
118
|
(extract(epoch from j.run_at) * 1000)::bigint as run_at,
|
|
91
119
|
(extract(epoch from j.visible_at) * 1000)::bigint as visible_at,
|
|
@@ -93,30 +121,6 @@ returning j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt
|
|
|
93
121
|
(extract(epoch from j.updated_at) * 1000)::bigint as updated_at
|
|
94
122
|
`.trim();
|
|
95
123
|
|
|
96
|
-
/**
|
|
97
|
-
* `and state = 'running'` is a FENCE, not a filter. Without it an ack from the worker that was
|
|
98
|
-
* cancelled — or from one whose lease lapsed and whose job another worker already re-claimed —
|
|
99
|
-
* overwrites the row it no longer owns: `x jobs cancel` would be undone by the next settle, and
|
|
100
|
-
* a re-delivered job would be marked done by the attempt that lost it.
|
|
101
|
-
*/
|
|
102
|
-
export const SQL_ACK = `
|
|
103
|
-
update x_jobs
|
|
104
|
-
set state = 'done', visible_at = null, claimed_by = null, updated_at = now()
|
|
105
|
-
where id = $1 and state = 'running'
|
|
106
|
-
`.trim();
|
|
107
|
-
|
|
108
|
-
export const SQL_NACK = `
|
|
109
|
-
update x_jobs
|
|
110
|
-
set state = $2,
|
|
111
|
-
attempt = case when $3::boolean then attempt else greatest(attempt - 1, 0) end,
|
|
112
|
-
run_at = now() + ($4::bigint * interval '1 millisecond'),
|
|
113
|
-
visible_at = null,
|
|
114
|
-
claimed_by = null,
|
|
115
|
-
last_error = coalesce($5, last_error),
|
|
116
|
-
updated_at = now()
|
|
117
|
-
where id = $1 and state = 'running'
|
|
118
|
-
`.trim();
|
|
119
|
-
|
|
120
124
|
/**
|
|
121
125
|
* Stop a job from outside. `state <> 'done'` and not `state = 'ready'`: the runaway backfill this
|
|
122
126
|
* exists for is `running`, and a job that already finished has nothing to stop. The worker holding
|
|
@@ -140,11 +144,12 @@ export const SQL_HEARTBEAT = `
|
|
|
140
144
|
update x_jobs
|
|
141
145
|
set visible_at = now() + ($2::bigint * interval '1 millisecond'), updated_at = now()
|
|
142
146
|
where id = $1 and state = 'running' and ($3::text is null or claimed_by = $3)
|
|
147
|
+
and ($4::int is null or claims = $4)
|
|
143
148
|
returning id
|
|
144
149
|
`.trim();
|
|
145
150
|
|
|
146
151
|
/**
|
|
147
|
-
*
|
|
152
|
+
* Six buckets a queue depth is SUMMED from, so a row may land in exactly one. `run_at > now()`
|
|
148
153
|
* unqualified was every state's future row: a `step.sleep` job counted as `suspended` AND
|
|
149
154
|
* `delayed`, a delayed dead-letter as `dead` AND `delayed`. The memory driver's `if/else if` chain
|
|
150
155
|
* has always been exclusive, so `x jobs` reported one depth under `x dev` and a larger one in
|
|
@@ -156,6 +161,7 @@ select queue,
|
|
|
156
161
|
count(*) filter (where state in ('ready', 'delayed') and run_at > now()) as delayed,
|
|
157
162
|
count(*) filter (where state = 'running') as running,
|
|
158
163
|
count(*) filter (where state = 'suspended') as suspended,
|
|
164
|
+
count(*) filter (where state = 'failed') as failed,
|
|
159
165
|
count(*) filter (where state = 'dead') as dead,
|
|
160
166
|
coalesce(max(extract(epoch from now() - run_at)) filter
|
|
161
167
|
(where state in ('ready', 'delayed') and run_at <= now()), 0) * 1000 as oldest_ready_ms
|
|
@@ -254,9 +260,43 @@ export const SQL_LEASE_RELEASE = `
|
|
|
254
260
|
delete from x_job_leases where lease_key = $1 and slot = $2::int and holder = $3
|
|
255
261
|
`.trim();
|
|
256
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Who holds the LIVE slots of one key, in slot order. Read only after an acquire was refused, to
|
|
265
|
+
* tell a key another run holds from one this run's own previous claim still does — the acquire
|
|
266
|
+
* itself stays the one atomic decision, and this never grants anything.
|
|
267
|
+
*/
|
|
268
|
+
export const SQL_LEASE_HOLDERS = `
|
|
269
|
+
select holder from x_job_leases
|
|
270
|
+
where lease_key = $1 and expires_at > now()
|
|
271
|
+
order by slot
|
|
272
|
+
`.trim();
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Both instants are the DATABASE's: `published_at` is `statement_timestamp()` and the expiry is
|
|
276
|
+
* counted from it.
|
|
277
|
+
* They were bound from the publisher's process clock, and every reader compares them with
|
|
278
|
+
* something else — `now()` in `SQL_EVENT_FIND`, a consumer's "after" — so a pod whose clock was
|
|
279
|
+
* off wrote an event that had expired before it was inserted, or that came "before" a question
|
|
280
|
+
* asked a minute earlier. `floor`, never a rounding cast: a stamp is never later than the row.
|
|
281
|
+
*
|
|
282
|
+
* The STATEMENT's time, never `now()`, in every event statement: `now()` is when the transaction
|
|
283
|
+
* began, so a bus handed a long transaction's connection stamped an event at its start — expired
|
|
284
|
+
* on commit, or "before" a question asked while the transaction was open
|
|
285
|
+
* (`events-pg.live.test.ts`). Outside a transaction the two are the same instant.
|
|
286
|
+
*
|
|
287
|
+
* $1 id, $2 name, $3 payload, $4 correlation key, $5 ttl (ms).
|
|
288
|
+
*/
|
|
257
289
|
export const SQL_EVENT_PUBLISH = `
|
|
258
290
|
insert into x_job_events (id, name, payload, correlation_key, published_at, expires_at)
|
|
259
|
-
values ($1, $2, $3::jsonb, $4,
|
|
291
|
+
values ($1, $2, $3::jsonb, $4, statement_timestamp(),
|
|
292
|
+
statement_timestamp() + ($5::bigint * interval '1 millisecond'))
|
|
293
|
+
returning floor(extract(epoch from published_at) * 1000)::bigint as published_at,
|
|
294
|
+
floor(extract(epoch from expires_at) * 1000)::bigint as expires_at
|
|
295
|
+
`.trim();
|
|
296
|
+
|
|
297
|
+
/** The bus's clock — the instant a publish issued now would be stamped with. */
|
|
298
|
+
export const SQL_EVENT_NOW = `
|
|
299
|
+
select floor(extract(epoch from statement_timestamp()) * 1000)::bigint as now
|
|
260
300
|
`.trim();
|
|
261
301
|
|
|
262
302
|
/**
|
|
@@ -267,7 +307,7 @@ export const SQL_EVENT_FIND = `
|
|
|
267
307
|
select payload, (extract(epoch from published_at) * 1000)::bigint as published_at
|
|
268
308
|
from x_job_events
|
|
269
309
|
where name = $1
|
|
270
|
-
and expires_at >
|
|
310
|
+
and expires_at > statement_timestamp()
|
|
271
311
|
and published_at >= to_timestamp($3 / 1000.0)
|
|
272
312
|
and ($2::text is null or correlation_key = $2)
|
|
273
313
|
order by published_at
|
|
@@ -284,96 +324,7 @@ select id, name, payload, correlation_key,
|
|
|
284
324
|
limit $2
|
|
285
325
|
`.trim();
|
|
286
326
|
|
|
287
|
-
export const SQL_EVENT_PURGE = `delete from x_job_events where expires_at <=
|
|
288
|
-
|
|
289
|
-
export const SQL_OUTBOX_STAGE = `
|
|
290
|
-
insert into x_outbox
|
|
291
|
-
(id, job, queue, input, idempotency_key, max_attempts, run_at, staged_at, tenant_id,
|
|
292
|
-
traceparent, enqueued_by)
|
|
293
|
-
values ($1, $2, $3, $4::jsonb, $5, $6, to_timestamp($7 / 1000.0), to_timestamp($8 / 1000.0),
|
|
294
|
-
$9, $10, $11)
|
|
295
|
-
`.trim();
|
|
296
|
-
|
|
297
|
-
/**
|
|
298
|
-
* The claim, and it has to be ONE statement. `for update skip locked` in a bare select holds its
|
|
299
|
-
* row locks only until that statement ends — under autocommit, before `claim()` even resolves — so
|
|
300
|
-
* two relays polling 200ms apart read the same unpublished rows and both hand them to `enqueue`.
|
|
301
|
-
* `SQL_ENQUEUE` collapses the repeat only while the first job is still LIVE: its conflict target
|
|
302
|
-
* is a partial index over the live states, so a second publish landing after that job reached a
|
|
303
|
-
* terminal state inserts a second row and the handler runs again. (The mechanism is Postgres
|
|
304
|
-
* semantics; how often the two orderings line up in a deployment was never measured.)
|
|
305
|
-
*
|
|
306
|
-
* So the lock and the claim commit together, the CTE shape `SQL_CLAIM` already uses, and
|
|
307
|
-
* `claimed_at` is a LEASE: `$2` is the window after which a row a dead relay was holding is
|
|
308
|
-
* claimable again, because a claim nothing can expire strands its rows forever.
|
|
309
|
-
*
|
|
310
|
-
* The outer `select ... order by staged_at, id` is not cosmetic. `update ... returning` has no
|
|
311
|
-
* defined row order and the relay publishes in the order it is handed rows, so an app staging
|
|
312
|
-
* `createInvoice` then `chargeCard` in one transaction depends on this line.
|
|
313
|
-
*
|
|
314
|
-
* `, id` is what makes that key TOTAL, and the CTE needs it as much as the projection does: every
|
|
315
|
-
* row staged in one transaction shares a `staged_at`, so `staged_at` alone leaves the tie to the
|
|
316
|
-
* planner — which rows a `limit` takes, and in which order they publish, then differ between two
|
|
317
|
-
* relays and between two runs of one relay. No column was added for it: `id` is a UUIDv7 minted by
|
|
318
|
-
* `uuid()`, monotonic and already the primary key, so the tiebreak IS stage order.
|
|
319
|
-
*/
|
|
320
|
-
export const SQL_OUTBOX_CLAIM = `
|
|
321
|
-
with claimable as (
|
|
322
|
-
select id
|
|
323
|
-
from x_outbox
|
|
324
|
-
where published_at is null
|
|
325
|
-
and (claimed_at is null
|
|
326
|
-
or claimed_at <= now() - ($2::bigint * interval '1 millisecond'))
|
|
327
|
-
order by staged_at, id
|
|
328
|
-
limit $1
|
|
329
|
-
for update skip locked
|
|
330
|
-
), claimed as (
|
|
331
|
-
update x_outbox o
|
|
332
|
-
set claimed_at = now(), claimed_by = $3
|
|
333
|
-
from claimable c
|
|
334
|
-
where o.id = c.id
|
|
335
|
-
returning o.id, o.job, o.queue, o.input, o.idempotency_key, o.max_attempts, o.tenant_id,
|
|
336
|
-
o.traceparent, o.enqueued_by, o.claimed_by, o.run_at, o.staged_at
|
|
337
|
-
)
|
|
338
|
-
select id, job, queue, input, idempotency_key, max_attempts, tenant_id,
|
|
339
|
-
traceparent, enqueued_by, claimed_by,
|
|
340
|
-
(extract(epoch from run_at) * 1000)::bigint as run_at,
|
|
341
|
-
(extract(epoch from staged_at) * 1000)::bigint as staged_at
|
|
342
|
-
from claimed
|
|
343
|
-
order by staged_at, id
|
|
344
|
-
`.trim();
|
|
345
|
-
|
|
346
|
-
/**
|
|
347
|
-
* Hand a claim back early. The relay stops its batch on the first publish that fails, and without
|
|
348
|
-
* this the rows behind it would wait out the whole lease before any relay could retry them — a
|
|
349
|
-
* pool blip during a failover becoming tens of seconds of unpublished, committed work.
|
|
350
|
-
*
|
|
351
|
-
* Fenced on `published_at is null` so it can never unclaim a row some other pass already
|
|
352
|
-
* published, AND on `claimed_by` so it can never unclaim one a NEWER claimant now holds: a relay
|
|
353
|
-
* whose lease lapsed while it stalled wakes into a world where its batch is another relay's, and
|
|
354
|
-
* an unfenced release frees rows that relay is mid-publish on — a third relay claims them and
|
|
355
|
-
* publishes them again, which is the duplicate the lease exists to prevent.
|
|
356
|
-
*/
|
|
357
|
-
export const SQL_OUTBOX_RELEASE = `
|
|
358
|
-
update x_outbox
|
|
359
|
-
set claimed_at = null, claimed_by = null
|
|
360
|
-
where id = any($1::uuid[]) and published_at is null and claimed_by = $2
|
|
361
|
-
`.trim();
|
|
362
|
-
|
|
363
|
-
/**
|
|
364
|
-
* Same fence, and here it is the more expensive one to miss: marking a row published is LOSING it,
|
|
365
|
-
* so a lapsed claimant stamping a row the current one has not published yet drops that job with
|
|
366
|
-
* nothing to notice. `published_at is null` makes the stamp first-writer-wins rather than a
|
|
367
|
-
* rewrite of an audit timestamp.
|
|
368
|
-
*/
|
|
369
|
-
export const SQL_OUTBOX_MARK_PUBLISHED = `
|
|
370
|
-
update x_outbox set published_at = to_timestamp($2 / 1000.0)
|
|
371
|
-
where id = $1 and published_at is null and claimed_by = $3
|
|
372
|
-
`.trim();
|
|
373
|
-
|
|
374
|
-
export const SQL_OUTBOX_PENDING_COUNT = `
|
|
375
|
-
select count(*)::bigint as pending from x_outbox where published_at is null
|
|
376
|
-
`.trim();
|
|
327
|
+
export const SQL_EVENT_PURGE = `delete from x_job_events where expires_at <= statement_timestamp()`;
|
|
377
328
|
|
|
378
329
|
export const SQL_STEP_GET = `
|
|
379
330
|
select run_id, name, status, output, attempts, error,
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// The cross-process wake, as SQL: the two channels a worker pod LISTENs on, and the fragments that
|
|
2
|
+
// decide whether a statement announces the row it wrote. A NOTIFY is part of its transaction — it
|
|
3
|
+
// is delivered when that commits and never when it rolls back — which is what lets the enqueue
|
|
4
|
+
// and the outbox stage carry one. The payload is the QUEUE NAME and nothing else: never an id, an
|
|
5
|
+
// input or a tenant, because every session listening reads it.
|
|
6
|
+
//
|
|
7
|
+
// Why a statement does not always notify: Postgres serialises the commit of every transaction
|
|
8
|
+
// that issued a NOTIFY behind one lock, held through that commit's WAL flush. A notification per
|
|
9
|
+
// enqueue would cap an app's enqueuing commits at one flush at a time. So each statement asks
|
|
10
|
+
// first whether the loops it would wake are already awake.
|
|
11
|
+
|
|
12
|
+
/** A job became claimable on the queue the payload names. An empty payload means any queue. */
|
|
13
|
+
export const JOBS_WAKE_CHANNEL = 'x_jobs_wake';
|
|
14
|
+
/** A row was committed to `x_outbox`. The payload is always empty: a relay serves every queue. */
|
|
15
|
+
export const OUTBOX_WAKE_CHANNEL = 'x_outbox_wake';
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* One notification per queue per slot, whatever the enqueue rate — save the first commits of a
|
|
19
|
+
* slot racing each other, which may each send one (`notifyJobReady`). It equals the worker's
|
|
20
|
+
* default poll floor on purpose: a woken worker polls again one floor later, so a row
|
|
21
|
+
* that stayed silent because its slot had already spoken is found by that pass. Enqueue latency
|
|
22
|
+
* is therefore never worse than the fixed 250 ms poll this replaced.
|
|
23
|
+
*/
|
|
24
|
+
export const WAKE_SLOT_MS = 250;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A row due this soon still notifies. `run_at` is stamped from the ENQUEUER's clock, so a pod
|
|
28
|
+
* running a few hundred ms ahead of the database writes "immediate" jobs that are due a moment
|
|
29
|
+
* from now — and the passes a woken worker makes (0, 250, 750 ms) are what pick those up.
|
|
30
|
+
*/
|
|
31
|
+
export const WAKE_DUE_WITHIN_MS = 1_000;
|
|
32
|
+
|
|
33
|
+
const SLOTS_PER_SECOND = 1_000 / WAKE_SLOT_MS;
|
|
34
|
+
|
|
35
|
+
/** A row due this soon is one that notified when it was written — the test both sides share. */
|
|
36
|
+
const dueSoon = (alias: string): string =>
|
|
37
|
+
`${alias}.run_at <= now() + interval '${WAKE_DUE_WITHIN_MS} milliseconds'`;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* `pg_notify` for the row `alias` names, or nothing: the row must be (about to be) due, and no
|
|
41
|
+
* OTHER row of its queue that was itself due soon may have been created in the current slot. Only
|
|
42
|
+
* such a row sent a wake — a far-future one sent none, so it silences nothing. The statement's own
|
|
43
|
+
* insert is invisible to this read — a CTE's writes are not in its statement's snapshot — so the
|
|
44
|
+
* row never silences itself. Read over `x_jobs_created_idx`, newest first, stopping at the slot.
|
|
45
|
+
*
|
|
46
|
+
* A read, not a reservation: two first-of-slot enqueues committing together can both notify. A
|
|
47
|
+
* spare wake costs one commit behind the NOTIFY lock; a slot row to reserve would put EVERY enqueue
|
|
48
|
+
* behind one row lock, which is the serialisation this exists to avoid.
|
|
49
|
+
*/
|
|
50
|
+
export const notifyJobReady = (alias: string): string => `
|
|
51
|
+
(select pg_notify('${JOBS_WAKE_CHANNEL}', ${alias}.queue)
|
|
52
|
+
where ${dueSoon(alias)}
|
|
53
|
+
and not exists (
|
|
54
|
+
select 1 from x_jobs o
|
|
55
|
+
where o.queue = ${alias}.queue
|
|
56
|
+
and o.created_at >= to_timestamp(
|
|
57
|
+
floor(extract(epoch from now()) * ${SLOTS_PER_SECOND}) / ${SLOTS_PER_SECOND})
|
|
58
|
+
and ${dueSoon('o')}
|
|
59
|
+
))`;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* `pg_notify` for a staged row, unless a committed row is already waiting unclaimed: the relay
|
|
63
|
+
* that row woke is about to pass, or is passing at its floor, and finds this one too. A row a
|
|
64
|
+
* relay HOLDS does not count — its holder may be dead, and its lease is not a wake.
|
|
65
|
+
*/
|
|
66
|
+
export const NOTIFY_STAGED = `
|
|
67
|
+
(select pg_notify('${OUTBOX_WAKE_CHANNEL}', '')
|
|
68
|
+
where not exists (
|
|
69
|
+
select 1 from x_outbox o where o.published_at is null and o.claimed_at is null
|
|
70
|
+
))`;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* An unconditional wake: an operator's requeue, promote or resume, and the listener's own probe.
|
|
74
|
+
* $1 channel, $2 payload.
|
|
75
|
+
*/
|
|
76
|
+
export const SQL_WAKE = 'select pg_notify($1, $2)';
|
package/src/driver-pg.ts
CHANGED
|
@@ -8,19 +8,27 @@ import type { Clock } from '@ultimat3/core';
|
|
|
8
8
|
import { finiteCount, systemClock, uuid } from '@ultimat3/core';
|
|
9
9
|
import type { BackfillLedger } from './backfill-ledger';
|
|
10
10
|
import { nowMs } from './clock';
|
|
11
|
+
import { nackOutcome } from './counters';
|
|
11
12
|
import type {
|
|
13
|
+
AckOptions,
|
|
12
14
|
ClaimedJob,
|
|
13
15
|
ClaimOptions,
|
|
14
16
|
EnqueueRequest,
|
|
15
17
|
EnqueueResult,
|
|
16
18
|
HeartbeatOptions,
|
|
17
19
|
JobDriver,
|
|
18
|
-
|
|
19
|
-
JobIntrospection,
|
|
20
|
+
JobRecord,
|
|
20
21
|
NackOptions,
|
|
21
22
|
QueueStats,
|
|
22
23
|
} from './driver';
|
|
23
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
assertClaimBounds,
|
|
26
|
+
assertClaimQueues,
|
|
27
|
+
DEFAULT_QUEUE,
|
|
28
|
+
nackState,
|
|
29
|
+
REQUEUEABLE_STATES,
|
|
30
|
+
} from './driver';
|
|
31
|
+
import { pgOperator } from './driver-pg-operator';
|
|
24
32
|
import type { BackfillRow, JobRow, StepRow } from './driver-pg-rows';
|
|
25
33
|
import { num, toBackfillRun, toJobRecord, toStepRecord } from './driver-pg-rows';
|
|
26
34
|
import {
|
|
@@ -37,13 +45,14 @@ import {
|
|
|
37
45
|
SQL_HEARTBEAT,
|
|
38
46
|
SQL_JOB_DEAD_LETTERS,
|
|
39
47
|
SQL_JOB_GET,
|
|
40
|
-
SQL_JOB_LIST,
|
|
41
48
|
SQL_JOB_LIVE_HOLDER,
|
|
42
49
|
SQL_JOB_REQUEUE,
|
|
43
50
|
SQL_LEASE_ACQUIRE,
|
|
51
|
+
SQL_LEASE_HOLDERS,
|
|
44
52
|
SQL_LEASE_RELEASE,
|
|
45
53
|
SQL_LEASE_RENEW,
|
|
46
54
|
SQL_NACK,
|
|
55
|
+
SQL_OUTBOX_PUBLISHED_JOB,
|
|
47
56
|
SQL_STATS,
|
|
48
57
|
SQL_STEP_GET,
|
|
49
58
|
SQL_STEP_LIST,
|
|
@@ -53,6 +62,8 @@ import {
|
|
|
53
62
|
} from './driver-pg-sql';
|
|
54
63
|
import { DriverUnavailableError, JobDuplicateError } from './errors';
|
|
55
64
|
import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
|
|
65
|
+
import type { JobIntrospection } from './introspection';
|
|
66
|
+
import { MAX_ERROR_STACK_LENGTH } from './introspection';
|
|
56
67
|
import type { HeldLease, LeaseStore } from './leases';
|
|
57
68
|
import type { StepStore } from './steps';
|
|
58
69
|
|
|
@@ -183,6 +194,10 @@ function pgLeaseStore(exec: () => PgExecutor): LeaseStore {
|
|
|
183
194
|
);
|
|
184
195
|
return num(rows[0]?.n);
|
|
185
196
|
},
|
|
197
|
+
async holders(key) {
|
|
198
|
+
const rows = await exec().query<{ holder: string }>(SQL_LEASE_HOLDERS, [key]);
|
|
199
|
+
return rows.map((row) => row.holder);
|
|
200
|
+
},
|
|
186
201
|
};
|
|
187
202
|
}
|
|
188
203
|
|
|
@@ -194,21 +209,15 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
194
209
|
return executor;
|
|
195
210
|
};
|
|
196
211
|
|
|
212
|
+
const job = async (jobId: string): Promise<JobRecord | undefined> => {
|
|
213
|
+
const rows = await exec().query<JobRow>(SQL_JOB_GET, [jobId]);
|
|
214
|
+
const row = rows[0];
|
|
215
|
+
return row === undefined ? undefined : toJobRecord(row);
|
|
216
|
+
};
|
|
217
|
+
|
|
197
218
|
const introspect: JobIntrospection = {
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
const row = rows[0];
|
|
201
|
-
return row === undefined ? undefined : toJobRecord(row);
|
|
202
|
-
},
|
|
203
|
-
async list(filter: JobFilter = {}) {
|
|
204
|
-
const rows = await exec().query<JobRow>(SQL_JOB_LIST, [
|
|
205
|
-
filter.queue ?? null,
|
|
206
|
-
filter.name ?? null,
|
|
207
|
-
filter.state ?? null,
|
|
208
|
-
finiteCount('the pg driver list', 'limit', filter.limit ?? 100),
|
|
209
|
-
]);
|
|
210
|
-
return rows.map(toJobRecord);
|
|
211
|
-
},
|
|
219
|
+
job,
|
|
220
|
+
...pgOperator(exec, job),
|
|
212
221
|
async deadLetters(limit = 100) {
|
|
213
222
|
const rows = await exec().query<JobRow>(SQL_JOB_DEAD_LETTERS, [
|
|
214
223
|
finiteCount('the pg driver dead letters', 'limit', limit),
|
|
@@ -216,7 +225,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
216
225
|
return rows.map(toJobRecord);
|
|
217
226
|
},
|
|
218
227
|
async requeue(jobId, requeueOptions) {
|
|
219
|
-
const current = await
|
|
228
|
+
const current = await job(jobId);
|
|
220
229
|
if (current === undefined) {
|
|
221
230
|
throw new DriverUnavailableError({
|
|
222
231
|
driver: 'pg',
|
|
@@ -262,7 +271,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
262
271
|
async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
|
|
263
272
|
const runAt = request.runAt ?? nowMs(clock);
|
|
264
273
|
const rows = await exec().query<{ id: string; run_id: string }>(SQL_ENQUEUE, [
|
|
265
|
-
uuid(),
|
|
274
|
+
request.id ?? uuid(),
|
|
266
275
|
request.name,
|
|
267
276
|
request.queue || DEFAULT_QUEUE,
|
|
268
277
|
JSON.stringify(request.input ?? null),
|
|
@@ -279,6 +288,17 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
279
288
|
return { id: inserted.id, runId: inserted.run_id, deduped: false };
|
|
280
289
|
}
|
|
281
290
|
|
|
291
|
+
// Nothing inserted under an id the CALLER allocated: the outbox publishing one staged row a
|
|
292
|
+
// second time. The job its first publish made is the answer, whatever state it has reached.
|
|
293
|
+
if (request.id !== undefined) {
|
|
294
|
+
const published = await exec().query<{ id: string; run_id: string }>(
|
|
295
|
+
SQL_OUTBOX_PUBLISHED_JOB,
|
|
296
|
+
[request.id],
|
|
297
|
+
);
|
|
298
|
+
const made = published[0];
|
|
299
|
+
if (made !== undefined) return { id: made.id, runId: made.run_id, deduped: true };
|
|
300
|
+
}
|
|
301
|
+
|
|
282
302
|
// `do nothing` fired: a live job OF THIS NAME, IN THIS TENANT, already owns this idempotency
|
|
283
303
|
// key. Both are in the lookup because both are in the index — without the name this returned
|
|
284
304
|
// whichever other job derived the same natural key; without the tenant it returned another
|
|
@@ -321,33 +341,48 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
321
341
|
const record = toJobRecord(row);
|
|
322
342
|
return {
|
|
323
343
|
...record,
|
|
344
|
+
claimedBy: claimOptions.workerId,
|
|
345
|
+
claim: record.claim ?? 0,
|
|
324
346
|
claimedAt: at,
|
|
325
347
|
visibleAt: record.visibleAt ?? at + claimOptions.visibilityTimeoutMs,
|
|
326
348
|
};
|
|
327
349
|
});
|
|
328
350
|
},
|
|
329
351
|
|
|
330
|
-
|
|
331
|
-
|
|
352
|
+
// Both settles answer whether they LANDED: no row back is a settle from a worker that no
|
|
353
|
+
// longer owns the job (`SQL_ACK` says why both fences exist), and the caller logs it.
|
|
354
|
+
async ack(jobId: string, by: AckOptions): Promise<boolean> {
|
|
355
|
+
const rows = await exec().query<{ name: string }>(SQL_ACK, [
|
|
356
|
+
jobId,
|
|
357
|
+
by.workerId,
|
|
358
|
+
Math.round(by.durationMs ?? 0),
|
|
359
|
+
by.claim,
|
|
360
|
+
by.counted !== false,
|
|
361
|
+
]);
|
|
362
|
+
return rows.length > 0;
|
|
332
363
|
},
|
|
333
364
|
|
|
334
|
-
async nack(jobId: string, nackOptions: NackOptions): Promise<
|
|
365
|
+
async nack(jobId: string, nackOptions: NackOptions): Promise<boolean> {
|
|
335
366
|
const counts = nackOptions.countsAsAttempt !== false;
|
|
336
|
-
|
|
337
|
-
//
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
: nackOptions.park === true
|
|
342
|
-
? 'suspended'
|
|
343
|
-
: 'ready';
|
|
344
|
-
await exec().query(SQL_NACK, [
|
|
367
|
+
const outcome = nackOutcome(nackOptions);
|
|
368
|
+
// The reading the memory driver takes — `nackState` is the one definition — and it reads
|
|
369
|
+
// `park` rather than `counts`: the attempt counter and the ready bucket are two facts, and a
|
|
370
|
+
// shed only ever meant the first.
|
|
371
|
+
const rows = await exec().query<{ name: string }>(SQL_NACK, [
|
|
345
372
|
jobId,
|
|
346
|
-
|
|
373
|
+
nackState(nackOptions),
|
|
347
374
|
counts,
|
|
348
375
|
nackOptions.delayMs,
|
|
349
376
|
nackOptions.error ?? null,
|
|
377
|
+
nackOptions.workerId,
|
|
378
|
+
nackOptions.stack?.slice(0, MAX_ERROR_STACK_LENGTH) ?? null,
|
|
379
|
+
outcome === 'retried' ? 1 : 0,
|
|
380
|
+
outcome === 'failed' ? 1 : 0,
|
|
381
|
+
outcome === 'dead' ? 1 : 0,
|
|
382
|
+
Math.round(nackOptions.durationMs ?? 0),
|
|
383
|
+
nackOptions.claim,
|
|
350
384
|
]);
|
|
385
|
+
return rows.length > 0;
|
|
351
386
|
},
|
|
352
387
|
|
|
353
388
|
async heartbeat(jobId: string, heartbeatOptions: HeartbeatOptions): Promise<boolean> {
|
|
@@ -355,6 +390,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
355
390
|
jobId,
|
|
356
391
|
heartbeatOptions.visibilityTimeoutMs,
|
|
357
392
|
heartbeatOptions.workerId ?? null,
|
|
393
|
+
heartbeatOptions.claim ?? null,
|
|
358
394
|
]);
|
|
359
395
|
// No row means the job is no longer ours: cancelled from outside, or re-claimed after this
|
|
360
396
|
// lease lapsed. Either way the caller has to stop running it.
|
|
@@ -368,6 +404,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
368
404
|
delayed: number | string;
|
|
369
405
|
running: number | string;
|
|
370
406
|
suspended: number | string;
|
|
407
|
+
failed: number | string;
|
|
371
408
|
dead: number | string;
|
|
372
409
|
oldest_ready_ms: number | string;
|
|
373
410
|
}>(SQL_STATS, []);
|
|
@@ -377,6 +414,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
377
414
|
delayed: num(row.delayed),
|
|
378
415
|
running: num(row.running),
|
|
379
416
|
suspended: num(row.suspended),
|
|
417
|
+
failed: num(row.failed),
|
|
380
418
|
dead: num(row.dead),
|
|
381
419
|
oldestReadyMs: Math.round(num(row.oldest_ready_ms)),
|
|
382
420
|
}));
|
|
@@ -402,10 +440,12 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
|
|
|
402
440
|
export function createPgLeader(
|
|
403
441
|
lockKey: number,
|
|
404
442
|
options: PgDriverOptions = {},
|
|
405
|
-
): { acquire(): Promise<boolean>; release(): Promise<void
|
|
443
|
+
): { acquire(): Promise<boolean>; release(): Promise<void>; readonly renewEveryMs: number } {
|
|
406
444
|
const exec = (): PgExecutor => resolveExecutor(options.executor);
|
|
407
445
|
let held = false;
|
|
408
446
|
return {
|
|
447
|
+
// Asked every time: a held advisory lock answers from the flag below, at no cost.
|
|
448
|
+
renewEveryMs: 0,
|
|
409
449
|
async acquire() {
|
|
410
450
|
if (held) return true;
|
|
411
451
|
const rows = await exec().query<{ locked: boolean }>(SQL_TRY_ADVISORY_LOCK, [lockKey]);
|
package/src/driver-redis.ts
CHANGED
|
@@ -73,10 +73,10 @@ export function createRedisDriver(_options: RedisDriverOptions = {}): JobDriver
|
|
|
73
73
|
claim(_options: ClaimOptions): Promise<readonly ClaimedJob[]> {
|
|
74
74
|
return unavailable('claim');
|
|
75
75
|
},
|
|
76
|
-
ack(_jobId: string): Promise<
|
|
76
|
+
ack(_jobId: string): Promise<boolean> {
|
|
77
77
|
return unavailable('ack');
|
|
78
78
|
},
|
|
79
|
-
nack(_jobId: string, _options: NackOptions): Promise<
|
|
79
|
+
nack(_jobId: string, _options: NackOptions): Promise<boolean> {
|
|
80
80
|
return unavailable('nack');
|
|
81
81
|
},
|
|
82
82
|
heartbeat(_jobId: string, _options: HeartbeatOptions): Promise<boolean> {
|