@ultimat3/jobs 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +165 -124
- package/README.md +360 -21
- package/package.json +6 -6
- package/src/backfill-ledger.ts +24 -2
- package/src/backfill-pass.ts +22 -5
- package/src/claim-exhausted.ts +76 -0
- package/src/concurrency.ts +144 -0
- package/src/counters.ts +105 -0
- package/src/describe.ts +29 -0
- package/src/driver-memory-operator.ts +319 -0
- package/src/driver-memory.ts +146 -59
- package/src/driver-nats.ts +7 -4
- package/src/driver-pg-ddl.ts +71 -0
- package/src/driver-pg-jobs-sql.ts +96 -25
- package/src/driver-pg-operator-sql.ts +323 -0
- package/src/driver-pg-operator.ts +316 -0
- package/src/driver-pg-outbox-sql.ts +110 -0
- package/src/driver-pg-rows.ts +34 -3
- package/src/driver-pg-settle-sql.ts +71 -0
- package/src/driver-pg-sql.ts +171 -151
- package/src/driver-pg-wake-sql.ts +76 -0
- package/src/driver-pg.ts +150 -105
- package/src/driver-redis.ts +7 -4
- package/src/driver.ts +133 -35
- package/src/enqueue-signal.ts +50 -0
- package/src/errors-concurrency.ts +91 -0
- package/src/errors-operator.ts +102 -0
- package/src/errors-requeue.ts +21 -3
- package/src/errors.ts +18 -30
- package/src/events-pg.ts +45 -45
- package/src/events.ts +38 -2
- package/src/execute.ts +161 -81
- package/src/export.ts +23 -4
- package/src/heartbeat.ts +9 -3
- package/src/idle-backoff.ts +74 -0
- package/src/index.ts +115 -16
- package/src/inspect-operator.ts +49 -0
- package/src/inspect.ts +46 -5
- package/src/introspection.ts +274 -0
- package/src/job.ts +83 -21
- package/src/leases.ts +49 -9
- package/src/limits.ts +33 -7
- package/src/metrics.ts +6 -1
- package/src/outbox-pg.ts +4 -2
- package/src/outbox-relay.ts +114 -39
- package/src/outbox.ts +59 -8
- package/src/progress.ts +78 -0
- package/src/purge.ts +1 -1
- package/src/queue-wake.ts +219 -0
- package/src/redact-input.ts +26 -0
- package/src/renewal-timer.ts +28 -10
- package/src/retry-classification.ts +13 -1
- package/src/retry.ts +10 -2
- package/src/run-deadline.ts +57 -0
- package/src/scheduler-leader.ts +26 -0
- package/src/scheduler-occurrences.ts +76 -0
- package/src/scheduler-pg.ts +58 -4
- package/src/scheduler-state.ts +73 -0
- package/src/scheduler.ts +150 -111
- package/src/settled.ts +161 -0
- package/src/steps-suspension.ts +46 -0
- package/src/steps-timeout.ts +33 -0
- package/src/steps.ts +55 -81
- package/src/worker-admit.ts +167 -0
- package/src/worker-fleet-slots.ts +157 -54
- package/src/worker-hand-back.ts +6 -1
- package/src/worker-key-busy.ts +107 -0
- package/src/worker-loop.ts +169 -0
- package/src/worker-registry.ts +118 -0
- package/src/worker-run.ts +9 -0
- package/src/worker-types.ts +23 -0
- package/src/worker.ts +142 -130
package/src/driver-memory.ts
CHANGED
|
@@ -3,28 +3,42 @@
|
|
|
3
3
|
// real claim/ack/nack paths rather than a mock that always succeeds.
|
|
4
4
|
|
|
5
5
|
import type { Clock } from '@ultimat3/core';
|
|
6
|
-
import {
|
|
6
|
+
import { finiteCount, systemClock, uuid } from '@ultimat3/core';
|
|
7
7
|
import type { BackfillLedger } from './backfill-ledger';
|
|
8
8
|
import { createMemoryBackfillLedger } from './backfill-ledger';
|
|
9
9
|
import { nowMs } from './clock';
|
|
10
|
+
import { nackOutcome } from './counters';
|
|
10
11
|
import type {
|
|
12
|
+
AckOptions,
|
|
11
13
|
ClaimedJob,
|
|
14
|
+
ClaimIdentity,
|
|
12
15
|
ClaimOptions,
|
|
13
16
|
EnqueueRequest,
|
|
14
17
|
EnqueueResult,
|
|
15
18
|
HeartbeatOptions,
|
|
16
19
|
JobDriver,
|
|
17
|
-
JobFilter,
|
|
18
|
-
JobIntrospection,
|
|
19
20
|
JobRecord,
|
|
20
21
|
NackOptions,
|
|
21
22
|
QueueStats,
|
|
22
23
|
} from './driver';
|
|
23
|
-
import {
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
import {
|
|
25
|
+
assertClaimBounds,
|
|
26
|
+
assertClaimQueues,
|
|
27
|
+
DEFAULT_QUEUE,
|
|
28
|
+
LEASE_LAPSED_FINAL_ATTEMPT,
|
|
29
|
+
LIVE_STATES,
|
|
30
|
+
nackState,
|
|
31
|
+
REQUEUEABLE_STATES,
|
|
32
|
+
} from './driver';
|
|
33
|
+
import { createMemoryOperator } from './driver-memory-operator';
|
|
34
|
+
import { signalEnqueued } from './enqueue-signal';
|
|
35
|
+
import { JobDuplicateError, LeaseLostError } from './errors';
|
|
36
|
+
import { JobNotFoundError, JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
|
|
37
|
+
import type { JobIntrospection } from './introspection';
|
|
38
|
+
import { MAX_ERROR_STACK_LENGTH } from './introspection';
|
|
26
39
|
import type { LeaseStore } from './leases';
|
|
27
40
|
import { createMemoryLeaseStore } from './leases';
|
|
41
|
+
import { isFinalAttempt } from './retry';
|
|
28
42
|
import type { StepStore } from './steps';
|
|
29
43
|
import { createMemoryStepStore } from './steps-memory';
|
|
30
44
|
|
|
@@ -37,8 +51,6 @@ export interface MemoryDriverOptions {
|
|
|
37
51
|
readonly leases?: LeaseStore;
|
|
38
52
|
}
|
|
39
53
|
|
|
40
|
-
const LIVE_STATES = new Set(['ready', 'delayed', 'running', 'suspended']);
|
|
41
|
-
|
|
42
54
|
/**
|
|
43
55
|
* The in-memory driver's own type: `JobDriver` with `close` REQUIRED.
|
|
44
56
|
*
|
|
@@ -52,7 +64,21 @@ export type MemoryJobDriver = JobDriver & { close(): Promise<void> };
|
|
|
52
64
|
|
|
53
65
|
export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJobDriver {
|
|
54
66
|
const clock = options.clock ?? systemClock;
|
|
55
|
-
const
|
|
67
|
+
const stepStore = options.steps ?? createMemoryStepStore();
|
|
68
|
+
// `SQL_STEP_PUT`'s fence: a write made under a claim lands only while that claim holds the row.
|
|
69
|
+
const steps: StepStore = {
|
|
70
|
+
...stepStore,
|
|
71
|
+
put(record, by) {
|
|
72
|
+
const row = by === undefined ? undefined : jobs.get(by.jobId);
|
|
73
|
+
if (
|
|
74
|
+
by !== undefined &&
|
|
75
|
+
!(row?.state === 'running' && row.claimedBy === by.workerId && row.claim === by.claim)
|
|
76
|
+
) {
|
|
77
|
+
return Promise.reject(new LeaseLostError({ job: by.job, jobId: by.jobId }));
|
|
78
|
+
}
|
|
79
|
+
return stepStore.put(record);
|
|
80
|
+
},
|
|
81
|
+
};
|
|
56
82
|
const backfills = options.backfills ?? createMemoryBackfillLedger(clock);
|
|
57
83
|
const leases =
|
|
58
84
|
options.leases ?? createMemoryLeaseStore(options.clock === undefined ? {} : { clock });
|
|
@@ -93,32 +119,47 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
93
119
|
* lease deadline — which `x jobs show` prints, and which is the very pair the claim scan's
|
|
94
120
|
* lease-expiry branch reads to decide a row was abandoned.
|
|
95
121
|
*/
|
|
96
|
-
const settle = (id: string, patch: Partial<JobRecord
|
|
122
|
+
const settle = (id: string, patch: Partial<JobRecord>, drop?: 'stack'): void => {
|
|
97
123
|
const existing = jobs.get(id);
|
|
98
124
|
if (existing === undefined) return;
|
|
99
125
|
const { visibleAt: _visibleAt, claimedBy: _claimedBy, ...released } = existing;
|
|
100
|
-
|
|
126
|
+
// `drop: 'stack'` is `SQL_NACK`'s `case when $5 is null then last_error_stack else $7 end`
|
|
127
|
+
// with no stack bound: a NEW failure replaces the old one's stack even with nothing — the key
|
|
128
|
+
// is removed, never set to `undefined`, as a null column is absent from a pg record.
|
|
129
|
+
const { lastErrorStack: _stack, ...unstacked } = released;
|
|
130
|
+
jobs.set(id, {
|
|
131
|
+
...(drop === 'stack' ? unstacked : released),
|
|
132
|
+
...patch,
|
|
133
|
+
updatedAt: nowMs(clock),
|
|
134
|
+
});
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
const operator = createMemoryOperator({
|
|
138
|
+
jobs,
|
|
139
|
+
steps,
|
|
140
|
+
clock,
|
|
141
|
+
settle,
|
|
142
|
+
liveHolder: (record) => {
|
|
143
|
+
const holder = liveByKey(record.name, record.idempotencyKey, record.tenantId);
|
|
144
|
+
return holder === undefined || holder.id === record.id ? undefined : holder;
|
|
145
|
+
},
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
/** The row, when this worker still holds it `running` — the fence both settles share. */
|
|
149
|
+
const heldBy = (jobId: string, by: ClaimIdentity): JobRecord | undefined => {
|
|
150
|
+
const record = jobs.get(jobId);
|
|
151
|
+
return record?.state === 'running' &&
|
|
152
|
+
record.claimedBy === by.workerId &&
|
|
153
|
+
record.claim === by.claim
|
|
154
|
+
? record
|
|
155
|
+
: undefined;
|
|
101
156
|
};
|
|
102
157
|
|
|
103
158
|
const introspect: JobIntrospection = {
|
|
104
159
|
job(jobId) {
|
|
105
160
|
return Promise.resolve(jobs.get(jobId));
|
|
106
161
|
},
|
|
107
|
-
|
|
108
|
-
// pg driver, and a synchronous throw out of a method typed `Promise<…>` is a second answer to
|
|
109
|
-
// one question.
|
|
110
|
-
async list(filter: JobFilter = {}) {
|
|
111
|
-
const rows = [...jobs.values()]
|
|
112
|
-
.filter((record) => filter.queue === undefined || record.queue === filter.queue)
|
|
113
|
-
.filter((record) => filter.name === undefined || record.name === filter.name)
|
|
114
|
-
.filter((record) => filter.state === undefined || record.state === filter.state)
|
|
115
|
-
// NEWEST first, as `createPgDriver`'s `order by created_at desc` is. Ascending here meant
|
|
116
|
-
// `x jobs ls` answered one thing against `x dev` and the opposite in production — and,
|
|
117
|
-
// because the limit is applied after the sort, a default page of the hundred OLDEST rows.
|
|
118
|
-
.sort((a, b) => b.createdAt - a.createdAt)
|
|
119
|
-
.slice(0, finiteCount('the memory driver list', 'limit', filter.limit ?? 100));
|
|
120
|
-
return rows;
|
|
121
|
-
},
|
|
162
|
+
...operator.members,
|
|
122
163
|
async deadLetters(limit = 100) {
|
|
123
164
|
const rows = [...jobs.values()]
|
|
124
165
|
.filter((record) => record.state === 'dead')
|
|
@@ -127,13 +168,9 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
127
168
|
return rows;
|
|
128
169
|
},
|
|
129
170
|
async requeue(jobId, requeueOptions) {
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
`no job ${jobId} in the memory driver`,
|
|
134
|
-
'requeue a job id returned by enqueue() or stats() — the memory driver holds no state across processes, so an id from another run will not resolve',
|
|
135
|
-
);
|
|
136
|
-
const record = existing;
|
|
171
|
+
const record = jobs.get(jobId);
|
|
172
|
+
// The one answer both drivers give an id nobody queued — it was `X_INVARIANT` here.
|
|
173
|
+
if (record === undefined) throw new JobNotFoundError({ jobId, driver: 'memory' });
|
|
137
174
|
// The two refusals `SQL_JOB_REQUEUE` and `SQL_JOB_LIVE_HOLDER` give, before anything moves.
|
|
138
175
|
if (!REQUEUEABLE_STATES.has(record.state)) {
|
|
139
176
|
throw new JobNotRequeueableError({ jobId, state: record.state });
|
|
@@ -143,7 +180,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
143
180
|
throw requeueKeyTaken({ ...record, holderId: holder.id });
|
|
144
181
|
}
|
|
145
182
|
if (requeueOptions?.fromStep !== undefined) {
|
|
146
|
-
// The target step and every step that started AFTER it — `
|
|
183
|
+
// The target step and every step that started AFTER it — `SQL_JOB_REQUEUE`, whose header
|
|
147
184
|
// says why a tie is kept. Earlier steps stay memoized; a later one replaying the old run's
|
|
148
185
|
// result is what this prevents.
|
|
149
186
|
const all = await steps.list(record.runId);
|
|
@@ -158,14 +195,18 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
158
195
|
}
|
|
159
196
|
// `settle` releases the claim — `SQL_JOB_REQUEUE` writes `claimed_by = null` too.
|
|
160
197
|
settle(jobId, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
|
|
198
|
+
signalEnqueued(record.queue);
|
|
161
199
|
const next = jobs.get(jobId);
|
|
162
200
|
return next ?? record;
|
|
163
201
|
},
|
|
164
202
|
cancel(jobId, reason) {
|
|
165
203
|
const existing = jobs.get(jobId);
|
|
166
|
-
//
|
|
167
|
-
//
|
|
168
|
-
|
|
204
|
+
// LIVE rows only, mirroring `SQL_CANCEL`: a job that already finished has nothing to stop.
|
|
205
|
+
// The fence was `state !== 'done'`, which let a cancel REWRITE a dead letter — the record of
|
|
206
|
+
// a failure, and its `lastError`, became the record of an operator's typo.
|
|
207
|
+
if (existing === undefined || !LIVE_STATES.has(existing.state)) {
|
|
208
|
+
return Promise.resolve(undefined);
|
|
209
|
+
}
|
|
169
210
|
// `settle`, not `update`: a cancellation RELEASES the claim, and `SQL_CANCEL` writes
|
|
170
211
|
// `visible_at = null, claimed_by = null` with the state. Left stamped, a cancelled row named
|
|
171
212
|
// the worker still holding it and carried that attempt's lease deadline — the pair
|
|
@@ -190,6 +231,12 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
190
231
|
// `Promise<…>` is a second answer to one question — caught by different code, and an
|
|
191
232
|
// unhandled exception rather than a settled promise wherever the caller holds the promise.
|
|
192
233
|
async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
|
|
234
|
+
// A caller-allocated id that already names a row is that row's publish, repeated: the same
|
|
235
|
+
// job, whatever state it has reached — `SQL_ENQUEUE`'s `not exists` is the pg half.
|
|
236
|
+
const published = request.id === undefined ? undefined : jobs.get(request.id);
|
|
237
|
+
if (published !== undefined) {
|
|
238
|
+
return { id: published.id, runId: published.runId, deduped: true };
|
|
239
|
+
}
|
|
193
240
|
const existing = liveByKey(request.name, request.idempotencyKey, request.tenantId);
|
|
194
241
|
if (existing !== undefined) {
|
|
195
242
|
if (request.onConflict === 'error') {
|
|
@@ -205,10 +252,13 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
205
252
|
const at = nowMs(clock);
|
|
206
253
|
const runAt = request.runAt ?? at;
|
|
207
254
|
const record: JobRecord = {
|
|
208
|
-
id: uuid(),
|
|
255
|
+
id: request.id ?? uuid(),
|
|
209
256
|
name: request.name,
|
|
210
257
|
queue: request.queue || DEFAULT_QUEUE,
|
|
211
|
-
|
|
258
|
+
// Through JSON, exactly as the pg driver binds it (`JSON.stringify(request.input ?? null)`):
|
|
259
|
+
// a live reference kept a `Date`, an `undefined` member and a non-enumerable property the
|
|
260
|
+
// queue in production never stores, so a job that read one passed here and failed there.
|
|
261
|
+
input: JSON.parse(JSON.stringify(request.input ?? null)) as unknown,
|
|
212
262
|
idempotencyKey: request.idempotencyKey,
|
|
213
263
|
runId: request.runId ?? uuid(),
|
|
214
264
|
attempt: 0,
|
|
@@ -235,6 +285,8 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
235
285
|
const wanted = new Set(claimOptions.queues);
|
|
236
286
|
const claimable = [...jobs.values()]
|
|
237
287
|
.filter((record) => wanted.has(record.queue))
|
|
288
|
+
// A paused queue is never claimed — `SQL_CLAIM`'s `not exists` over `x_job_pauses`.
|
|
289
|
+
.filter((record) => !operator.isQueuePaused(record.queue))
|
|
238
290
|
.filter((record) => {
|
|
239
291
|
if (record.runAt > at) return false;
|
|
240
292
|
if (record.state === 'ready' || record.state === 'delayed') return true;
|
|
@@ -246,12 +298,30 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
246
298
|
.slice(0, claimOptions.limit);
|
|
247
299
|
|
|
248
300
|
const out: ClaimedJob[] = [];
|
|
301
|
+
const buried: JobRecord[] = [];
|
|
249
302
|
for (const record of claimable) {
|
|
303
|
+
// `SQL_CLAIM`'s second arm. A `running` row reaches this loop only with a lapsed lease, and
|
|
304
|
+
// one that lapsed on its FINAL attempt has nothing left to be re-delivered for: handing
|
|
305
|
+
// it out was an attempt past `maxAttempts`, every lease, forever — a job that kills its
|
|
306
|
+
// worker took one per visibility timeout and never reached the dead-letter queue.
|
|
307
|
+
if (
|
|
308
|
+
record.state === 'running' &&
|
|
309
|
+
isFinalAttempt({ attempts: record.maxAttempts }, record.attempt)
|
|
310
|
+
) {
|
|
311
|
+
// `$5` of the statement: a job declaring `retry.deadLetter: false` is dropped.
|
|
312
|
+
const state = claimOptions.dropExhausted?.includes(record.name) ? 'failed' : 'dead';
|
|
313
|
+
settle(record.id, { state, lastError: LEASE_LAPSED_FINAL_ATTEMPT }, 'stack');
|
|
314
|
+
operator.counters.add(record.name, state, 0, at);
|
|
315
|
+
const dead = jobs.get(record.id);
|
|
316
|
+
if (dead !== undefined) buried.push(dead);
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
250
319
|
const claimed: ClaimedJob = {
|
|
251
320
|
...record,
|
|
252
321
|
state: 'running',
|
|
253
322
|
attempt: record.attempt + 1,
|
|
254
323
|
claimedBy: claimOptions.workerId,
|
|
324
|
+
claim: (record.claim ?? 0) + 1,
|
|
255
325
|
claimedAt: at,
|
|
256
326
|
visibleAt: at + claimOptions.visibilityTimeoutMs,
|
|
257
327
|
updatedAt: at,
|
|
@@ -259,32 +329,33 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
259
329
|
jobs.set(record.id, claimed);
|
|
260
330
|
out.push(claimed);
|
|
261
331
|
}
|
|
262
|
-
|
|
332
|
+
if (buried.length > 0) claimOptions.onExhausted?.(buried);
|
|
333
|
+
return out;
|
|
263
334
|
},
|
|
264
335
|
|
|
265
|
-
// Both settlements are FENCED on `running
|
|
266
|
-
// whose job was cancelled — or whose lease lapsed and whose job another
|
|
267
|
-
// would otherwise overwrite a row it no longer owns.
|
|
268
|
-
|
|
269
|
-
|
|
336
|
+
// Both settlements are FENCED on `running` AND on the claimer, as `SQL_ACK`/`SQL_NACK` are: an
|
|
337
|
+
// ack from a worker whose job was cancelled — or whose lease lapsed and whose job another
|
|
338
|
+
// worker re-claimed — would otherwise overwrite a row it no longer owns. The counter moves in
|
|
339
|
+
// the same step as the row, so the two cannot disagree.
|
|
340
|
+
ack(jobId: string, by: AckOptions): Promise<boolean> {
|
|
341
|
+
const record = heldBy(jobId, by);
|
|
342
|
+
if (record === undefined) return Promise.resolve(false);
|
|
270
343
|
settle(jobId, { state: 'done' });
|
|
271
|
-
|
|
344
|
+
if (by.counted !== false) {
|
|
345
|
+
operator.counters.add(record.name, 'done', by.durationMs ?? 0, nowMs(clock));
|
|
346
|
+
}
|
|
347
|
+
return Promise.resolve(true);
|
|
272
348
|
},
|
|
273
349
|
|
|
274
|
-
nack(jobId: string, nackOptions: NackOptions): Promise<
|
|
275
|
-
const record =
|
|
276
|
-
if (record === undefined
|
|
350
|
+
nack(jobId: string, nackOptions: NackOptions): Promise<boolean> {
|
|
351
|
+
const record = heldBy(jobId, nackOptions);
|
|
352
|
+
if (record === undefined) return Promise.resolve(false);
|
|
277
353
|
const at = nowMs(clock);
|
|
278
354
|
const counts = nackOptions.countsAsAttempt !== false;
|
|
279
355
|
const patch: Partial<JobRecord> = {
|
|
280
356
|
// `park`, never `counts`: parking is what leaves the ready bucket, and burning an attempt
|
|
281
357
|
// is a separate fact. A shed sets neither and stays `ready`, which is what it is.
|
|
282
|
-
state:
|
|
283
|
-
nackOptions.deadLetter === true
|
|
284
|
-
? 'dead'
|
|
285
|
-
: nackOptions.park === true
|
|
286
|
-
? 'suspended'
|
|
287
|
-
: 'ready',
|
|
358
|
+
state: nackState(nackOptions),
|
|
288
359
|
runAt: at + nackOptions.delayMs,
|
|
289
360
|
// A suspension must not burn an attempt, or a 3-day sleep dead-letters the run. Floored
|
|
290
361
|
// where `SQL_NACK` floors it (`greatest(attempt - 1, 0)`): the fence above is what keeps
|
|
@@ -292,9 +363,17 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
292
363
|
// being read as the only one.
|
|
293
364
|
attempt: counts ? record.attempt : Math.max(0, record.attempt - 1),
|
|
294
365
|
...(nackOptions.error === undefined ? {} : { lastError: nackOptions.error }),
|
|
366
|
+
// The stack belongs to the ERROR beside it, as `SQL_NACK` binds it: no error, no stack.
|
|
367
|
+
...(nackOptions.error === undefined || nackOptions.stack === undefined
|
|
368
|
+
? {}
|
|
369
|
+
: { lastErrorStack: nackOptions.stack.slice(0, MAX_ERROR_STACK_LENGTH) }),
|
|
295
370
|
};
|
|
296
|
-
settle(jobId, patch);
|
|
297
|
-
|
|
371
|
+
settle(jobId, patch, nackOptions.error === undefined ? undefined : 'stack');
|
|
372
|
+
const outcome = nackOutcome(nackOptions);
|
|
373
|
+
if (outcome !== undefined) {
|
|
374
|
+
operator.counters.add(record.name, outcome, nackOptions.durationMs ?? 0, at);
|
|
375
|
+
}
|
|
376
|
+
return Promise.resolve(true);
|
|
298
377
|
},
|
|
299
378
|
|
|
300
379
|
heartbeat(jobId: string, heartbeatOptions: HeartbeatOptions): Promise<boolean> {
|
|
@@ -304,7 +383,9 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
304
383
|
if (
|
|
305
384
|
record === undefined ||
|
|
306
385
|
record.state !== 'running' ||
|
|
307
|
-
(heartbeatOptions.workerId !== undefined &&
|
|
386
|
+
(heartbeatOptions.workerId !== undefined &&
|
|
387
|
+
record.claimedBy !== heartbeatOptions.workerId) ||
|
|
388
|
+
(heartbeatOptions.claim !== undefined && record.claim !== heartbeatOptions.claim)
|
|
308
389
|
) {
|
|
309
390
|
return Promise.resolve(false);
|
|
310
391
|
}
|
|
@@ -322,6 +403,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
322
403
|
delayed: 0,
|
|
323
404
|
running: 0,
|
|
324
405
|
suspended: 0,
|
|
406
|
+
failed: 0,
|
|
325
407
|
dead: 0,
|
|
326
408
|
oldestReadyMs: 0,
|
|
327
409
|
};
|
|
@@ -337,10 +419,15 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
|
|
|
337
419
|
} else if (waiting) next.delayed += 1;
|
|
338
420
|
else if (record.state === 'running') next.running += 1;
|
|
339
421
|
else if (record.state === 'suspended') next.suspended += 1;
|
|
422
|
+
else if (record.state === 'failed') next.failed += 1;
|
|
340
423
|
else if (record.state === 'dead') next.dead += 1;
|
|
341
424
|
byQueue.set(record.queue, next);
|
|
342
425
|
}
|
|
343
|
-
|
|
426
|
+
// Code units, as `collate "C"` orders them in `SQL_STATS` — never `localeCompare`, which
|
|
427
|
+
// answers by the runtime's locale and so differently from the database on the same names.
|
|
428
|
+
return Promise.resolve(
|
|
429
|
+
[...byQueue.values()].sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0)),
|
|
430
|
+
);
|
|
344
431
|
},
|
|
345
432
|
|
|
346
433
|
close(): Promise<void> {
|
package/src/driver-nats.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// a durable pull consumer per worker pool (`fetch` == claim, `ack`/`nak` map 1:1),
|
|
4
4
|
// `ack_wait` as the visibility timeout, and a KV bucket for step records.
|
|
5
5
|
|
|
6
|
+
import { NotImplementedError } from '@ultimat3/core';
|
|
6
7
|
import type {
|
|
7
8
|
ClaimedJob,
|
|
8
9
|
ClaimOptions,
|
|
@@ -13,7 +14,6 @@ import type {
|
|
|
13
14
|
NackOptions,
|
|
14
15
|
QueueStats,
|
|
15
16
|
} from './driver';
|
|
16
|
-
import { JobsNotImplementedError } from './errors';
|
|
17
17
|
import type { StepRecord, StepStore } from './steps';
|
|
18
18
|
|
|
19
19
|
// Names the seam that actually replaces the stub, and NOTHING ELSE — the two other repairs this
|
|
@@ -34,7 +34,10 @@ const FIX =
|
|
|
34
34
|
'call setJobDriver(createPgDriver()) at boot instead of this driver; nothing needs moving first, because enqueue here refuses too, so no job was ever written to it';
|
|
35
35
|
|
|
36
36
|
const unavailable = (method: string): never => {
|
|
37
|
-
throw new
|
|
37
|
+
throw new NotImplementedError({
|
|
38
|
+
cause: `nats jobs driver (${method}) is not implemented`,
|
|
39
|
+
fix: FIX,
|
|
40
|
+
});
|
|
38
41
|
};
|
|
39
42
|
|
|
40
43
|
const natsStepStore = (): StepStore => ({
|
|
@@ -70,10 +73,10 @@ export function createNatsDriver(_options: NatsDriverOptions = {}): JobDriver {
|
|
|
70
73
|
claim(_options: ClaimOptions): Promise<readonly ClaimedJob[]> {
|
|
71
74
|
return unavailable('claim');
|
|
72
75
|
},
|
|
73
|
-
ack(_jobId: string): Promise<
|
|
76
|
+
ack(_jobId: string): Promise<boolean> {
|
|
74
77
|
return unavailable('ack');
|
|
75
78
|
},
|
|
76
|
-
nack(_jobId: string, _options: NackOptions): Promise<
|
|
79
|
+
nack(_jobId: string, _options: NackOptions): Promise<boolean> {
|
|
77
80
|
return unavailable('nack');
|
|
78
81
|
},
|
|
79
82
|
heartbeat(_jobId: string, _options: HeartbeatOptions): Promise<boolean> {
|
package/src/driver-pg-ddl.ts
CHANGED
|
@@ -132,6 +132,12 @@ alter table x_outbox add column if not exists claimed_at timestamptz;
|
|
|
132
132
|
|
|
133
133
|
alter table x_outbox add column if not exists claimed_by text;
|
|
134
134
|
|
|
135
|
+
-- The run the published job carries, allocated when the row is STAGED so the enqueue that staged
|
|
136
|
+
-- it can answer which run it started. The row id is the published job id for the same reason.
|
|
137
|
+
-- Added by alter because x_outbox shipped without it. A row staged before this column existed is
|
|
138
|
+
-- given a run id by the default, which is what the driver would have minted at its publish.
|
|
139
|
+
alter table x_outbox add column if not exists run_id uuid not null default gen_random_uuid();
|
|
140
|
+
|
|
135
141
|
-- The scheduler watermark. Without a durable one a redeployed scheduler has no idea what the
|
|
136
142
|
-- pod it replaced already fired, so runRound takes the arming branch and every occurrence
|
|
137
143
|
-- between the two processes is dropped with nothing logged.
|
|
@@ -141,6 +147,13 @@ create table if not exists x_scheduler_state (
|
|
|
141
147
|
updated_at timestamptz not null default now()
|
|
142
148
|
);
|
|
143
149
|
|
|
150
|
+
-- The last occurrence that DISPATCHED, and when. Apart from the watermark, which arming a task or
|
|
151
|
+
-- skipping missed occurrences also moves: an operator asking when a task last ran is asking about
|
|
152
|
+
-- jobs that were queued. Added by alter because x_scheduler_state shipped without them.
|
|
153
|
+
alter table x_scheduler_state add column if not exists fired_occurrence_at timestamptz;
|
|
154
|
+
|
|
155
|
+
alter table x_scheduler_state add column if not exists fired_at timestamptz;
|
|
156
|
+
|
|
144
157
|
-- Leader election as an EXPIRING LEASE rather than a session advisory lock: the executor this
|
|
145
158
|
-- package is handed is a pool, and a session-level pg_try_advisory_lock is released the moment
|
|
146
159
|
-- that connection goes back to it. A row with an expiry needs no connection affinity at all.
|
|
@@ -176,4 +189,62 @@ create table if not exists x_job_events (
|
|
|
176
189
|
|
|
177
190
|
create index if not exists x_job_events_lookup_idx
|
|
178
191
|
on x_job_events (name, published_at);
|
|
192
|
+
|
|
193
|
+
-- What the body last reported through progress, as one value: done, total, note and the instant.
|
|
194
|
+
-- A column and not a table, because it is read with the row and dies with it. Added by alter
|
|
195
|
+
-- because x_jobs shipped without it.
|
|
196
|
+
alter table x_jobs add column if not exists progress jsonb;
|
|
197
|
+
|
|
198
|
+
-- The stack of the thrown value last_error describes, bounded by the driver before it is bound.
|
|
199
|
+
alter table x_jobs add column if not exists last_error_stack text;
|
|
200
|
+
|
|
201
|
+
-- How many times the row has been claimed, moved by the claim itself and never reset. A settle is
|
|
202
|
+
-- fenced on it as well as on claimed_by: a worker that takes back its own lapsed job has the same
|
|
203
|
+
-- id as the body still unwinding from the first claim, and only the ordinal tells the two apart.
|
|
204
|
+
alter table x_jobs add column if not exists claims int not null default 0;
|
|
205
|
+
|
|
206
|
+
-- The keyset a paged listing seeks by. Newest first, id as the tiebreak, so a page is an index
|
|
207
|
+
-- range and never a sort of the whole table.
|
|
208
|
+
create index if not exists x_jobs_created_idx on x_jobs (created_at desc, id desc);
|
|
209
|
+
|
|
210
|
+
-- A paused queue or task, one row each. A queue pause is a fact about the QUEUE and not a column
|
|
211
|
+
-- on every one of its rows: the claim reads this table with not exists, and a pause that had to
|
|
212
|
+
-- rewrite a million rows would not take effect within one poll interval.
|
|
213
|
+
create table if not exists x_job_pauses (
|
|
214
|
+
kind text not null,
|
|
215
|
+
name text not null,
|
|
216
|
+
paused_at timestamptz not null default now(),
|
|
217
|
+
primary key (kind, name)
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
-- The worker registry. One row per live worker, rewritten on its heartbeat and expired by time
|
|
221
|
+
-- like a lease: a killed worker announces nothing and its row stops being read. Never cleaned up
|
|
222
|
+
-- by anything that has to be running.
|
|
223
|
+
create table if not exists x_job_workers (
|
|
224
|
+
id text primary key,
|
|
225
|
+
host text not null,
|
|
226
|
+
queues jsonb not null,
|
|
227
|
+
concurrency int not null,
|
|
228
|
+
in_flight jsonb not null,
|
|
229
|
+
started_at timestamptz not null,
|
|
230
|
+
heartbeat_at timestamptz not null default now(),
|
|
231
|
+
expires_at timestamptz not null
|
|
232
|
+
);
|
|
233
|
+
|
|
234
|
+
-- Per-job history, in buckets. A settle adds to the one-minute bucket in the SAME statement that
|
|
235
|
+
-- moves the row, and the scheduler leader folds old buckets into wider ones, so the key count per
|
|
236
|
+
-- job name is fixed whatever the throughput.
|
|
237
|
+
create table if not exists x_job_counters (
|
|
238
|
+
job text not null,
|
|
239
|
+
bucket_ms int not null,
|
|
240
|
+
bucket_start timestamptz not null,
|
|
241
|
+
done bigint not null default 0,
|
|
242
|
+
retried bigint not null default 0,
|
|
243
|
+
failed bigint not null default 0,
|
|
244
|
+
dead bigint not null default 0,
|
|
245
|
+
duration_ms bigint not null default 0,
|
|
246
|
+
primary key (job, bucket_ms, bucket_start)
|
|
247
|
+
);
|
|
248
|
+
|
|
249
|
+
create index if not exists x_job_counters_age_idx on x_job_counters (bucket_ms, bucket_start);
|
|
179
250
|
`.trim();
|
|
@@ -3,34 +3,95 @@
|
|
|
3
3
|
// none decodes `timestamptz` as TEXT, so `toJobRecord` read `Number('2026-01-01 00:00:00+00')`
|
|
4
4
|
// and `x jobs ls` / `show` / `cancel` printed `NaN` for every timestamp.
|
|
5
5
|
|
|
6
|
+
import { JOBS_WAKE_CHANNEL } from './driver-pg-wake-sql';
|
|
7
|
+
|
|
6
8
|
/**
|
|
7
9
|
* The `JobRow` shape as a projection. Asking Postgres for epoch ms is what makes the decoding
|
|
8
10
|
* independent of the client's type map — never `select *`, whose correctness is the driver's
|
|
9
11
|
* opinion about `timestamptz` rather than this statement's.
|
|
10
12
|
*/
|
|
11
13
|
export const JOB_ROW_COLUMNS = `id, name, queue, input, idempotency_key, run_id, attempt,
|
|
12
|
-
max_attempts, state, tenant_id, last_error, claimed_by,
|
|
13
|
-
traceparent, enqueued_by,
|
|
14
|
+
max_attempts, state, tenant_id, last_error, last_error_stack, claimed_by, claims,
|
|
15
|
+
traceparent, enqueued_by, progress,
|
|
14
16
|
(extract(epoch from run_at) * 1000)::bigint as run_at,
|
|
15
17
|
(extract(epoch from visible_at) * 1000)::bigint as visible_at,
|
|
16
18
|
(extract(epoch from created_at) * 1000)::bigint as created_at,
|
|
17
19
|
(extract(epoch from updated_at) * 1000)::bigint as updated_at`;
|
|
18
20
|
|
|
21
|
+
/** `JOB_ROW_COLUMNS` by their output names — what a statement reading a derived table selects. */
|
|
22
|
+
const JOB_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attempt, max_attempts,
|
|
23
|
+
state, tenant_id, last_error, last_error_stack, claimed_by, claims, traceparent,
|
|
24
|
+
enqueued_by, progress, run_at, visible_at, created_at, updated_at`;
|
|
25
|
+
|
|
19
26
|
export const SQL_JOB_GET = `
|
|
20
27
|
select ${JOB_ROW_COLUMNS}
|
|
21
28
|
from x_jobs where id = $1
|
|
22
29
|
`.trim();
|
|
23
30
|
|
|
31
|
+
/**
|
|
32
|
+
* The predicates a page shares, whichever way it reads. $1 queue, $2 name, $3 state, $5 id prefix,
|
|
33
|
+
* $6 created from (ms), $7 created to (ms), $10 tenant. The prefix is compared, never `like`d: a
|
|
34
|
+
* `_` or `%` in it is a character, as `startsWith` reads it in the memory driver.
|
|
35
|
+
*/
|
|
36
|
+
const LIST_MATCH = `($1::text is null or queue = $1)
|
|
37
|
+
and ($2::text is null or name = $2)
|
|
38
|
+
and ($3::text is null or state = $3)
|
|
39
|
+
and ($5::text is null or left(id::text, length($5::text)) = $5::text)
|
|
40
|
+
and ($6::bigint is null or created_at >= to_timestamp($6::bigint / 1000.0))
|
|
41
|
+
and ($7::bigint is null or created_at < to_timestamp($7::bigint / 1000.0))
|
|
42
|
+
and ($10::text is null or tenant_id = $10)`;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* One page, newest first, sought by KEYSET. `(created_at, id) < (cursor)` is a row comparison the
|
|
46
|
+
* `x_jobs_created_idx` range answers directly; an offset would re-read every row before it and
|
|
47
|
+
* shift under an insert.
|
|
48
|
+
*
|
|
49
|
+
* The cursor carries the last row's `createdAt` in epoch MS, and the column holds microseconds —
|
|
50
|
+
* so the seek reads the cursor row's own `created_at` by id rather than trusting a rounded value,
|
|
51
|
+
* which would skip or repeat its millisecond-mates. A cursor row deleted mid-walk falls back to
|
|
52
|
+
* the millisecond after it: rows of that one millisecond may then repeat, and none is skipped.
|
|
53
|
+
*
|
|
54
|
+
* `LIST_MATCH`'s parameters, plus $4 limit, $8 cursor createdAt (ms), $9 cursor id.
|
|
55
|
+
*/
|
|
24
56
|
export const SQL_JOB_LIST = `
|
|
25
57
|
select ${JOB_ROW_COLUMNS}
|
|
26
58
|
from x_jobs
|
|
27
|
-
where
|
|
28
|
-
and ($
|
|
29
|
-
|
|
30
|
-
|
|
59
|
+
where ${LIST_MATCH}
|
|
60
|
+
and ($9::uuid is null or (created_at, id) < (
|
|
61
|
+
coalesce(
|
|
62
|
+
(select c.created_at from x_jobs c where c.id = $9::uuid),
|
|
63
|
+
to_timestamp(($8::bigint + 1) / 1000.0)
|
|
64
|
+
),
|
|
65
|
+
$9::uuid
|
|
66
|
+
))
|
|
67
|
+
order by created_at desc, id desc
|
|
31
68
|
limit $4
|
|
32
69
|
`.trim();
|
|
33
70
|
|
|
71
|
+
/**
|
|
72
|
+
* The page BEFORE a cursor: the `limit` rows nearest it on the newer side — read oldest first so
|
|
73
|
+
* `limit` keeps the nearest, then turned newest first like every page. The same parameters as
|
|
74
|
+
* `SQL_JOB_LIST`, $8/$9 being the first row of the page the reader is leaving. A cursor row
|
|
75
|
+
* deleted meanwhile falls back to its millisecond: a row of it may repeat, none is skipped.
|
|
76
|
+
*/
|
|
77
|
+
export const SQL_JOB_LIST_BEFORE = `
|
|
78
|
+
select ${JOB_ROW_NAMES} from (
|
|
79
|
+
select ${JOB_ROW_COLUMNS}
|
|
80
|
+
from x_jobs
|
|
81
|
+
where ${LIST_MATCH}
|
|
82
|
+
and (created_at, id) > (
|
|
83
|
+
coalesce(
|
|
84
|
+
(select c.created_at from x_jobs c where c.id = $9::uuid),
|
|
85
|
+
to_timestamp($8::bigint / 1000.0)
|
|
86
|
+
),
|
|
87
|
+
$9::uuid
|
|
88
|
+
)
|
|
89
|
+
order by created_at asc, id asc
|
|
90
|
+
limit $4
|
|
91
|
+
) page
|
|
92
|
+
order by created_at desc, id desc
|
|
93
|
+
`.trim();
|
|
94
|
+
|
|
34
95
|
export const SQL_JOB_DEAD_LETTERS = `
|
|
35
96
|
select ${JOB_ROW_COLUMNS}
|
|
36
97
|
from x_jobs where state = 'dead' order by updated_at desc limit $1
|
|
@@ -41,14 +102,37 @@ select ${JOB_ROW_COLUMNS}
|
|
|
41
102
|
* to FINISHED states and releasing the claim: requeueing a running row left `claimed_by` set with
|
|
42
103
|
* state `ready`, and a second worker claimed it — the job ran twice. `REQUEUEABLE_STATES`
|
|
43
104
|
* (`driver.ts`) is the same set; a row this matches nothing for was refused before it was sent,
|
|
44
|
-
* or raced to live in between.
|
|
105
|
+
* or raced to live in between. `woke` announces the row in the statement that made it claimable
|
|
106
|
+
* (`driver-pg-wake-sql.ts`): `x jobs retry` is a job starting now, not at an idle worker's poll.
|
|
107
|
+
*
|
|
108
|
+
* $2 is `fromStep`, or null — and the steps are dropped HERE, off the row this statement moved.
|
|
109
|
+
* They were a statement of their own, sent first: a requeue that then matched nothing (the row
|
|
110
|
+
* raced to live) had already deleted a running job's steps, and a crash between the two left a
|
|
111
|
+
* dead letter with its steps gone and no retry queued.
|
|
112
|
+
*
|
|
113
|
+
* "From that step onward": the step `fromStep` names and every step that started AFTER it.
|
|
114
|
+
* Deleting the named step alone left later steps replaying results computed from the old run.
|
|
115
|
+
* Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
|
|
116
|
+
* later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
|
|
45
117
|
*/
|
|
46
118
|
export const SQL_JOB_REQUEUE = `
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
119
|
+
with requeued as (
|
|
120
|
+
update x_jobs
|
|
121
|
+
set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
|
|
122
|
+
claimed_by = null, visible_at = null
|
|
123
|
+
where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
|
|
124
|
+
returning ${JOB_ROW_COLUMNS},
|
|
125
|
+
pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
|
|
126
|
+
), dropped as (
|
|
127
|
+
delete from x_job_steps s
|
|
128
|
+
using requeued r
|
|
129
|
+
where $2::text is not null
|
|
130
|
+
and s.run_id = r.run_id
|
|
131
|
+
and (s.name = $2::text
|
|
132
|
+
or s.started_at > (select t.started_at from x_job_steps t
|
|
133
|
+
where t.run_id = r.run_id and t.name = $2::text))
|
|
134
|
+
)
|
|
135
|
+
select ${JOB_ROW_NAMES}, woke from requeued
|
|
52
136
|
`.trim();
|
|
53
137
|
|
|
54
138
|
/**
|
|
@@ -61,16 +145,3 @@ select id from x_jobs
|
|
|
61
145
|
and id <> $4 and state in ('ready', 'delayed', 'running', 'suspended')
|
|
62
146
|
limit 1
|
|
63
147
|
`.trim();
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* "From that step onward": the step `fromStep` names and every step that started AFTER it.
|
|
67
|
-
* Deleting the named step alone left later steps replaying results computed from the old run.
|
|
68
|
-
* Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
|
|
69
|
-
* later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
|
|
70
|
-
*/
|
|
71
|
-
export const SQL_STEPS_FROM = `
|
|
72
|
-
delete from x_job_steps
|
|
73
|
-
where run_id = $1
|
|
74
|
-
and (name = $2
|
|
75
|
-
or started_at > (select started_at from x_job_steps where run_id = $1 and name = $2))
|
|
76
|
-
`.trim();
|