@ultimat3/jobs 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +165 -124
- package/README.md +360 -21
- package/package.json +6 -6
- package/src/backfill-ledger.ts +24 -2
- package/src/backfill-pass.ts +22 -5
- package/src/claim-exhausted.ts +76 -0
- package/src/concurrency.ts +144 -0
- package/src/counters.ts +105 -0
- package/src/describe.ts +29 -0
- package/src/driver-memory-operator.ts +319 -0
- package/src/driver-memory.ts +146 -59
- package/src/driver-nats.ts +7 -4
- package/src/driver-pg-ddl.ts +71 -0
- package/src/driver-pg-jobs-sql.ts +96 -25
- package/src/driver-pg-operator-sql.ts +323 -0
- package/src/driver-pg-operator.ts +316 -0
- package/src/driver-pg-outbox-sql.ts +110 -0
- package/src/driver-pg-rows.ts +34 -3
- package/src/driver-pg-settle-sql.ts +71 -0
- package/src/driver-pg-sql.ts +171 -151
- package/src/driver-pg-wake-sql.ts +76 -0
- package/src/driver-pg.ts +150 -105
- package/src/driver-redis.ts +7 -4
- package/src/driver.ts +133 -35
- package/src/enqueue-signal.ts +50 -0
- package/src/errors-concurrency.ts +91 -0
- package/src/errors-operator.ts +102 -0
- package/src/errors-requeue.ts +21 -3
- package/src/errors.ts +18 -30
- package/src/events-pg.ts +45 -45
- package/src/events.ts +38 -2
- package/src/execute.ts +161 -81
- package/src/export.ts +23 -4
- package/src/heartbeat.ts +9 -3
- package/src/idle-backoff.ts +74 -0
- package/src/index.ts +115 -16
- package/src/inspect-operator.ts +49 -0
- package/src/inspect.ts +46 -5
- package/src/introspection.ts +274 -0
- package/src/job.ts +83 -21
- package/src/leases.ts +49 -9
- package/src/limits.ts +33 -7
- package/src/metrics.ts +6 -1
- package/src/outbox-pg.ts +4 -2
- package/src/outbox-relay.ts +114 -39
- package/src/outbox.ts +59 -8
- package/src/progress.ts +78 -0
- package/src/purge.ts +1 -1
- package/src/queue-wake.ts +219 -0
- package/src/redact-input.ts +26 -0
- package/src/renewal-timer.ts +28 -10
- package/src/retry-classification.ts +13 -1
- package/src/retry.ts +10 -2
- package/src/run-deadline.ts +57 -0
- package/src/scheduler-leader.ts +26 -0
- package/src/scheduler-occurrences.ts +76 -0
- package/src/scheduler-pg.ts +58 -4
- package/src/scheduler-state.ts +73 -0
- package/src/scheduler.ts +150 -111
- package/src/settled.ts +161 -0
- package/src/steps-suspension.ts +46 -0
- package/src/steps-timeout.ts +33 -0
- package/src/steps.ts +55 -81
- package/src/worker-admit.ts +167 -0
- package/src/worker-fleet-slots.ts +157 -54
- package/src/worker-hand-back.ts +6 -1
- package/src/worker-key-busy.ts +107 -0
- package/src/worker-loop.ts +169 -0
- package/src/worker-registry.ts +118 -0
- package/src/worker-run.ts +9 -0
- package/src/worker-types.ts +23 -0
- package/src/worker.ts +142 -130
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
// The memory driver's operator surface: paging, bulk, pause, promote, the worker registry,
|
|
2
|
+
// progress and counters, over the SAME map the claim loop reads. Apart from `driver-memory.ts`
|
|
3
|
+
// because that file is the six-method queue; this is everything `JobIntrospection` added, written
|
|
4
|
+
// to answer exactly as the pg statements in `driver-pg-operator-sql.ts` do.
|
|
5
|
+
|
|
6
|
+
import type { Clock } from '@ultimat3/core';
|
|
7
|
+
import { assert, finiteOption } from '@ultimat3/core';
|
|
8
|
+
import { nowMs } from './clock';
|
|
9
|
+
import type { MemoryCounters } from './counters';
|
|
10
|
+
import { createMemoryCounters } from './counters';
|
|
11
|
+
import type { JobRecord } from './driver';
|
|
12
|
+
import { REQUEUEABLE_STATES } from './driver';
|
|
13
|
+
import { signalEnqueued } from './enqueue-signal';
|
|
14
|
+
import { JobNotRemovableError } from './errors-operator';
|
|
15
|
+
import type {
|
|
16
|
+
BulkFilter,
|
|
17
|
+
BulkResult,
|
|
18
|
+
JobFilter,
|
|
19
|
+
JobIntrospection,
|
|
20
|
+
PausedName,
|
|
21
|
+
TaskFire,
|
|
22
|
+
WorkerRecord,
|
|
23
|
+
} from './introspection';
|
|
24
|
+
import {
|
|
25
|
+
MAX_BULK_ROWS,
|
|
26
|
+
MAX_TASK_FIRES,
|
|
27
|
+
MAX_WORKER_IN_FLIGHT,
|
|
28
|
+
PROMOTABLE_STATES,
|
|
29
|
+
pageLimit,
|
|
30
|
+
parseJobCursor,
|
|
31
|
+
} from './introspection';
|
|
32
|
+
import type { StepStore } from './steps';
|
|
33
|
+
|
|
34
|
+
export interface MemoryQueueState {
|
|
35
|
+
readonly jobs: Map<string, JobRecord>;
|
|
36
|
+
readonly steps: StepStore;
|
|
37
|
+
readonly clock: Clock;
|
|
38
|
+
/** The LIVE row holding this record's `(name, tenant, key)`, other than itself. */
|
|
39
|
+
liveHolder(record: JobRecord): JobRecord | undefined;
|
|
40
|
+
/** Rewrite a row and release its claim — the driver's own `settle`. */
|
|
41
|
+
settle(id: string, patch: Partial<JobRecord>): void;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type MemoryOperatorMembers = Omit<
|
|
45
|
+
JobIntrospection,
|
|
46
|
+
'job' | 'deadLetters' | 'requeue' | 'cancel'
|
|
47
|
+
>;
|
|
48
|
+
|
|
49
|
+
export interface MemoryOperator {
|
|
50
|
+
readonly members: MemoryOperatorMembers;
|
|
51
|
+
readonly counters: MemoryCounters;
|
|
52
|
+
/** The claim scan's question: is this queue held back? */
|
|
53
|
+
isQueuePaused(queue: string): boolean;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Newest first by `(createdAt, id)` — the keyset `SQL_JOB_LIST` orders and seeks by. */
|
|
57
|
+
const newestFirst = (a: JobRecord, b: JobRecord): number =>
|
|
58
|
+
b.createdAt - a.createdAt || (a.id < b.id ? 1 : a.id > b.id ? -1 : 0);
|
|
59
|
+
|
|
60
|
+
const inBulk = (record: JobRecord, filter: BulkFilter): boolean =>
|
|
61
|
+
record.state === filter.state &&
|
|
62
|
+
(filter.queue === undefined || record.queue === filter.queue) &&
|
|
63
|
+
(filter.name === undefined || record.name === filter.name) &&
|
|
64
|
+
(filter.tenantId === undefined || record.tenantId === filter.tenantId);
|
|
65
|
+
|
|
66
|
+
type Keyset = Pick<JobRecord, 'createdAt' | 'id'>;
|
|
67
|
+
|
|
68
|
+
/** `a` strictly newer than `b` by `(createdAt, id)`, in the keyset's own order. */
|
|
69
|
+
const newerThan = (a: Keyset, b: Keyset): boolean =>
|
|
70
|
+
a.createdAt > b.createdAt || (a.createdAt === b.createdAt && a.id > b.id);
|
|
71
|
+
|
|
72
|
+
const byName = (a: PausedName, b: PausedName): number =>
|
|
73
|
+
a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
|
|
74
|
+
|
|
75
|
+
function pauseSet(clock: Clock): {
|
|
76
|
+
pause(name: string): void;
|
|
77
|
+
resume(name: string): void;
|
|
78
|
+
has(name: string): boolean;
|
|
79
|
+
list(): readonly PausedName[];
|
|
80
|
+
} {
|
|
81
|
+
const paused = new Map<string, number>();
|
|
82
|
+
return {
|
|
83
|
+
// The FIRST pause is the one recorded, as `on conflict do nothing` keeps it in pg.
|
|
84
|
+
pause: (name) => {
|
|
85
|
+
if (!paused.has(name)) paused.set(name, nowMs(clock));
|
|
86
|
+
},
|
|
87
|
+
resume: (name) => {
|
|
88
|
+
paused.delete(name);
|
|
89
|
+
},
|
|
90
|
+
has: (name) => paused.has(name),
|
|
91
|
+
list: () => [...paused].map(([name, pausedAt]) => ({ name, pausedAt })).sort(byName),
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function createMemoryOperator(state: MemoryQueueState): MemoryOperator {
|
|
96
|
+
const { jobs, clock } = state;
|
|
97
|
+
const counters = createMemoryCounters();
|
|
98
|
+
const queues = pauseSet(clock);
|
|
99
|
+
const tasks = pauseSet(clock);
|
|
100
|
+
const workers = new Map<string, WorkerRecord & { readonly expiresAt: number }>();
|
|
101
|
+
const fires = new Map<string, TaskFire>();
|
|
102
|
+
|
|
103
|
+
// `eligible` is the verb's own predicate, applied BEFORE the bound as the pg statement's `where`
|
|
104
|
+
// is applied before its `limit`: rows the verb would skip never spend the bound.
|
|
105
|
+
const bulk = async (
|
|
106
|
+
filter: BulkFilter,
|
|
107
|
+
apply: (record: JobRecord) => Promise<boolean> | boolean,
|
|
108
|
+
eligible: (record: JobRecord) => boolean = () => true,
|
|
109
|
+
): Promise<BulkResult> => {
|
|
110
|
+
let affected = 0;
|
|
111
|
+
const matching = [...jobs.values()].filter(
|
|
112
|
+
(record) => inBulk(record, filter) && eligible(record),
|
|
113
|
+
);
|
|
114
|
+
for (const record of matching.sort(newestFirst).reverse().slice(0, MAX_BULK_ROWS)) {
|
|
115
|
+
if (await apply(record)) affected += 1;
|
|
116
|
+
}
|
|
117
|
+
const remaining = [...jobs.values()].filter((record) => inBulk(record, filter)).length;
|
|
118
|
+
return { affected, remaining };
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const members: MemoryOperatorMembers = {
|
|
122
|
+
// `async`, so a refused bound REJECTS here exactly as it does on the pg driver.
|
|
123
|
+
async list(filter: JobFilter = {}) {
|
|
124
|
+
const limit = pageLimit('the memory driver list', filter.limit);
|
|
125
|
+
assert(
|
|
126
|
+
filter.after === undefined || filter.before === undefined,
|
|
127
|
+
'list was asked for a page both after and before a cursor',
|
|
128
|
+
'pass one of after: jobCursor(lastRow) or before: jobCursor(firstRow)',
|
|
129
|
+
);
|
|
130
|
+
const after =
|
|
131
|
+
filter.after === undefined
|
|
132
|
+
? undefined
|
|
133
|
+
: parseJobCursor(filter.after, 'the memory driver list');
|
|
134
|
+
const before =
|
|
135
|
+
filter.before === undefined
|
|
136
|
+
? undefined
|
|
137
|
+
: parseJobCursor(filter.before, 'the memory driver list', 'before');
|
|
138
|
+
const { idPrefix, createdFrom, createdTo, tenantId } = filter;
|
|
139
|
+
const matching = [...jobs.values()]
|
|
140
|
+
.filter((record) => filter.queue === undefined || record.queue === filter.queue)
|
|
141
|
+
.filter((record) => filter.name === undefined || record.name === filter.name)
|
|
142
|
+
.filter((record) => filter.state === undefined || record.state === filter.state)
|
|
143
|
+
.filter((record) => tenantId === undefined || record.tenantId === tenantId)
|
|
144
|
+
.filter((record) => idPrefix === undefined || record.id.startsWith(idPrefix))
|
|
145
|
+
.filter((record) => createdFrom === undefined || record.createdAt >= createdFrom)
|
|
146
|
+
.filter((record) => createdTo === undefined || record.createdAt < createdTo)
|
|
147
|
+
.filter((record) => after === undefined || newerThan(after, record))
|
|
148
|
+
.filter((record) => before === undefined || newerThan(record, before))
|
|
149
|
+
.sort(newestFirst);
|
|
150
|
+
// Backwards: the rows NEAREST the cursor, which are the oldest of the newer ones.
|
|
151
|
+
return before === undefined
|
|
152
|
+
? matching.slice(0, limit)
|
|
153
|
+
: matching.slice(Math.max(0, matching.length - limit));
|
|
154
|
+
},
|
|
155
|
+
|
|
156
|
+
async remove(jobId) {
|
|
157
|
+
const record = jobs.get(jobId);
|
|
158
|
+
if (record === undefined) return undefined;
|
|
159
|
+
if (record.state === 'running') throw new JobNotRemovableError({ jobId });
|
|
160
|
+
jobs.delete(jobId);
|
|
161
|
+
await state.steps.clear(record.runId);
|
|
162
|
+
return record;
|
|
163
|
+
},
|
|
164
|
+
|
|
165
|
+
async requeueMany(filter) {
|
|
166
|
+
assert(
|
|
167
|
+
REQUEUEABLE_STATES.has(filter.state),
|
|
168
|
+
`requeueMany was asked for ${filter.state} jobs, and only a finished job can be requeued`,
|
|
169
|
+
"pass state: 'dead', 'failed', 'cancelled' or 'done' to requeueMany",
|
|
170
|
+
);
|
|
171
|
+
// A key a live job holds stays where it is — the single requeue's `X_JOB_DUPLICATE`,
|
|
172
|
+
// answered as "not this one" so one conflict does not fail a thousand rows.
|
|
173
|
+
const free = (record: JobRecord): boolean => state.liveHolder(record) === undefined;
|
|
174
|
+
// `SQL_JOB_REQUEUE_MANY`'s candidates: no live holder, and ONE row per idempotency namespace
|
|
175
|
+
// — the oldest — both decided BEFORE the bound. Decided inside `apply`, a held row spent a
|
|
176
|
+
// slot of `MAX_BULK_ROWS` and was then skipped: a thousand held rows ahead of one free row
|
|
177
|
+
// answered `affected: 0` here and `affected: 1` on Postgres.
|
|
178
|
+
const oldest = new Map<string, JobRecord>();
|
|
179
|
+
for (const record of jobs.values()) {
|
|
180
|
+
if (!inBulk(record, filter) || !free(record)) continue;
|
|
181
|
+
const key = JSON.stringify([record.name, record.tenantId ?? '', record.idempotencyKey]);
|
|
182
|
+
const seen = oldest.get(key);
|
|
183
|
+
if (seen === undefined || newerThan(seen, record)) oldest.set(key, record);
|
|
184
|
+
}
|
|
185
|
+
const candidates = new Set([...oldest.values()].map((record) => record.id));
|
|
186
|
+
const result = await bulk(
|
|
187
|
+
filter,
|
|
188
|
+
(record) => {
|
|
189
|
+
if (!free(record)) return false;
|
|
190
|
+
state.settle(record.id, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
|
|
191
|
+
return true;
|
|
192
|
+
},
|
|
193
|
+
(record) => candidates.has(record.id),
|
|
194
|
+
);
|
|
195
|
+
// The pg driver's `SQL_WAKE`, in-process: an operator's repair starts now.
|
|
196
|
+
if (result.affected > 0) signalEnqueued(filter.queue);
|
|
197
|
+
// What a second call could still move: rows of the filter whose key no live job holds. A
|
|
198
|
+
// row requeued just now holds its key, so its dead siblings left with it.
|
|
199
|
+
const remaining = [...jobs.values()].filter(
|
|
200
|
+
(record) => inBulk(record, filter) && free(record),
|
|
201
|
+
).length;
|
|
202
|
+
return { affected: result.affected, remaining };
|
|
203
|
+
},
|
|
204
|
+
|
|
205
|
+
async promoteMany(filter) {
|
|
206
|
+
assert(
|
|
207
|
+
PROMOTABLE_STATES.has(filter.state),
|
|
208
|
+
`promoteMany was asked for ${filter.state} jobs, and only a job waiting on its run time can be promoted`,
|
|
209
|
+
"pass state: 'delayed' or 'ready' to promoteMany",
|
|
210
|
+
);
|
|
211
|
+
const at = nowMs(clock);
|
|
212
|
+
const result = await bulk(
|
|
213
|
+
filter,
|
|
214
|
+
(record) => {
|
|
215
|
+
jobs.set(record.id, { ...record, state: 'ready', runAt: at, updatedAt: at });
|
|
216
|
+
return true;
|
|
217
|
+
},
|
|
218
|
+
(record) => record.runAt > at,
|
|
219
|
+
);
|
|
220
|
+
if (result.affected > 0) signalEnqueued(filter.queue);
|
|
221
|
+
// Rows of the filter still waiting — a due `ready` row has nothing left to promote.
|
|
222
|
+
const waiting = [...jobs.values()].filter(
|
|
223
|
+
(record) => inBulk(record, filter) && record.runAt > at,
|
|
224
|
+
).length;
|
|
225
|
+
return { affected: result.affected, remaining: waiting };
|
|
226
|
+
},
|
|
227
|
+
|
|
228
|
+
async removeMany(filter) {
|
|
229
|
+
if (filter.state === 'running') throw new JobNotRemovableError({});
|
|
230
|
+
return bulk(filter, async (record) => {
|
|
231
|
+
jobs.delete(record.id);
|
|
232
|
+
await state.steps.clear(record.runId);
|
|
233
|
+
return true;
|
|
234
|
+
});
|
|
235
|
+
},
|
|
236
|
+
|
|
237
|
+
promote(jobId) {
|
|
238
|
+
const record = jobs.get(jobId);
|
|
239
|
+
const at = nowMs(clock);
|
|
240
|
+
const waiting =
|
|
241
|
+
record !== undefined &&
|
|
242
|
+
(record.state === 'ready' || record.state === 'delayed') &&
|
|
243
|
+
record.runAt > at;
|
|
244
|
+
if (record === undefined || !waiting) return Promise.resolve(undefined);
|
|
245
|
+
const next: JobRecord = { ...record, state: 'ready', runAt: at, updatedAt: at };
|
|
246
|
+
jobs.set(jobId, next);
|
|
247
|
+
signalEnqueued(next.queue);
|
|
248
|
+
return Promise.resolve(next);
|
|
249
|
+
},
|
|
250
|
+
|
|
251
|
+
pauseQueue: (queue) => Promise.resolve(queues.pause(queue)),
|
|
252
|
+
resumeQueue(queue) {
|
|
253
|
+
queues.resume(queue);
|
|
254
|
+
signalEnqueued(queue);
|
|
255
|
+
return Promise.resolve();
|
|
256
|
+
},
|
|
257
|
+
pausedQueues: () => Promise.resolve(queues.list()),
|
|
258
|
+
pauseTask: (task) => Promise.resolve(tasks.pause(task)),
|
|
259
|
+
resumeTask: (task) => Promise.resolve(tasks.resume(task)),
|
|
260
|
+
pausedTasks: () => Promise.resolve(tasks.list()),
|
|
261
|
+
|
|
262
|
+
recordTaskFire({ task, occurrenceMs }) {
|
|
263
|
+
fires.set(task, { task, occurrenceMs, firedAt: nowMs(clock) });
|
|
264
|
+
return Promise.resolve();
|
|
265
|
+
},
|
|
266
|
+
taskFires: () =>
|
|
267
|
+
Promise.resolve(
|
|
268
|
+
[...fires.values()]
|
|
269
|
+
// Code units, as `collate "C"` orders them in `SQL_TASK_FIRES`.
|
|
270
|
+
.sort((a, b) => (a.task < b.task ? -1 : a.task > b.task ? 1 : 0))
|
|
271
|
+
.slice(0, MAX_TASK_FIRES),
|
|
272
|
+
),
|
|
273
|
+
|
|
274
|
+
announceWorker(worker, ttlMs) {
|
|
275
|
+
const at = nowMs(clock);
|
|
276
|
+
workers.set(worker.id, {
|
|
277
|
+
...worker,
|
|
278
|
+
queues: [...worker.queues],
|
|
279
|
+
inFlight: worker.inFlight.slice(0, MAX_WORKER_IN_FLIGHT),
|
|
280
|
+
heartbeatAt: at,
|
|
281
|
+
expiresAt: at + finiteOption('the memory worker registry', 'ttlMs', ttlMs),
|
|
282
|
+
});
|
|
283
|
+
return Promise.resolve();
|
|
284
|
+
},
|
|
285
|
+
forgetWorker(workerId) {
|
|
286
|
+
workers.delete(workerId);
|
|
287
|
+
return Promise.resolve();
|
|
288
|
+
},
|
|
289
|
+
workers() {
|
|
290
|
+
const at = nowMs(clock);
|
|
291
|
+
// Expired by reading, never by cleanup: a killed worker announces nothing and is gone.
|
|
292
|
+
for (const [id, worker] of workers) if (worker.expiresAt <= at) workers.delete(id);
|
|
293
|
+
return Promise.resolve(
|
|
294
|
+
[...workers.values()]
|
|
295
|
+
.map(({ expiresAt: _expiresAt, ...worker }) => worker)
|
|
296
|
+
.sort((a, b) => a.startedAt - b.startedAt || (a.id < b.id ? -1 : 1)),
|
|
297
|
+
);
|
|
298
|
+
},
|
|
299
|
+
|
|
300
|
+
recordProgress(jobId, by, progress) {
|
|
301
|
+
const record = jobs.get(jobId);
|
|
302
|
+
// The settle's fence: progress from a claim the row no longer carries is dropped.
|
|
303
|
+
if (
|
|
304
|
+
record?.state === 'running' &&
|
|
305
|
+
record.claimedBy === by.workerId &&
|
|
306
|
+
record.claim === by.claim
|
|
307
|
+
) {
|
|
308
|
+
jobs.set(jobId, { ...record, progress });
|
|
309
|
+
}
|
|
310
|
+
return Promise.resolve();
|
|
311
|
+
},
|
|
312
|
+
|
|
313
|
+
counters: (query) => Promise.resolve(counters.list(query)),
|
|
314
|
+
counterTotals: (sinceMs) => Promise.resolve(counters.totals(sinceMs)),
|
|
315
|
+
rollupCounters: () => Promise.resolve(counters.rollup(nowMs(clock))),
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
return { members, counters, isQueuePaused: (queue) => queues.has(queue) };
|
|
319
|
+
}
|