@ayepi/work 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +206 -0
- package/dist/index.cjs +2020 -0
- package/dist/index.d.cts +1167 -0
- package/dist/index.d.ts +1167 -0
- package/dist/index.js +1920 -0
- package/package.json +66 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,1920 @@
|
|
|
1
|
+
import { ageDoer, balancedDoer, priorityDoer, unlimitedDoer, unlimitedDoer as unlimitedDoer$1 } from "@ayepi/core/doer";
|
|
2
|
+
import { DEFAULT_RETRY_OPTIONS, RetryAbort, RetryAbort as RetryAbort$1, backoff, backoff as backoff$1, getDefaultRetryOptions, getDefaultRetryOptions as getDefaultRetryOptions$1, retry, retry as retry$1, setDefaultRetryOptions } from "@ayepi/core/retry";
|
|
3
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { dirname } from "node:path";
|
|
5
|
+
import { randomUUID } from "node:crypto";
|
|
6
|
+
import { DEFAULT_BUCKETS, createMetrics, createMetrics as createMetrics$1, formatPrometheus } from "@ayepi/core/stats";
|
|
7
|
+
//#region src/json.ts
|
|
8
|
+
/** Wrapper key marking a tagged value (`{ [TAG]: 'Date', value: … }`). */
|
|
9
|
+
const TAG = "$ayepi";
|
|
10
|
+
const isTagged = (v) => v !== null && typeof v === "object" && TAG in v;
|
|
11
|
+
/**
|
|
12
|
+
* The default {@link JsonCodec}. Tags values JSON can't represent natively so they
|
|
13
|
+
* survive a `stringify` → `parse` round-trip:
|
|
14
|
+
*
|
|
15
|
+
* | Value | Encoded as |
|
|
16
|
+
* |-------------|-------------------------------------|
|
|
17
|
+
* | `undefined` | `{ $ayepi:'undefined' }` |
|
|
18
|
+
* | `bigint` | `{ $ayepi:'BigInt', value:'123' }` |
|
|
19
|
+
* | `Date` | `{ $ayepi:'Date', value:<iso> }` |
|
|
20
|
+
* | `Map` | `{ $ayepi:'Map', value:[[k,v]…] }` |
|
|
21
|
+
* | `Set` | `{ $ayepi:'Set', value:[…] }` |
|
|
22
|
+
* | `Error` | `{ $ayepi:'Error', value:{name,message,stack} }` |
|
|
23
|
+
*/
|
|
24
|
+
const defaultCodec = {
|
|
25
|
+
stringify(value) {
|
|
26
|
+
return JSON.stringify(value, function(key, encoded) {
|
|
27
|
+
const raw = this[key];
|
|
28
|
+
if (raw === void 0) return { [TAG]: "undefined" };
|
|
29
|
+
if (typeof raw === "bigint") return {
|
|
30
|
+
[TAG]: "BigInt",
|
|
31
|
+
value: raw.toString()
|
|
32
|
+
};
|
|
33
|
+
if (raw instanceof Date) return {
|
|
34
|
+
[TAG]: "Date",
|
|
35
|
+
value: raw.toISOString()
|
|
36
|
+
};
|
|
37
|
+
if (raw instanceof Map) return {
|
|
38
|
+
[TAG]: "Map",
|
|
39
|
+
value: [...raw.entries()]
|
|
40
|
+
};
|
|
41
|
+
if (raw instanceof Set) return {
|
|
42
|
+
[TAG]: "Set",
|
|
43
|
+
value: [...raw.values()]
|
|
44
|
+
};
|
|
45
|
+
if (raw instanceof Error) return {
|
|
46
|
+
[TAG]: "Error",
|
|
47
|
+
value: {
|
|
48
|
+
name: raw.name,
|
|
49
|
+
message: raw.message,
|
|
50
|
+
stack: raw.stack
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
return encoded;
|
|
54
|
+
});
|
|
55
|
+
},
|
|
56
|
+
parse(text) {
|
|
57
|
+
return JSON.parse(text, (_key, value) => {
|
|
58
|
+
if (!isTagged(value)) return value;
|
|
59
|
+
switch (value[TAG]) {
|
|
60
|
+
case "undefined": return;
|
|
61
|
+
case "BigInt": return BigInt(value.value);
|
|
62
|
+
case "Date": return new Date(value.value);
|
|
63
|
+
case "Map": return new Map(value.value);
|
|
64
|
+
case "Set": return new Set(value.value);
|
|
65
|
+
case "Error": {
|
|
66
|
+
const e = value.value;
|
|
67
|
+
const err = new Error(e.message);
|
|
68
|
+
err.name = e.name;
|
|
69
|
+
if (e.stack) err.stack = e.stack;
|
|
70
|
+
return err;
|
|
71
|
+
}
|
|
72
|
+
default: return value;
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
//#endregion
|
|
78
|
+
//#region src/internal.ts
|
|
79
|
+
/**
|
|
80
|
+
* # Internals
|
|
81
|
+
*
|
|
82
|
+
* Dependency-free helpers shared across the engine: id generation, the
|
|
83
|
+
* backoff-with-jitter formula, an async sleep, a shallow merge, and the identity
|
|
84
|
+
* `logWith` (so `@ayepi/log` is injected, never imported).
|
|
85
|
+
*
|
|
86
|
+
* @module
|
|
87
|
+
*/
|
|
88
|
+
/** A v4 UUID. Thin wrapper over `node:crypto` so callers don't import it directly. */
|
|
89
|
+
const uuid = () => randomUUID();
|
|
90
|
+
/** Resolve after `ms` (an unref'd timer, so it never keeps the process alive). */
|
|
91
|
+
function sleep(ms) {
|
|
92
|
+
return new Promise((resolve) => {
|
|
93
|
+
setTimeout(resolve, ms).unref?.();
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
/** Shallow merge of two objects into a new object (`b` wins on collisions). */
|
|
97
|
+
function merge(a, b) {
|
|
98
|
+
return {
|
|
99
|
+
...a ?? {},
|
|
100
|
+
...b ?? {}
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
/** The default {@link LogWith}: runs `inner` with no added context. */
|
|
104
|
+
const identityLogWith = (_add, inner) => inner();
|
|
105
|
+
//#endregion
|
|
106
|
+
//#region src/memory.ts
|
|
107
|
+
/**
|
|
108
|
+
* # In-memory backend
|
|
109
|
+
*
|
|
110
|
+
* A zero-dependency implementation of all three {@link Backend} ports that
|
|
111
|
+
* **simulates the distributed protocol** — visibility-timeout leases with
|
|
112
|
+
* heartbeat-driven redelivery, TTL'd store with an atomic `setIfNotExists`, and
|
|
113
|
+
* in-process fanout. Share one backend between several `createWork` instances to model
|
|
114
|
+
* a multi-pod deployment in tests, and inject `now` to drive time deterministically.
|
|
115
|
+
*
|
|
116
|
+
* The {@link memoryQueue} can additionally be **file-backed** (`file: './work-queue.json'`)
|
|
117
|
+
* so pending work survives a process restart — single-process durability without standing up
|
|
118
|
+
* Redis/SQS. State is written atomically (temp file + rename) after every mutation; on startup
|
|
119
|
+
* the file is reloaded and any in-flight (leased) item is redelivered, since the worker that
|
|
120
|
+
* held its lease is gone.
|
|
121
|
+
*
|
|
122
|
+
* @module
|
|
123
|
+
*/
|
|
124
|
+
/** The default {@link QueueFsLike}, backed by synchronous `node:fs`. */
|
|
125
|
+
const nodeQueueFs = {
|
|
126
|
+
readFile: (p) => existsSync(p) ? readFileSync(p, "utf8") : void 0,
|
|
127
|
+
writeFile: (p, d) => writeFileSync(p, d),
|
|
128
|
+
rename: (a, b) => renameSync(a, b),
|
|
129
|
+
mkdir: (p) => void mkdirSync(p, { recursive: true })
|
|
130
|
+
};
|
|
131
|
+
/**
|
|
132
|
+
* In-process {@link PubSub} (the `localBroker` shape). Share one instance across
|
|
133
|
+
* engines to fan a publish out to every "pod".
|
|
134
|
+
*/
|
|
135
|
+
function memoryPubSub() {
|
|
136
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
137
|
+
return {
|
|
138
|
+
publish(m) {
|
|
139
|
+
for (const l of [...listeners]) l(m);
|
|
140
|
+
},
|
|
141
|
+
subscribe(l) {
|
|
142
|
+
listeners.add(l);
|
|
143
|
+
return () => void listeners.delete(l);
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* In-memory {@link Store} with lazy TTL expiry and an atomic {@link Store.setIfNotExists}.
|
|
149
|
+
* The compare-and-set every distributed claim relies on.
|
|
150
|
+
*/
|
|
151
|
+
function memoryStore(opts = {}) {
|
|
152
|
+
const now = opts.now ?? Date.now;
|
|
153
|
+
const map = /* @__PURE__ */ new Map();
|
|
154
|
+
const live = (key) => {
|
|
155
|
+
const e = map.get(key);
|
|
156
|
+
if (!e) return;
|
|
157
|
+
if (e.expires !== void 0 && e.expires <= now()) {
|
|
158
|
+
map.delete(key);
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
return e;
|
|
162
|
+
};
|
|
163
|
+
return {
|
|
164
|
+
get: (key) => live(key)?.value,
|
|
165
|
+
set: (key, value, ttl) => void map.set(key, {
|
|
166
|
+
value,
|
|
167
|
+
expires: ttl ? now() + ttl : void 0
|
|
168
|
+
}),
|
|
169
|
+
delete: (key) => void map.delete(key),
|
|
170
|
+
setIfNotExists: (key, value, ttl) => {
|
|
171
|
+
if (live(key)) return false;
|
|
172
|
+
map.set(key, {
|
|
173
|
+
value,
|
|
174
|
+
expires: ttl ? now() + ttl : void 0
|
|
175
|
+
});
|
|
176
|
+
return true;
|
|
177
|
+
},
|
|
178
|
+
increment: (key, by, ttl) => {
|
|
179
|
+
const cur = Number(live(key)?.value ?? "0") + by;
|
|
180
|
+
map.set(key, {
|
|
181
|
+
value: String(cur),
|
|
182
|
+
expires: ttl ? now() + ttl : void 0
|
|
183
|
+
});
|
|
184
|
+
return cur;
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* In-memory {@link Queue} with visibility-timeout leasing. `pop` first **reclaims**
|
|
190
|
+
* items whose lease expired (a dead worker → redelivery, `attempt++`), then leases
|
|
191
|
+
* fresh visible items. `ack`/`heartbeat`/`fail` are token-gated, so a stale worker
|
|
192
|
+
* whose lease already lapsed cannot ack work another worker now owns.
|
|
193
|
+
*
|
|
194
|
+
* Pass `file` to make it **durable**: state is reloaded on construction and rewritten
|
|
195
|
+
* atomically after every mutation. A heartbeat is *not* persisted (lease expiry is
|
|
196
|
+
* reset on reload anyway), so steady-state heartbeating doesn't touch the disk.
|
|
197
|
+
*/
|
|
198
|
+
function memoryQueue(opts = {}) {
|
|
199
|
+
const now = opts.now ?? Date.now;
|
|
200
|
+
const items = [];
|
|
201
|
+
const dead = [];
|
|
202
|
+
const find = (token) => items.find((i) => i.leaseToken === token);
|
|
203
|
+
const file = opts.file;
|
|
204
|
+
const fs = opts.fs ?? nodeQueueFs;
|
|
205
|
+
const tmpFile = file !== void 0 ? `${file}.tmp` : "";
|
|
206
|
+
let dirEnsured = false;
|
|
207
|
+
const report = (err) => {
|
|
208
|
+
try {
|
|
209
|
+
opts.onError?.(err);
|
|
210
|
+
} catch {}
|
|
211
|
+
};
|
|
212
|
+
const persist = () => {
|
|
213
|
+
if (file === void 0) return;
|
|
214
|
+
try {
|
|
215
|
+
if (!dirEnsured) {
|
|
216
|
+
const dir = dirname(file);
|
|
217
|
+
if (dir !== ".") fs.mkdir(dir);
|
|
218
|
+
dirEnsured = true;
|
|
219
|
+
}
|
|
220
|
+
fs.writeFile(tmpFile, JSON.stringify({
|
|
221
|
+
items,
|
|
222
|
+
dead
|
|
223
|
+
}));
|
|
224
|
+
fs.rename(tmpFile, file);
|
|
225
|
+
} catch (err) {
|
|
226
|
+
report(err);
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
const load = () => {
|
|
230
|
+
if (file === void 0) return;
|
|
231
|
+
let raw;
|
|
232
|
+
try {
|
|
233
|
+
raw = fs.readFile(file);
|
|
234
|
+
} catch (err) {
|
|
235
|
+
report(err);
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
if (raw === void 0) return;
|
|
239
|
+
try {
|
|
240
|
+
const parsed = JSON.parse(raw);
|
|
241
|
+
for (const i of parsed.items ?? []) {
|
|
242
|
+
if (i.leaseToken !== void 0) {
|
|
243
|
+
i.leaseToken = void 0;
|
|
244
|
+
i.leaseUntil = void 0;
|
|
245
|
+
i.attempt += 1;
|
|
246
|
+
i.visibleAt = now();
|
|
247
|
+
}
|
|
248
|
+
items.push(i);
|
|
249
|
+
}
|
|
250
|
+
for (const d of parsed.dead ?? []) dead.push(d);
|
|
251
|
+
} catch (err) {
|
|
252
|
+
report(err);
|
|
253
|
+
}
|
|
254
|
+
};
|
|
255
|
+
load();
|
|
256
|
+
return {
|
|
257
|
+
dead,
|
|
258
|
+
size: () => items.length,
|
|
259
|
+
push(body, o) {
|
|
260
|
+
if (o?.dedupeKey && items.some((i) => i.dedupeKey === o.dedupeKey)) return;
|
|
261
|
+
items.push({
|
|
262
|
+
body,
|
|
263
|
+
visibleAt: now() + (o?.delay ?? 0),
|
|
264
|
+
attempt: 0,
|
|
265
|
+
dedupeKey: o?.dedupeKey
|
|
266
|
+
});
|
|
267
|
+
persist();
|
|
268
|
+
},
|
|
269
|
+
pop(max, visibility) {
|
|
270
|
+
const t = now();
|
|
271
|
+
let changed = false;
|
|
272
|
+
for (const i of items) if (i.leaseToken && i.leaseUntil !== void 0 && i.leaseUntil <= t) {
|
|
273
|
+
i.leaseToken = void 0;
|
|
274
|
+
i.leaseUntil = void 0;
|
|
275
|
+
i.attempt += 1;
|
|
276
|
+
changed = true;
|
|
277
|
+
}
|
|
278
|
+
const out = [];
|
|
279
|
+
for (const i of items) {
|
|
280
|
+
if (out.length >= max) break;
|
|
281
|
+
if (i.leaseToken || i.visibleAt > t) continue;
|
|
282
|
+
const token = uuid();
|
|
283
|
+
i.leaseToken = token;
|
|
284
|
+
i.leaseUntil = t + visibility;
|
|
285
|
+
out.push({
|
|
286
|
+
body: i.body,
|
|
287
|
+
handle: token,
|
|
288
|
+
attempt: i.attempt + 1
|
|
289
|
+
});
|
|
290
|
+
changed = true;
|
|
291
|
+
}
|
|
292
|
+
if (changed) persist();
|
|
293
|
+
return out;
|
|
294
|
+
},
|
|
295
|
+
heartbeat(p, visibility) {
|
|
296
|
+
const i = find(p.handle);
|
|
297
|
+
if (i) i.leaseUntil = now() + visibility;
|
|
298
|
+
},
|
|
299
|
+
ack(p) {
|
|
300
|
+
const idx = items.findIndex((i) => i.leaseToken === p.handle);
|
|
301
|
+
if (idx >= 0) {
|
|
302
|
+
items.splice(idx, 1);
|
|
303
|
+
persist();
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
fail(p, delay) {
|
|
307
|
+
const i = find(p.handle);
|
|
308
|
+
if (!i) return;
|
|
309
|
+
i.leaseToken = void 0;
|
|
310
|
+
i.leaseUntil = void 0;
|
|
311
|
+
i.attempt = p.attempt;
|
|
312
|
+
i.visibleAt = now() + (delay ?? 0);
|
|
313
|
+
persist();
|
|
314
|
+
},
|
|
315
|
+
deadLetter(body, error) {
|
|
316
|
+
dead.push({
|
|
317
|
+
body,
|
|
318
|
+
error
|
|
319
|
+
});
|
|
320
|
+
persist();
|
|
321
|
+
}
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* The three in-memory ports together, sharing one clock. The default backend when
|
|
326
|
+
* `createWork()` is called with no ports.
|
|
327
|
+
*
|
|
328
|
+
* @example A two-pod test on one shared backend:
|
|
329
|
+
* ```ts
|
|
330
|
+
* const backend = memoryBackend()
|
|
331
|
+
* const podA = createWork({ ...backend, work: [add] })
|
|
332
|
+
* const podB = createWork({ ...backend, work: [add] })
|
|
333
|
+
* ```
|
|
334
|
+
*
|
|
335
|
+
* @example A single durable process — pending work survives a restart:
|
|
336
|
+
* ```ts
|
|
337
|
+
* const backend = memoryBackend({ queue: { file: './work-queue.json' } })
|
|
338
|
+
* const work = createWork({ ...backend, work: [add] })
|
|
339
|
+
* ```
|
|
340
|
+
*/
|
|
341
|
+
function memoryBackend(opts = {}) {
|
|
342
|
+
return {
|
|
343
|
+
queue: memoryQueue({
|
|
344
|
+
now: opts.now,
|
|
345
|
+
...opts.queue
|
|
346
|
+
}),
|
|
347
|
+
pubsub: memoryPubSub(),
|
|
348
|
+
store: memoryStore(opts)
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
//#endregion
|
|
352
|
+
//#region src/errors.ts
|
|
353
|
+
/**
|
|
354
|
+
* Throw from a work handler to **defer** the item to a future time (a reschedule, not a failure).
|
|
355
|
+
*
|
|
356
|
+
* ```ts
|
|
357
|
+
* defineWork('poll', async (input, ctx) => {
|
|
358
|
+
* if (!(await upstreamReady())) throw new WorkDelayError({ delay: 5 * 60_000 }); // retry in 5 min, attempt unchanged
|
|
359
|
+
* return doWork(input);
|
|
360
|
+
* });
|
|
361
|
+
* ```
|
|
362
|
+
*/
|
|
363
|
+
var WorkDelayError = class extends Error {
|
|
364
|
+
when;
|
|
365
|
+
constructor(when, message = "work deferred") {
|
|
366
|
+
super(message);
|
|
367
|
+
this.when = when;
|
|
368
|
+
this.name = "WorkDelayError";
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
//#endregion
|
|
372
|
+
//#region src/dependency.ts
|
|
373
|
+
/**
|
|
374
|
+
* # Dependencies
|
|
375
|
+
*
|
|
376
|
+
* A dependency is **itself a work item** ({@link DEPENDENCY_TYPE}) — so it lives on the
|
|
377
|
+
* durable queue and survives a service disruption like any other work. Its handler is
|
|
378
|
+
* **non-blocking**: each run reads the state of the works it waits `on`, and either
|
|
379
|
+
*
|
|
380
|
+
* - fires (queues its `queue` dependents, once, under a distributed {@link WorkContext.claim}), or
|
|
381
|
+
* - **re-queues itself** with a small delay to check again later.
|
|
382
|
+
*
|
|
383
|
+
* It never holds a worker slot waiting, so a backlog of dependencies can't starve other
|
|
384
|
+
* work. Build one with {@link dependency} and enqueue it like anything else — typically
|
|
385
|
+
* alongside the works it depends on:
|
|
386
|
+
*
|
|
387
|
+
* ```ts
|
|
388
|
+
* const a = stepA(), b = stepB()
|
|
389
|
+
* ctx.queue([a, b, dependency({ on: [a, b], queue: [finalize()], config: 'all-success' })])
|
|
390
|
+
* ```
|
|
391
|
+
*
|
|
392
|
+
* @module
|
|
393
|
+
*/
|
|
394
|
+
/** The built-in work type name for a dependency. */
|
|
395
|
+
const DEPENDENCY_TYPE = "@work/dependency";
|
|
396
|
+
/** Default re-check interval (ms). */
|
|
397
|
+
const DEFAULT_POLL = 1e3;
|
|
398
|
+
const TERMINAL = new Set([
|
|
399
|
+
"success",
|
|
400
|
+
"failed",
|
|
401
|
+
"dead"
|
|
402
|
+
]);
|
|
403
|
+
const isTerminal = (s) => s !== void 0 && TERMINAL.has(s.status);
|
|
404
|
+
const isSuccess = (s) => s?.status === "success";
|
|
405
|
+
/**
|
|
406
|
+
* Evaluate a {@link DependencyCondition} against the watched items' states. A missing
|
|
407
|
+
* state counts as "not yet done". Pure and JSON-driven, so every instance agrees.
|
|
408
|
+
*/
|
|
409
|
+
function conditionMet(condition, states) {
|
|
410
|
+
if (condition === "all-done") return states.every(isTerminal);
|
|
411
|
+
if (condition === "all-success") return states.every(isSuccess);
|
|
412
|
+
const pred = condition.of === "success" ? isSuccess : isTerminal;
|
|
413
|
+
return states.filter(pred).length >= condition.count;
|
|
414
|
+
}
|
|
415
|
+
const toId = (w) => typeof w === "string" ? w : w.id;
|
|
416
|
+
const toSerialized = (w) => ({
|
|
417
|
+
id: w.id,
|
|
418
|
+
type: w.type,
|
|
419
|
+
input: w.input
|
|
420
|
+
});
|
|
421
|
+
const rehydrate = (s) => ({
|
|
422
|
+
id: s.id,
|
|
423
|
+
type: s.type,
|
|
424
|
+
input: s.input
|
|
425
|
+
});
|
|
426
|
+
/** Build a {@link DEPENDENCY_TYPE} work item from a (re-usable) input — a fresh queue id, same key. */
|
|
427
|
+
const buildDependency = (input) => ({
|
|
428
|
+
id: uuid(),
|
|
429
|
+
type: DEPENDENCY_TYPE,
|
|
430
|
+
input
|
|
431
|
+
});
|
|
432
|
+
/**
|
|
433
|
+
* Build a dependency: when the works it waits `on` satisfy `config`, it queues its
|
|
434
|
+
* `queue` dependents (once). Enqueue it like any work.
|
|
435
|
+
*/
|
|
436
|
+
function dependency(opts) {
|
|
437
|
+
return buildDependency({
|
|
438
|
+
key: uuid(),
|
|
439
|
+
on: opts.on.map(toId),
|
|
440
|
+
queue: opts.queue.map(toSerialized),
|
|
441
|
+
config: opts.config ?? "all-success",
|
|
442
|
+
poll: opts.poll ?? DEFAULT_POLL,
|
|
443
|
+
deadline: opts.timeout !== void 0 ? Date.now() + opts.timeout : void 0
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* The non-blocking handler for {@link DEPENDENCY_TYPE}: check once, then fire or
|
|
448
|
+
* re-queue. Registered automatically by every work system. Remembers terminal statuses
|
|
449
|
+
* (`resolved`) so it neither re-reads settled works nor treats an evicted state as a
|
|
450
|
+
* failure.
|
|
451
|
+
*/
|
|
452
|
+
const dependencyHandler = async (input, ctx) => {
|
|
453
|
+
const resolved = { ...input.resolved };
|
|
454
|
+
const unknownIds = input.on.filter((id) => !(id in resolved));
|
|
455
|
+
const fresh = await ctx.states(unknownIds);
|
|
456
|
+
fresh.forEach((s, i) => {
|
|
457
|
+
if (s && TERMINAL.has(s.status)) resolved[unknownIds[i]] = s.status;
|
|
458
|
+
});
|
|
459
|
+
const freshById = new Map(unknownIds.map((id, i) => [id, fresh[i]]));
|
|
460
|
+
const states = input.on.map((id) => resolved[id] ? { status: resolved[id] } : freshById.get(id));
|
|
461
|
+
if (conditionMet(input.config, states)) {
|
|
462
|
+
if (await ctx.claim(`dep:${input.key}:fired`)) ctx.queue(input.queue.map(rehydrate));
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
if (input.deadline !== void 0 && Date.now() >= input.deadline) throw new Error(`dependency: timed out waiting for [${input.on.join(", ")}]`);
|
|
466
|
+
ctx.queue(buildDependency({
|
|
467
|
+
...input,
|
|
468
|
+
resolved
|
|
469
|
+
}), { delay: input.poll });
|
|
470
|
+
};
|
|
471
|
+
//#endregion
|
|
472
|
+
//#region src/schedule.ts
|
|
473
|
+
const MS_PER_MINUTE = 6e4;
|
|
474
|
+
/** Cap the forward scan so a never-matching expression can't loop forever (~1 year). */
|
|
475
|
+
const MAX_SCAN_MINUTES = 366 * 24 * 60;
|
|
476
|
+
/** `[min, max]` inclusive range for each of the five fields. */
|
|
477
|
+
const FIELD_BOUNDS = [
|
|
478
|
+
[0, 59],
|
|
479
|
+
[0, 23],
|
|
480
|
+
[1, 31],
|
|
481
|
+
[1, 12],
|
|
482
|
+
[0, 6]
|
|
483
|
+
];
|
|
484
|
+
/** Expand one cron field — `*`, a number, an `a-b` range, a `<range>/<step>`, or a comma list — into the set of matching numbers. */
|
|
485
|
+
function parseField(field, min, max) {
|
|
486
|
+
const out = /* @__PURE__ */ new Set();
|
|
487
|
+
for (const part of field.split(",")) {
|
|
488
|
+
const [rangePart, stepPart] = part.split("/");
|
|
489
|
+
const step = stepPart ? Number(stepPart) : 1;
|
|
490
|
+
if (!Number.isInteger(step) || step < 1) throw new Error(`cron: bad step in "${part}"`);
|
|
491
|
+
let lo = min;
|
|
492
|
+
let hi = max;
|
|
493
|
+
if (rangePart !== "*" && rangePart !== "") {
|
|
494
|
+
const [a, b] = rangePart.split("-");
|
|
495
|
+
lo = Number(a);
|
|
496
|
+
hi = b !== void 0 ? Number(b) : lo;
|
|
497
|
+
if (!Number.isInteger(lo) || !Number.isInteger(hi) || lo < min || hi > max || lo > hi) throw new Error(`cron: bad range "${part}"`);
|
|
498
|
+
}
|
|
499
|
+
for (let v = lo; v <= hi; v += step) out.add(v);
|
|
500
|
+
}
|
|
501
|
+
return out;
|
|
502
|
+
}
|
|
503
|
+
/** Parse a 5-field cron expression (`min hour dom mon dow`). */
|
|
504
|
+
function parseCron(expr) {
|
|
505
|
+
const fields = expr.trim().split(/\s+/);
|
|
506
|
+
if (fields.length !== 5) throw new Error(`cron: expected 5 fields, got ${fields.length} in "${expr}"`);
|
|
507
|
+
const [minute, hour, dom, month, dow] = fields.map((f, i) => parseField(f, FIELD_BOUNDS[i][0], FIELD_BOUNDS[i][1]));
|
|
508
|
+
return {
|
|
509
|
+
minute,
|
|
510
|
+
hour,
|
|
511
|
+
dom,
|
|
512
|
+
month,
|
|
513
|
+
dow,
|
|
514
|
+
domRestricted: fields[2] !== "*",
|
|
515
|
+
dowRestricted: fields[4] !== "*"
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
/** Does `date` (local time) satisfy the cron fields? Standard dom/dow OR semantics when both are restricted. */
|
|
519
|
+
function matches(c, date) {
|
|
520
|
+
if (!c.minute.has(date.getMinutes()) || !c.hour.has(date.getHours()) || !c.month.has(date.getMonth() + 1)) return false;
|
|
521
|
+
const domOk = c.dom.has(date.getDate());
|
|
522
|
+
const dowOk = c.dow.has(date.getDay());
|
|
523
|
+
if (c.domRestricted && c.dowRestricted) return domOk || dowOk;
|
|
524
|
+
if (c.domRestricted) return domOk;
|
|
525
|
+
if (c.dowRestricted) return dowOk;
|
|
526
|
+
return true;
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* The next epoch-ms strictly after `fromMs` that matches `expr`, or `undefined` if
|
|
530
|
+
* none within ~a year. Minute-granular (cron's resolution).
|
|
531
|
+
*/
|
|
532
|
+
function nextAfter(expr, fromMs) {
|
|
533
|
+
const c = parseCron(expr);
|
|
534
|
+
const start = /* @__PURE__ */ new Date(Math.floor(fromMs / MS_PER_MINUTE) * MS_PER_MINUTE + MS_PER_MINUTE);
|
|
535
|
+
start.setSeconds(0, 0);
|
|
536
|
+
for (let i = 0; i < MAX_SCAN_MINUTES; i++) {
|
|
537
|
+
const candidate = new Date(start.getTime() + i * MS_PER_MINUTE);
|
|
538
|
+
if (matches(c, candidate)) return candidate.getTime();
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
const unref$1 = (t) => void t.unref?.();
|
|
542
|
+
/** Compute the next fire time (epoch ms) for a schedule, or `undefined` to stop. */
|
|
543
|
+
function computeNext(config, from) {
|
|
544
|
+
if (config.cron !== void 0) return nextAfter(config.cron, from);
|
|
545
|
+
if (config.next) {
|
|
546
|
+
const r = config.next(from);
|
|
547
|
+
if (r === void 0 || r === null) return;
|
|
548
|
+
return r instanceof Date ? r.getTime() : r;
|
|
549
|
+
}
|
|
550
|
+
throw new Error(`schedule "${config.name}": provide either "cron" or "next"`);
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* Start a schedule's tick loop. Returns a cancel function. One instance fires per
|
|
554
|
+
* occurrence (claimed via a `setNX` lease keyed by the fire's second-bucket).
|
|
555
|
+
*/
|
|
556
|
+
function startSchedule(config, deps) {
|
|
557
|
+
let cancelled = false;
|
|
558
|
+
let nextAt = computeNext(config, deps.now());
|
|
559
|
+
let timer;
|
|
560
|
+
const tick = async () => {
|
|
561
|
+
if (cancelled) return;
|
|
562
|
+
const t = deps.now();
|
|
563
|
+
if (nextAt !== void 0 && t >= nextAt) {
|
|
564
|
+
const bucket = Math.floor(nextAt / 1e3);
|
|
565
|
+
if (await deps.store.setIfNotExists(`${deps.prefix}sched:${config.name}:${bucket}`, "1", deps.leaseTtl)) {
|
|
566
|
+
const inst = config.run();
|
|
567
|
+
if (inst) deps.enqueueRaw(inst.type, inst.input);
|
|
568
|
+
}
|
|
569
|
+
nextAt = computeNext(config, deps.now());
|
|
570
|
+
}
|
|
571
|
+
if (!cancelled) {
|
|
572
|
+
timer = setTimeout(() => void tick(), deps.tick);
|
|
573
|
+
unref$1(timer);
|
|
574
|
+
}
|
|
575
|
+
};
|
|
576
|
+
timer = setTimeout(() => void tick(), 0);
|
|
577
|
+
unref$1(timer);
|
|
578
|
+
return () => {
|
|
579
|
+
cancelled = true;
|
|
580
|
+
clearTimeout(timer);
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
//#endregion
|
|
584
|
+
//#region src/stats.ts
|
|
585
|
+
/**
|
|
586
|
+
* # Work stats — per-type metrics over `@ayepi/core`'s {@link Metrics}
|
|
587
|
+
*
|
|
588
|
+
* A thin recorder the engine drives at each lifecycle transition. It owns a
|
|
589
|
+
* {@link Metrics} registry (bring your own via {@link WorkSystemOptions.metrics} to enable
|
|
590
|
+
* quantiles or share one across systems) and records, **labelled by work type**:
|
|
591
|
+
*
|
|
592
|
+
* - **counters** — `queued` / `started` / `succeeded` / `failed` / `retried` / `deferred` / `rescheduled`
|
|
593
|
+
* - **gauges** — live `active` / `pending` / `running`, the `peak_active` high-water mark, and
|
|
594
|
+
* `last_*_at` transition timestamps (epoch ms)
|
|
595
|
+
* - **summaries** (ms) — `wait_time` (poll lag = `runAt − startAt`), `total_time` (end-to-end
|
|
596
|
+
* `endAt − queueAt`), `success_time` / `error_time` (run duration), `delay_time` /
|
|
597
|
+
* `reschedule_time` (re-queue horizons), and `attempts` (tries at terminal)
|
|
598
|
+
*
|
|
599
|
+
* Read it via {@link WorkSystem.metrics} (`list()` / `get()` / `subscribe()`) or the
|
|
600
|
+
* {@link WorkSystem.stats} list snapshot.
|
|
601
|
+
*
|
|
602
|
+
* @module
|
|
603
|
+
*/
|
|
604
|
+
/** Metric names (dotted; `formatPrometheus` sanitizes the dots to underscores). Exported so consumers can reference series by name. */
|
|
605
|
+
const WORK_METRICS = {
|
|
606
|
+
queued: "work.queued",
|
|
607
|
+
started: "work.started",
|
|
608
|
+
succeeded: "work.succeeded",
|
|
609
|
+
failed: "work.failed",
|
|
610
|
+
retried: "work.retried",
|
|
611
|
+
deferred: "work.deferred",
|
|
612
|
+
rescheduled: "work.rescheduled",
|
|
613
|
+
active: "work.active",
|
|
614
|
+
pending: "work.pending",
|
|
615
|
+
running: "work.running",
|
|
616
|
+
peak: "work.peak_active",
|
|
617
|
+
lastQueued: "work.last_queued_at",
|
|
618
|
+
lastStarted: "work.last_started_at",
|
|
619
|
+
lastSucceeded: "work.last_succeeded_at",
|
|
620
|
+
lastFailed: "work.last_failed_at",
|
|
621
|
+
waitTime: "work.wait_time",
|
|
622
|
+
totalTime: "work.total_time",
|
|
623
|
+
successTime: "work.success_time",
|
|
624
|
+
errorTime: "work.error_time",
|
|
625
|
+
delayTime: "work.delay_time",
|
|
626
|
+
rescheduleTime: "work.reschedule_time",
|
|
627
|
+
attempts: "work.attempts"
|
|
628
|
+
};
|
|
629
|
+
/** Create the work stats recorder over `metrics` (a fresh registry by default). */
|
|
630
|
+
const createWorkStats = (metrics = createMetrics$1()) => {
|
|
631
|
+
const cache = /* @__PURE__ */ new Map();
|
|
632
|
+
const ms = { unit: "ms" };
|
|
633
|
+
const handles = (type) => {
|
|
634
|
+
let h = cache.get(type);
|
|
635
|
+
if (h) return h;
|
|
636
|
+
const l = { type };
|
|
637
|
+
h = {
|
|
638
|
+
queued: metrics.counter(WORK_METRICS.queued, l, { description: "items enqueued" }),
|
|
639
|
+
started: metrics.counter(WORK_METRICS.started, l, { description: "runs begun" }),
|
|
640
|
+
succeeded: metrics.counter(WORK_METRICS.succeeded, l, { description: "runs that succeeded" }),
|
|
641
|
+
failed: metrics.counter(WORK_METRICS.failed, l, { description: "items dead-lettered" }),
|
|
642
|
+
retried: metrics.counter(WORK_METRICS.retried, l, { description: "attempt-advancing retries" }),
|
|
643
|
+
deferred: metrics.counter(WORK_METRICS.deferred, l, { description: "handler/classifier reschedules" }),
|
|
644
|
+
rescheduled: metrics.counter(WORK_METRICS.rescheduled, l, { description: "early-arrival re-pushes" }),
|
|
645
|
+
active: metrics.gauge(WORK_METRICS.active, l, { description: "items in flight (pending + running)" }),
|
|
646
|
+
pending: metrics.gauge(WORK_METRICS.pending, l, { description: "items admitted, awaiting a doer slot" }),
|
|
647
|
+
running: metrics.gauge(WORK_METRICS.running, l, { description: "items executing" }),
|
|
648
|
+
peak: metrics.gauge(WORK_METRICS.peak, l, { description: "high-water mark of in-flight items" }),
|
|
649
|
+
lastQueued: metrics.gauge(WORK_METRICS.lastQueued, l, {
|
|
650
|
+
...ms,
|
|
651
|
+
description: "last enqueue time (epoch ms)"
|
|
652
|
+
}),
|
|
653
|
+
lastStarted: metrics.gauge(WORK_METRICS.lastStarted, l, {
|
|
654
|
+
...ms,
|
|
655
|
+
description: "last start time (epoch ms)"
|
|
656
|
+
}),
|
|
657
|
+
lastSucceeded: metrics.gauge(WORK_METRICS.lastSucceeded, l, {
|
|
658
|
+
...ms,
|
|
659
|
+
description: "last success time (epoch ms)"
|
|
660
|
+
}),
|
|
661
|
+
lastFailed: metrics.gauge(WORK_METRICS.lastFailed, l, {
|
|
662
|
+
...ms,
|
|
663
|
+
description: "last dead-letter time (epoch ms)"
|
|
664
|
+
}),
|
|
665
|
+
waitTime: metrics.summary(WORK_METRICS.waitTime, l, {
|
|
666
|
+
...ms,
|
|
667
|
+
description: "poll lag: runAt - startAt"
|
|
668
|
+
}),
|
|
669
|
+
totalTime: metrics.summary(WORK_METRICS.totalTime, l, {
|
|
670
|
+
...ms,
|
|
671
|
+
description: "end-to-end: endAt - queueAt"
|
|
672
|
+
}),
|
|
673
|
+
successTime: metrics.summary(WORK_METRICS.successTime, l, {
|
|
674
|
+
...ms,
|
|
675
|
+
description: "successful run duration"
|
|
676
|
+
}),
|
|
677
|
+
errorTime: metrics.summary(WORK_METRICS.errorTime, l, {
|
|
678
|
+
...ms,
|
|
679
|
+
description: "failed-to-terminal run duration"
|
|
680
|
+
}),
|
|
681
|
+
delayTime: metrics.summary(WORK_METRICS.delayTime, l, {
|
|
682
|
+
...ms,
|
|
683
|
+
description: "reschedule horizon"
|
|
684
|
+
}),
|
|
685
|
+
rescheduleTime: metrics.summary(WORK_METRICS.rescheduleTime, l, {
|
|
686
|
+
...ms,
|
|
687
|
+
description: "early-arrival re-push horizon"
|
|
688
|
+
}),
|
|
689
|
+
attempts: metrics.summary(WORK_METRICS.attempts, l, {
|
|
690
|
+
unit: "count",
|
|
691
|
+
description: "attempts used at terminal"
|
|
692
|
+
})
|
|
693
|
+
};
|
|
694
|
+
cache.set(type, h);
|
|
695
|
+
return h;
|
|
696
|
+
};
|
|
697
|
+
return {
|
|
698
|
+
metrics,
|
|
699
|
+
queued: (type, at) => {
|
|
700
|
+
const h = handles(type);
|
|
701
|
+
h.queued.inc();
|
|
702
|
+
h.lastQueued.set(at);
|
|
703
|
+
},
|
|
704
|
+
claimed: (type) => {
|
|
705
|
+
const h = handles(type);
|
|
706
|
+
h.pending.add(1);
|
|
707
|
+
h.active.add(1);
|
|
708
|
+
h.peak.max(h.active.value());
|
|
709
|
+
},
|
|
710
|
+
started: (type, at, pollLagMs) => {
|
|
711
|
+
const h = handles(type);
|
|
712
|
+
h.started.inc();
|
|
713
|
+
h.pending.add(-1);
|
|
714
|
+
h.running.add(1);
|
|
715
|
+
h.waitTime.observe(Math.max(0, pollLagMs));
|
|
716
|
+
h.lastStarted.set(at);
|
|
717
|
+
},
|
|
718
|
+
released: (type, wasRunning) => {
|
|
719
|
+
const h = handles(type);
|
|
720
|
+
(wasRunning ? h.running : h.pending).add(-1);
|
|
721
|
+
h.active.add(-1);
|
|
722
|
+
},
|
|
723
|
+
succeeded: (type, at, runMs, totalMs, attempt) => {
|
|
724
|
+
const h = handles(type);
|
|
725
|
+
h.succeeded.inc();
|
|
726
|
+
h.successTime.observe(Math.max(0, runMs));
|
|
727
|
+
h.totalTime.observe(Math.max(0, totalMs));
|
|
728
|
+
h.attempts.observe(attempt);
|
|
729
|
+
h.lastSucceeded.set(at);
|
|
730
|
+
},
|
|
731
|
+
failed: (type, at, runMs, totalMs, attempt) => {
|
|
732
|
+
const h = handles(type);
|
|
733
|
+
h.failed.inc();
|
|
734
|
+
if (runMs !== void 0) h.errorTime.observe(Math.max(0, runMs));
|
|
735
|
+
h.totalTime.observe(Math.max(0, totalMs));
|
|
736
|
+
h.attempts.observe(attempt);
|
|
737
|
+
h.lastFailed.set(at);
|
|
738
|
+
},
|
|
739
|
+
retried: (type) => void handles(type).retried.inc(),
|
|
740
|
+
deferred: (type, delayMs) => {
|
|
741
|
+
const h = handles(type);
|
|
742
|
+
h.deferred.inc();
|
|
743
|
+
h.delayTime.observe(Math.max(0, delayMs));
|
|
744
|
+
},
|
|
745
|
+
rescheduled: (type, delayMs) => {
|
|
746
|
+
const h = handles(type);
|
|
747
|
+
h.rescheduled.inc();
|
|
748
|
+
h.rescheduleTime.observe(Math.max(0, delayMs));
|
|
749
|
+
}
|
|
750
|
+
};
|
|
751
|
+
};
|
|
752
|
+
//#endregion
|
|
753
|
+
//#region src/types.ts
|
|
754
|
+
/**
|
|
755
|
+
* # Types — the type-safe definition surface
|
|
756
|
+
*
|
|
757
|
+
* Defining a work type with {@link defineWork} yields a **callable builder**: call it
|
|
758
|
+
* with the work's exact input and you get a type-checked, queueable {@link Work} (with
|
|
759
|
+
* a build-time id) that also carries its output type. {@link createWork} takes a
|
|
760
|
+
* `const` tuple of builders and produces a {@link WorkSystem} whose `enqueue` is fully
|
|
761
|
+
* checked — by instance (`enqueue(add({ a, b }))`) or by name (`enqueue('add', { a, b })`).
|
|
762
|
+
*
|
|
763
|
+
* The **group result type** is the union of every registered work's non-void output
|
|
764
|
+
* ({@link GroupResult}). All durations are **milliseconds**.
|
|
765
|
+
*
|
|
766
|
+
* @module
|
|
767
|
+
*/
|
|
768
|
+
const build = (name, input) => ({
|
|
769
|
+
id: uuid(),
|
|
770
|
+
type: name,
|
|
771
|
+
input
|
|
772
|
+
});
|
|
773
|
+
/**
|
|
774
|
+
* Define a work type. Returns a {@link WorkBuilder} — a function that builds queueable
|
|
775
|
+
* instances — typed by its input `I` and output `O`.
|
|
776
|
+
*/
|
|
777
|
+
function defineWork(name, handler, opts = {}) {
|
|
778
|
+
return Object.assign((input) => build(name, input), {
|
|
779
|
+
type: name,
|
|
780
|
+
def: {
|
|
781
|
+
name,
|
|
782
|
+
handler,
|
|
783
|
+
options: opts
|
|
784
|
+
}
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
/**
|
|
788
|
+
* Define a **batched** work type. Items still enqueue, retry, prioritize, and join
|
|
789
|
+
* groups individually, but execute together via {@link BatchConfig.run} once `size`
|
|
790
|
+
* accumulate or `maxWait` ms elapse — so each `.result()` resolves to its aligned
|
|
791
|
+
* output. The per-type {@link WorkOptions.doer} governs how many *batches* run at once.
|
|
792
|
+
*/
|
|
793
|
+
function defineBatchWork(name, config) {
|
|
794
|
+
const { size, maxWait, run, ...options } = config;
|
|
795
|
+
const batch = {
|
|
796
|
+
size,
|
|
797
|
+
maxWait,
|
|
798
|
+
run
|
|
799
|
+
};
|
|
800
|
+
const handler = async (input) => (await run([input]))[0];
|
|
801
|
+
return Object.assign((input) => build(name, input), {
|
|
802
|
+
type: name,
|
|
803
|
+
def: {
|
|
804
|
+
name,
|
|
805
|
+
handler,
|
|
806
|
+
options,
|
|
807
|
+
batch
|
|
808
|
+
}
|
|
809
|
+
});
|
|
810
|
+
}
|
|
811
|
+
//#endregion
|
|
812
|
+
//#region src/engine.ts
|
|
813
|
+
/**
|
|
814
|
+
* # Engine
|
|
815
|
+
*
|
|
816
|
+
* `createWork` ties the ports together into a running {@link WorkSystem}: a typed
|
|
817
|
+
* registry, a **doer**-driven worker loop (the doer decides how many items to pull and
|
|
818
|
+
* which to run next), per-item heartbeats, retry by **re-enqueue**, group linking +
|
|
819
|
+
* result resolution, batching, distributed wait handles, a built-in non-blocking
|
|
820
|
+
* dependency type, and scheduling. A `skipQueue` work runs in-process (doer + retries,
|
|
821
|
+
* no queue/store/heartbeat). Distributed coordination uses the store's
|
|
822
|
+
* `setIfNotExists`/`increment` atoms and pub/sub fanout, so the same logic runs on one
|
|
823
|
+
* process or a fleet.
|
|
824
|
+
*
|
|
825
|
+
* @module
|
|
826
|
+
*/
|
|
827
|
+
/** Default idle poll interval (ms). */
|
|
828
|
+
const DEFAULT_POLL_INTERVAL = 1e3;
|
|
829
|
+
/** Default lease/visibility timeout for a pulled item (ms). */
|
|
830
|
+
const DEFAULT_VISIBILITY = 3e4;
|
|
831
|
+
/** Heartbeat interval = visibility / this, so a lease is refreshed well before it lapses. */
|
|
832
|
+
const HEARTBEAT_DIVISOR = 3;
|
|
833
|
+
/** Default key namespace. */
|
|
834
|
+
const DEFAULT_PREFIX = "work:";
|
|
835
|
+
/** Hard cap on how many items one poll fetches, regardless of doer appetite. */
|
|
836
|
+
const POLL_BATCH_CAP = 512;
|
|
837
|
+
/** TTL for results / states / group bookkeeping (ms). */
|
|
838
|
+
const RESULT_TTL = 864e5;
|
|
839
|
+
/** TTL for the "someone is waiting" registry key (ms). */
|
|
840
|
+
const WAIT_TTL = 36e5;
|
|
841
|
+
/** How often a distributed waiter re-polls the store alongside pub/sub (ms). */
|
|
842
|
+
const WAIT_POLL = 250;
|
|
843
|
+
/** Grace before the orphan check, so an in-process awaiter can register first (ms). */
|
|
844
|
+
const UNHANDLED_GRACE = 100;
|
|
845
|
+
/** Re-delivery delay for a work type this instance doesn't know (ms). */
|
|
846
|
+
const UNKNOWN_TYPE_DELAY = 5e3;
|
|
847
|
+
/** A popped item this far (ms) before its `startAt` is put back rather than run (handles backends that can't honor a long delay). */
|
|
848
|
+
const SCHED_TOLERANCE = 1e3;
|
|
849
|
+
/** Scheduler tick interval (ms). */
|
|
850
|
+
const SCHED_TICK = 1e3;
|
|
851
|
+
/** TTL for a schedule's per-fire lease (ms). */
|
|
852
|
+
const SCHED_LEASE_TTL = 9e4;
|
|
853
|
+
/** Max time `stop()` waits for in-flight work to drain (ms). */
|
|
854
|
+
const STOP_DRAIN = 5e3;
|
|
855
|
+
/** A dependency dead-letters on timeout rather than retrying. */
|
|
856
|
+
const DEP_RETRY_ATTEMPTS = 1;
|
|
857
|
+
/** Default max messages redriven from the DLQ per idle poll. */
|
|
858
|
+
const REDRIVE_DEFAULT = 10;
|
|
859
|
+
const unref = (t) => void t.unref?.();
|
|
860
|
+
const errString = (err) => err instanceof Error ? `${err.name}: ${err.message}` : String(err);
|
|
861
|
+
/**
|
|
862
|
+
* Create a work system. Zero-config (`createWork()`) uses the bundled in-memory backend
|
|
863
|
+
* and an {@link unlimitedDoer}; pass `work: [...] as const` for a typed registry, a
|
|
864
|
+
* `doer` to govern concurrency, and/or `queue`/`pubsub`/`store` to go distributed.
|
|
865
|
+
*/
|
|
866
|
+
function createWork(opts = {}) {
|
|
867
|
+
const mem = opts.queue && opts.pubsub && opts.store ? null : memoryBackend({ now: opts.now });
|
|
868
|
+
const backend = {
|
|
869
|
+
queue: opts.queue ?? mem.queue,
|
|
870
|
+
pubsub: opts.pubsub ?? mem.pubsub,
|
|
871
|
+
store: opts.store ?? mem.store
|
|
872
|
+
};
|
|
873
|
+
const { pubsub, store } = backend;
|
|
874
|
+
const globalCodec = opts.codec ?? defaultCodec;
|
|
875
|
+
const globalDoer = opts.doer ?? unlimitedDoer$1();
|
|
876
|
+
const pollInterval = opts.pollInterval ?? DEFAULT_POLL_INTERVAL;
|
|
877
|
+
const visibility = opts.visibility ?? DEFAULT_VISIBILITY;
|
|
878
|
+
const heartbeatEvery = opts.heartbeat ?? Math.floor(visibility / HEARTBEAT_DIVISOR);
|
|
879
|
+
const prefix = opts.prefix ?? DEFAULT_PREFIX;
|
|
880
|
+
const logWith = opts.logWith ?? identityLogWith;
|
|
881
|
+
const now = opts.now ?? Date.now;
|
|
882
|
+
const random = opts.random ?? Math.random;
|
|
883
|
+
const attemptsOf = (r) => r.attempts ?? getDefaultRetryOptions$1().attempts ?? 1;
|
|
884
|
+
const stats = createWorkStats(opts.metrics);
|
|
885
|
+
const k = (suffix) => prefix + suffix;
|
|
886
|
+
/** Report a swallowed non-critical error (best-effort — a throwing `onError` is itself ignored). */
|
|
887
|
+
const report = (err, phase) => {
|
|
888
|
+
try {
|
|
889
|
+
opts.onError?.(err, phase);
|
|
890
|
+
} catch {}
|
|
891
|
+
};
|
|
892
|
+
/**
|
|
893
|
+
* Run a load-bearing backend call (store/queue write) with optional {@link WorkSystemOptions.portRetry}
|
|
894
|
+
* resilience — a transient blip is absorbed before the error reaches the engine's commit/queue handling.
|
|
895
|
+
* (The bundled Redis/SQS backends also retry per call; this is an engine-level safety net for any backend.)
|
|
896
|
+
*/
|
|
897
|
+
const port = (fn) => opts.portRetry ? retry$1(fn, { ...opts.portRetry }) : fn();
|
|
898
|
+
/**
|
|
899
|
+
* The lifecycle combinator: run `fn`, then **always** run `release` (the teardown for state
|
|
900
|
+
* acquired *before* the call — an active-set entry, a heartbeat, …). The `finally` guarantees the
|
|
901
|
+
* release on every path (success, throw, early return), so no failure can leak that state. Pair it
|
|
902
|
+
* with an acquire that returns its own `release` (e.g. {@link claim}) so start↔stop can't drift apart.
|
|
903
|
+
*/
|
|
904
|
+
const scoped = async (release, fn) => {
|
|
905
|
+
try {
|
|
906
|
+
return await fn();
|
|
907
|
+
} finally {
|
|
908
|
+
release();
|
|
909
|
+
}
|
|
910
|
+
};
|
|
911
|
+
/** Publish a best-effort notification; never throws synchronously (so callers can fire-and-forget it). */
|
|
912
|
+
const publish = (obj) => {
|
|
913
|
+
try {
|
|
914
|
+
return Promise.resolve(pubsub.publish(JSON.stringify(obj))).then(() => void 0);
|
|
915
|
+
} catch (err) {
|
|
916
|
+
return Promise.reject(err instanceof Error ? err : new Error(String(err)));
|
|
917
|
+
}
|
|
918
|
+
};
|
|
919
|
+
/** Fire a best-effort pub/sub notification detached — waiters also reach it via the store poll. */
|
|
920
|
+
const notify = (obj) => void publish(obj).catch((err) => report(err, "commit"));
|
|
921
|
+
const emit = (event) => {
|
|
922
|
+
try {
|
|
923
|
+
opts.onEvent?.(event);
|
|
924
|
+
} catch {}
|
|
925
|
+
const type = event.type;
|
|
926
|
+
if (type !== void 0) try {
|
|
927
|
+
registry.get(type)?.def.options.onEvent?.(event);
|
|
928
|
+
} catch {}
|
|
929
|
+
};
|
|
930
|
+
const registry = /* @__PURE__ */ new Map();
|
|
931
|
+
for (const builder of opts.work ?? []) registry.set(builder.type, builder);
|
|
932
|
+
const codecFor = (type) => registry.get(type)?.def.options.codec ?? globalCodec;
|
|
933
|
+
const doerFor = (type) => registry.get(type)?.def.options.doer ?? globalDoer;
|
|
934
|
+
/** The queue a type's items live on (its own, or the system default). New pushes for `type` go here. */
|
|
935
|
+
const queueFor = (type) => registry.get(type)?.def.options.queue ?? backend.queue;
|
|
936
|
+
/** Every distinct queue the registry uses (the default + any per-type queues) — polled fairly by the loop. */
|
|
937
|
+
const distinctQueues = () => {
|
|
938
|
+
const set = new Set([backend.queue]);
|
|
939
|
+
for (const b of registry.values()) if (b.def.options.queue) set.add(b.def.options.queue);
|
|
940
|
+
return [...set];
|
|
941
|
+
};
|
|
942
|
+
/** Resolve an item's effective options: queue-time > type `options(input)` > type constants > defaults. */
|
|
943
|
+
const resolveOptions = (type, input, qOpts) => {
|
|
944
|
+
const tOpts = registry.get(type)?.def.options;
|
|
945
|
+
const computed = tOpts?.options?.(input) ?? {};
|
|
946
|
+
return {
|
|
947
|
+
delay: qOpts?.delay ?? computed.delay ?? 0,
|
|
948
|
+
runAt: qOpts?.runAt ?? computed.runAt,
|
|
949
|
+
priority: qOpts?.priority ?? computed.priority ?? tOpts?.priority ?? 0,
|
|
950
|
+
group: qOpts?.group ?? computed.group ?? tOpts?.group,
|
|
951
|
+
retry: {
|
|
952
|
+
...getDefaultRetryOptions$1(),
|
|
953
|
+
...opts.retry,
|
|
954
|
+
...tOpts?.retry,
|
|
955
|
+
...computed.retry,
|
|
956
|
+
...qOpts?.retry
|
|
957
|
+
},
|
|
958
|
+
skipQueue: qOpts?.skipQueue ?? computed.skipQueue ?? tOpts?.skipQueue ?? false
|
|
959
|
+
};
|
|
960
|
+
};
|
|
961
|
+
const allStates = /* @__PURE__ */ new Map();
|
|
962
|
+
const groupIndex = /* @__PURE__ */ new Map();
|
|
963
|
+
const recordState = (state, groupId) => {
|
|
964
|
+
allStates.set(state.id, state);
|
|
965
|
+
let g = groupIndex.get(groupId);
|
|
966
|
+
if (!g) {
|
|
967
|
+
g = /* @__PURE__ */ new Map();
|
|
968
|
+
groupIndex.set(groupId, g);
|
|
969
|
+
}
|
|
970
|
+
g.set(state.id, state);
|
|
971
|
+
};
|
|
972
|
+
const setState = async (state, groupId) => {
|
|
973
|
+
const { result: _result, ...forStore } = state;
|
|
974
|
+
await port(() => Promise.resolve(store.set(k(`state:${state.id}`), JSON.stringify(forStore), RESULT_TTL)));
|
|
975
|
+
recordState(state, groupId);
|
|
976
|
+
};
|
|
977
|
+
const readState = async (id) => {
|
|
978
|
+
const s = await Promise.resolve(store.get(k(`state:${id}`)));
|
|
979
|
+
if (s === void 0) return allStates.get(id);
|
|
980
|
+
try {
|
|
981
|
+
return JSON.parse(s);
|
|
982
|
+
} catch {
|
|
983
|
+
return;
|
|
984
|
+
}
|
|
985
|
+
};
|
|
986
|
+
/** Atomic add via the store `increment` atom, falling back to a (single-process-safe) get+set. */
|
|
987
|
+
const storeIncrement = (key, by, ttl) => port(async () => {
|
|
988
|
+
if (store.increment) return await Promise.resolve(store.increment(key, by, ttl));
|
|
989
|
+
const cur = Number(await Promise.resolve(store.get(key)) ?? "0") + by;
|
|
990
|
+
await Promise.resolve(store.set(key, String(cur), ttl));
|
|
991
|
+
return cur;
|
|
992
|
+
});
|
|
993
|
+
const groupIncr = (groupId, by) => storeIncrement(k(`group:${groupId}:open`), by, RESULT_TTL);
|
|
994
|
+
/** Roll back a group hold (`groupIncr(+1)`) whose item never reached the queue / a terminal path — best-effort, so the group can still settle. */
|
|
995
|
+
const undoHold = (groupId) => groupIncr(groupId, -1).then(() => void 0).catch((err) => report(err, "commit"));
|
|
996
|
+
const settleGroup = async (groupId) => {
|
|
997
|
+
if (await groupIncr(groupId, -1) > 0) return;
|
|
998
|
+
await port(() => Promise.resolve(store.set(k(`group:${groupId}:done`), "1", RESULT_TTL)));
|
|
999
|
+
notify({
|
|
1000
|
+
kind: "group-done",
|
|
1001
|
+
groupId
|
|
1002
|
+
});
|
|
1003
|
+
const r = await Promise.resolve(store.get(k(`group:${groupId}:result`)));
|
|
1004
|
+
emit({
|
|
1005
|
+
kind: "group-done",
|
|
1006
|
+
groupId,
|
|
1007
|
+
result: r === void 0 ? void 0 : globalCodec.parse(r),
|
|
1008
|
+
at: now()
|
|
1009
|
+
});
|
|
1010
|
+
maybeUnhandled(groupId);
|
|
1011
|
+
};
|
|
1012
|
+
const makeContext = (impl) => {
|
|
1013
|
+
const queue = (works, options) => Array.isArray(works) ? works.map((w) => impl.enqueueOne(w, options)) : impl.enqueueOne(works, options);
|
|
1014
|
+
return {
|
|
1015
|
+
id: impl.id,
|
|
1016
|
+
groupId: impl.groupId,
|
|
1017
|
+
attempt: impl.attempt,
|
|
1018
|
+
queue,
|
|
1019
|
+
setResult: impl.setResult,
|
|
1020
|
+
states: impl.states,
|
|
1021
|
+
claim: impl.claim
|
|
1022
|
+
};
|
|
1023
|
+
};
|
|
1024
|
+
const storeStates = (ids) => Promise.all(ids.map(readState));
|
|
1025
|
+
const storeClaim = (key) => Promise.resolve(store.setIfNotExists(k(key), "1", RESULT_TTL));
|
|
1026
|
+
/** Push an envelope onto **its type's** queue (default or per-type), invisible for `delay` ms. */
|
|
1027
|
+
const pushEnvelope = (env, delay) => port(() => Promise.resolve(queueFor(env.type).push(JSON.stringify(env), { delay })).then(() => void 0));
|
|
1028
|
+
/** Resolve when a deferred/scheduled item should next run (absolute epoch ms). */
|
|
1029
|
+
const resolveRunAt = (when) => when.runAt ?? now() + (when.delay ?? 0);
|
|
1030
|
+
const submitQueued = (id, type, input, groupId, ro) => {
|
|
1031
|
+
const queueAt = now();
|
|
1032
|
+
const startAt = ro.runAt ?? queueAt + ro.delay;
|
|
1033
|
+
return (async () => {
|
|
1034
|
+
await groupIncr(groupId, 1);
|
|
1035
|
+
try {
|
|
1036
|
+
await setState({
|
|
1037
|
+
id,
|
|
1038
|
+
type,
|
|
1039
|
+
status: "pending",
|
|
1040
|
+
attempt: 1,
|
|
1041
|
+
queueAt,
|
|
1042
|
+
startAt,
|
|
1043
|
+
priority: ro.priority,
|
|
1044
|
+
group: ro.group
|
|
1045
|
+
}, groupId);
|
|
1046
|
+
await pushEnvelope({
|
|
1047
|
+
id,
|
|
1048
|
+
type,
|
|
1049
|
+
groupId,
|
|
1050
|
+
input: codecFor(type).stringify(input),
|
|
1051
|
+
queueAt,
|
|
1052
|
+
startAt,
|
|
1053
|
+
attempt: 1,
|
|
1054
|
+
priority: ro.priority,
|
|
1055
|
+
group: ro.group,
|
|
1056
|
+
retry: ro.retry
|
|
1057
|
+
}, Math.max(0, startAt - queueAt));
|
|
1058
|
+
} catch (err) {
|
|
1059
|
+
await undoHold(groupId);
|
|
1060
|
+
throw err;
|
|
1061
|
+
}
|
|
1062
|
+
stats.queued(type, queueAt);
|
|
1063
|
+
emit({
|
|
1064
|
+
kind: "queued",
|
|
1065
|
+
id,
|
|
1066
|
+
type,
|
|
1067
|
+
groupId,
|
|
1068
|
+
at: queueAt
|
|
1069
|
+
});
|
|
1070
|
+
})();
|
|
1071
|
+
};
|
|
1072
|
+
/** Re-enter the queue for a retry: a fresh delivery with `attempt + 1` and a recomputed `startAt`. */
|
|
1073
|
+
const rePush = async (env, delay) => {
|
|
1074
|
+
const startAt = now() + delay;
|
|
1075
|
+
const next = {
|
|
1076
|
+
...env,
|
|
1077
|
+
attempt: env.attempt + 1,
|
|
1078
|
+
startAt
|
|
1079
|
+
};
|
|
1080
|
+
await setState({
|
|
1081
|
+
id: env.id,
|
|
1082
|
+
type: env.type,
|
|
1083
|
+
status: "pending",
|
|
1084
|
+
attempt: next.attempt,
|
|
1085
|
+
queueAt: env.queueAt,
|
|
1086
|
+
startAt,
|
|
1087
|
+
priority: env.priority,
|
|
1088
|
+
group: env.group
|
|
1089
|
+
}, env.groupId);
|
|
1090
|
+
await pushEnvelope(next, delay);
|
|
1091
|
+
};
|
|
1092
|
+
const enqueueImpl = (a, b, c) => {
|
|
1093
|
+
const fromName = typeof a === "string";
|
|
1094
|
+
const id = fromName ? uuid() : a.id;
|
|
1095
|
+
const type = fromName ? a : a.type;
|
|
1096
|
+
const input = fromName ? b : a.input;
|
|
1097
|
+
const qOpts = fromName ? c : b;
|
|
1098
|
+
const groupId = uuid();
|
|
1099
|
+
const ro = resolveOptions(type, input, qOpts);
|
|
1100
|
+
return makeHandle(id, type, groupId, ro.skipQueue ? runImmediate(id, type, input, groupId, ro) : submitQueued(id, type, input, groupId, ro));
|
|
1101
|
+
};
|
|
1102
|
+
const enqueueRaw = (type, input) => void enqueueImpl(type, input);
|
|
1103
|
+
const makeHandle = (id, type, groupId, ready) => {
|
|
1104
|
+
const markWaiting = () => Promise.resolve(store.setIfNotExists(k(`wait:${groupId}`), "1", WAIT_TTL));
|
|
1105
|
+
const result = async () => {
|
|
1106
|
+
await ready;
|
|
1107
|
+
await markWaiting();
|
|
1108
|
+
return waitForItem(id, type);
|
|
1109
|
+
};
|
|
1110
|
+
const group = async () => {
|
|
1111
|
+
await ready;
|
|
1112
|
+
await markWaiting();
|
|
1113
|
+
return waitForGroup(groupId);
|
|
1114
|
+
};
|
|
1115
|
+
return {
|
|
1116
|
+
id,
|
|
1117
|
+
groupId,
|
|
1118
|
+
result,
|
|
1119
|
+
group,
|
|
1120
|
+
then: (onF, onR) => group().then(onF, onR)
|
|
1121
|
+
};
|
|
1122
|
+
};
|
|
1123
|
+
const waitForItem = (id, type) => new Promise((resolve, reject) => {
|
|
1124
|
+
let settled = false;
|
|
1125
|
+
const teardown = [];
|
|
1126
|
+
const cleanup = () => {
|
|
1127
|
+
settled = true;
|
|
1128
|
+
for (const t of teardown) t();
|
|
1129
|
+
};
|
|
1130
|
+
const check = async () => {
|
|
1131
|
+
if (settled) return;
|
|
1132
|
+
const res = await Promise.resolve(store.get(k(`result:${id}`)));
|
|
1133
|
+
if (res !== void 0) {
|
|
1134
|
+
cleanup();
|
|
1135
|
+
resolve(codecFor(type).parse(res));
|
|
1136
|
+
return;
|
|
1137
|
+
}
|
|
1138
|
+
const err = await Promise.resolve(store.get(k(`item:${id}:error`)));
|
|
1139
|
+
if (err !== void 0) {
|
|
1140
|
+
cleanup();
|
|
1141
|
+
reject(new Error(err));
|
|
1142
|
+
}
|
|
1143
|
+
};
|
|
1144
|
+
teardown.push(pubsub.subscribe((m) => {
|
|
1145
|
+
try {
|
|
1146
|
+
const e = JSON.parse(m);
|
|
1147
|
+
if (e.kind === "done" && e.id === id) check();
|
|
1148
|
+
} catch {}
|
|
1149
|
+
}));
|
|
1150
|
+
const poll = setInterval(() => void check(), WAIT_POLL);
|
|
1151
|
+
unref(poll);
|
|
1152
|
+
teardown.push(() => clearInterval(poll));
|
|
1153
|
+
check();
|
|
1154
|
+
});
|
|
1155
|
+
const waitForGroup = (groupId) => new Promise((resolve) => {
|
|
1156
|
+
let settled = false;
|
|
1157
|
+
const teardown = [];
|
|
1158
|
+
const cleanup = () => {
|
|
1159
|
+
settled = true;
|
|
1160
|
+
for (const t of teardown) t();
|
|
1161
|
+
};
|
|
1162
|
+
const check = async () => {
|
|
1163
|
+
if (settled) return;
|
|
1164
|
+
if (await Promise.resolve(store.get(k(`group:${groupId}:done`))) === void 0) return;
|
|
1165
|
+
cleanup();
|
|
1166
|
+
const r = await Promise.resolve(store.get(k(`group:${groupId}:result`)));
|
|
1167
|
+
resolve(r === void 0 ? void 0 : globalCodec.parse(r));
|
|
1168
|
+
};
|
|
1169
|
+
teardown.push(pubsub.subscribe((m) => {
|
|
1170
|
+
try {
|
|
1171
|
+
const e = JSON.parse(m);
|
|
1172
|
+
if (e.kind === "group-done" && e.groupId === groupId) check();
|
|
1173
|
+
} catch {}
|
|
1174
|
+
}));
|
|
1175
|
+
const poll = setInterval(() => void check(), WAIT_POLL);
|
|
1176
|
+
unref(poll);
|
|
1177
|
+
teardown.push(() => clearInterval(poll));
|
|
1178
|
+
check();
|
|
1179
|
+
});
|
|
1180
|
+
const maybeUnhandled = (groupId) => {
|
|
1181
|
+
if (!opts.unhandledWorkGroup) return;
|
|
1182
|
+
unref(setTimeout(async () => {
|
|
1183
|
+
if (!await Promise.resolve(store.setIfNotExists(k(`group-handled:${groupId}`), "1", RESULT_TTL))) return;
|
|
1184
|
+
if (await Promise.resolve(store.get(k(`wait:${groupId}`))) !== void 0) return;
|
|
1185
|
+
const r = await Promise.resolve(store.get(k(`group:${groupId}:result`)));
|
|
1186
|
+
const states = [...groupIndex.get(groupId)?.values() ?? []];
|
|
1187
|
+
opts.unhandledWorkGroup?.({
|
|
1188
|
+
groupId,
|
|
1189
|
+
lastResult: r === void 0 ? void 0 : globalCodec.parse(r),
|
|
1190
|
+
states
|
|
1191
|
+
});
|
|
1192
|
+
}, UNHANDLED_GRACE));
|
|
1193
|
+
};
|
|
1194
|
+
const activeMap = /* @__PURE__ */ new Map();
|
|
1195
|
+
/**
|
|
1196
|
+
* `skipQueue`: hand the first attempt straight to the doer (no queue hop, no lease,
|
|
1197
|
+
* no heartbeat) for low latency. State/results/group still go through the store, and
|
|
1198
|
+
* a **failure re-enqueues** the item (attempt + 1) onto the durable queue — so the
|
|
1199
|
+
* retry survives a crash and any instance in the fleet can pick it up. (The first run
|
|
1200
|
+
* itself is best-effort: that's the latency-for-durability trade `skipQueue` makes.)
|
|
1201
|
+
*/
|
|
1202
|
+
const runImmediate = (id, type, input, groupId, ro) => {
|
|
1203
|
+
const queueAt = now();
|
|
1204
|
+
const startAt = ro.runAt ?? queueAt + ro.delay;
|
|
1205
|
+
const builder = registry.get(type);
|
|
1206
|
+
const env = {
|
|
1207
|
+
id,
|
|
1208
|
+
type,
|
|
1209
|
+
groupId,
|
|
1210
|
+
input: codecFor(type).stringify(input),
|
|
1211
|
+
queueAt,
|
|
1212
|
+
startAt,
|
|
1213
|
+
attempt: 1,
|
|
1214
|
+
priority: ro.priority,
|
|
1215
|
+
group: ro.group,
|
|
1216
|
+
retry: ro.retry
|
|
1217
|
+
};
|
|
1218
|
+
return (async () => {
|
|
1219
|
+
await groupIncr(groupId, 1);
|
|
1220
|
+
try {
|
|
1221
|
+
await setState({
|
|
1222
|
+
id,
|
|
1223
|
+
type,
|
|
1224
|
+
status: "pending",
|
|
1225
|
+
attempt: 1,
|
|
1226
|
+
queueAt,
|
|
1227
|
+
startAt,
|
|
1228
|
+
priority: ro.priority,
|
|
1229
|
+
group: ro.group
|
|
1230
|
+
}, groupId);
|
|
1231
|
+
stats.queued(type, queueAt);
|
|
1232
|
+
emit({
|
|
1233
|
+
kind: "queued",
|
|
1234
|
+
id,
|
|
1235
|
+
type,
|
|
1236
|
+
groupId,
|
|
1237
|
+
at: queueAt
|
|
1238
|
+
});
|
|
1239
|
+
} catch (err) {
|
|
1240
|
+
await undoHold(groupId);
|
|
1241
|
+
throw err;
|
|
1242
|
+
}
|
|
1243
|
+
if (!builder) {
|
|
1244
|
+
await deadLetterItem(null, env, `unknown work type: ${type}`, null);
|
|
1245
|
+
return;
|
|
1246
|
+
}
|
|
1247
|
+
const release = claim(env, null, null);
|
|
1248
|
+
try {
|
|
1249
|
+
doerFor(type).do(() => execute(null, env, input, builder.def, null, release));
|
|
1250
|
+
} catch (err) {
|
|
1251
|
+
release();
|
|
1252
|
+
await undoHold(groupId);
|
|
1253
|
+
throw err;
|
|
1254
|
+
}
|
|
1255
|
+
})();
|
|
1256
|
+
};
|
|
1257
|
+
/** Keep a leased item's lease (and its heartbeat key) alive on the **queue it came from**. */
|
|
1258
|
+
const startHeartbeat = (p, id, q) => {
|
|
1259
|
+
const hb = setInterval(() => {
|
|
1260
|
+
Promise.resolve(q.heartbeat(p, visibility));
|
|
1261
|
+
Promise.resolve(store.set(k(`hb:${id}`), String(now()), visibility));
|
|
1262
|
+
}, heartbeatEvery);
|
|
1263
|
+
unref(hb);
|
|
1264
|
+
return hb;
|
|
1265
|
+
};
|
|
1266
|
+
/**
|
|
1267
|
+
* Acquire the per-item processing state — register it in the active set and (if leased) start its
|
|
1268
|
+
* heartbeat — and return the **paired teardown** that clears the heartbeat and drops it from the
|
|
1269
|
+
* active set. This closure is the *sole owner* of that state: whoever runs the item passes the
|
|
1270
|
+
* returned `release` to {@link scoped}, so it is torn down on every path and can never drift/leak.
|
|
1271
|
+
* (A `skipQueue` item has no lease `p`/`q`, so no heartbeat.)
|
|
1272
|
+
*/
|
|
1273
|
+
const claim = (env, p, q) => {
|
|
1274
|
+
activeMap.set(env.id, {
|
|
1275
|
+
id: env.id,
|
|
1276
|
+
type: env.type,
|
|
1277
|
+
groupId: env.groupId,
|
|
1278
|
+
status: "pending",
|
|
1279
|
+
attempt: env.attempt,
|
|
1280
|
+
priority: env.priority,
|
|
1281
|
+
group: env.group,
|
|
1282
|
+
queueAt: env.queueAt,
|
|
1283
|
+
startAt: env.startAt
|
|
1284
|
+
});
|
|
1285
|
+
stats.claimed(env.type);
|
|
1286
|
+
const hb = p && q ? startHeartbeat(p, env.id, q) : null;
|
|
1287
|
+
return () => {
|
|
1288
|
+
if (hb) clearInterval(hb);
|
|
1289
|
+
const wasRunning = activeMap.get(env.id)?.status === "running";
|
|
1290
|
+
activeMap.delete(env.id);
|
|
1291
|
+
stats.released(env.type, wasRunning);
|
|
1292
|
+
};
|
|
1293
|
+
};
|
|
1294
|
+
/**
|
|
1295
|
+
* Re-enqueue an item to become runnable at `startAt` **without advancing its attempt** — a
|
|
1296
|
+
* reschedule, not a retry. It **acks the current delivery and pushes a fresh message**, so a
|
|
1297
|
+
* backend's per-message delivery count (e.g. SQS `ApproximateReceiveCount`) **resets** — a
|
|
1298
|
+
* scheduled item that bounces (re-deferred each poll until due) can never trip the backend's
|
|
1299
|
+
* native redrive/DLQ. Returns the resolved `startAt`.
|
|
1300
|
+
*/
|
|
1301
|
+
const reschedule = async (p, env, q, startAt) => {
|
|
1302
|
+
const next = {
|
|
1303
|
+
...env,
|
|
1304
|
+
startAt
|
|
1305
|
+
};
|
|
1306
|
+
await setState({
|
|
1307
|
+
id: env.id,
|
|
1308
|
+
type: env.type,
|
|
1309
|
+
status: "pending",
|
|
1310
|
+
attempt: env.attempt,
|
|
1311
|
+
queueAt: env.queueAt,
|
|
1312
|
+
startAt,
|
|
1313
|
+
priority: env.priority,
|
|
1314
|
+
group: env.group
|
|
1315
|
+
}, env.groupId);
|
|
1316
|
+
if (p && q) await port(() => Promise.resolve(q.ack(p)));
|
|
1317
|
+
await pushEnvelope(next, Math.max(0, startAt - now()));
|
|
1318
|
+
return startAt;
|
|
1319
|
+
};
|
|
1320
|
+
/** A handler/classifier-initiated deferral: {@link reschedule} the item to `runAt` and emit a `deferred` event. */
|
|
1321
|
+
const deferItem = async (p, env, q, runAt) => {
|
|
1322
|
+
const startAt = await reschedule(p, env, q, Math.max(runAt, now()));
|
|
1323
|
+
const at = now();
|
|
1324
|
+
stats.deferred(env.type, startAt - at);
|
|
1325
|
+
emit({
|
|
1326
|
+
kind: "deferred",
|
|
1327
|
+
id: env.id,
|
|
1328
|
+
type: env.type,
|
|
1329
|
+
groupId: env.groupId,
|
|
1330
|
+
runAt: startAt,
|
|
1331
|
+
at
|
|
1332
|
+
});
|
|
1333
|
+
};
|
|
1334
|
+
const deadLetterItem = async (p, env, error, q, runAt) => {
|
|
1335
|
+
const at = now();
|
|
1336
|
+
stats.failed(env.type, at, runAt === void 0 ? void 0 : at - runAt, at - env.queueAt, env.attempt);
|
|
1337
|
+
await port(() => Promise.resolve(store.set(k(`item:${env.id}:error`), error, RESULT_TTL)));
|
|
1338
|
+
await setState({
|
|
1339
|
+
id: env.id,
|
|
1340
|
+
type: env.type,
|
|
1341
|
+
status: "dead",
|
|
1342
|
+
attempt: env.attempt,
|
|
1343
|
+
error,
|
|
1344
|
+
queueAt: env.queueAt,
|
|
1345
|
+
startAt: env.startAt,
|
|
1346
|
+
runAt,
|
|
1347
|
+
endAt: at,
|
|
1348
|
+
priority: env.priority,
|
|
1349
|
+
group: env.group
|
|
1350
|
+
}, env.groupId);
|
|
1351
|
+
if (p && q) {
|
|
1352
|
+
await Promise.resolve(q.deadLetter?.(p.body, error));
|
|
1353
|
+
await port(() => Promise.resolve(q.ack(p)));
|
|
1354
|
+
}
|
|
1355
|
+
notify({
|
|
1356
|
+
kind: "done",
|
|
1357
|
+
id: env.id,
|
|
1358
|
+
groupId: env.groupId,
|
|
1359
|
+
error
|
|
1360
|
+
});
|
|
1361
|
+
emit({
|
|
1362
|
+
kind: "failed",
|
|
1363
|
+
id: env.id,
|
|
1364
|
+
type: env.type,
|
|
1365
|
+
groupId: env.groupId,
|
|
1366
|
+
attempt: env.attempt,
|
|
1367
|
+
error,
|
|
1368
|
+
willRetry: false,
|
|
1369
|
+
at
|
|
1370
|
+
});
|
|
1371
|
+
await settleGroup(env.groupId);
|
|
1372
|
+
};
|
|
1373
|
+
/** The synchronous part of marking running (the active-map flip) — kept on the critical path. */
|
|
1374
|
+
const markRunningSync = (env, runAt) => {
|
|
1375
|
+
stats.started(env.type, runAt, runAt - env.startAt);
|
|
1376
|
+
const a = activeMap.get(env.id);
|
|
1377
|
+
if (a) {
|
|
1378
|
+
a.status = "running";
|
|
1379
|
+
a.runAt = runAt;
|
|
1380
|
+
}
|
|
1381
|
+
};
|
|
1382
|
+
/** The async part (persist the `running` state + emit `started`) — can run **alongside** the handler. */
|
|
1383
|
+
const persistRunning = async (env, runAt) => {
|
|
1384
|
+
await setState({
|
|
1385
|
+
id: env.id,
|
|
1386
|
+
type: env.type,
|
|
1387
|
+
status: "running",
|
|
1388
|
+
attempt: env.attempt,
|
|
1389
|
+
queueAt: env.queueAt,
|
|
1390
|
+
startAt: env.startAt,
|
|
1391
|
+
runAt,
|
|
1392
|
+
priority: env.priority,
|
|
1393
|
+
group: env.group
|
|
1394
|
+
}, env.groupId);
|
|
1395
|
+
emit({
|
|
1396
|
+
kind: "started",
|
|
1397
|
+
id: env.id,
|
|
1398
|
+
type: env.type,
|
|
1399
|
+
groupId: env.groupId,
|
|
1400
|
+
attempt: env.attempt,
|
|
1401
|
+
at: runAt
|
|
1402
|
+
});
|
|
1403
|
+
};
|
|
1404
|
+
const markRunning = async (env, runAt) => {
|
|
1405
|
+
markRunningSync(env, runAt);
|
|
1406
|
+
await persistRunning(env, runAt);
|
|
1407
|
+
};
|
|
1408
|
+
const finishSuccess = async (p, env, runAt, output, q) => {
|
|
1409
|
+
const at = now();
|
|
1410
|
+
stats.succeeded(env.type, at, at - runAt, at - env.queueAt, env.attempt);
|
|
1411
|
+
await port(() => Promise.resolve(store.set(k(`result:${env.id}`), codecFor(env.type).stringify(output), RESULT_TTL)));
|
|
1412
|
+
await setState({
|
|
1413
|
+
id: env.id,
|
|
1414
|
+
type: env.type,
|
|
1415
|
+
status: "success",
|
|
1416
|
+
attempt: env.attempt,
|
|
1417
|
+
result: output,
|
|
1418
|
+
queueAt: env.queueAt,
|
|
1419
|
+
startAt: env.startAt,
|
|
1420
|
+
runAt,
|
|
1421
|
+
endAt: at,
|
|
1422
|
+
priority: env.priority,
|
|
1423
|
+
group: env.group
|
|
1424
|
+
}, env.groupId);
|
|
1425
|
+
if (p && q) await port(() => Promise.resolve(q.ack(p)));
|
|
1426
|
+
notify({
|
|
1427
|
+
kind: "done",
|
|
1428
|
+
id: env.id,
|
|
1429
|
+
groupId: env.groupId
|
|
1430
|
+
});
|
|
1431
|
+
emit({
|
|
1432
|
+
kind: "succeeded",
|
|
1433
|
+
id: env.id,
|
|
1434
|
+
type: env.type,
|
|
1435
|
+
groupId: env.groupId,
|
|
1436
|
+
attempt: env.attempt,
|
|
1437
|
+
result: output,
|
|
1438
|
+
at
|
|
1439
|
+
});
|
|
1440
|
+
await settleGroup(env.groupId);
|
|
1441
|
+
};
|
|
1442
|
+
const finishFailure = async (p, env, runAt, err, q) => {
|
|
1443
|
+
if (env.attempt < attemptsOf(env.retry)) {
|
|
1444
|
+
const delay = backoff$1(env.attempt, env.retry, random);
|
|
1445
|
+
stats.retried(env.type);
|
|
1446
|
+
emit({
|
|
1447
|
+
kind: "failed",
|
|
1448
|
+
id: env.id,
|
|
1449
|
+
type: env.type,
|
|
1450
|
+
groupId: env.groupId,
|
|
1451
|
+
attempt: env.attempt,
|
|
1452
|
+
error: errString(err),
|
|
1453
|
+
willRetry: true,
|
|
1454
|
+
at: now()
|
|
1455
|
+
});
|
|
1456
|
+
if (p && q) await port(() => Promise.resolve(q.ack(p)));
|
|
1457
|
+
await rePush(env, delay);
|
|
1458
|
+
} else await deadLetterItem(p, env, errString(err), q, runAt);
|
|
1459
|
+
};
|
|
1460
|
+
/** Ask the per-type (else system) {@link FailureClassifier} what a handler error means; a throwing classifier falls back to the default. */
|
|
1461
|
+
const classifyFailure = async (err, env) => {
|
|
1462
|
+
const decide = registry.get(env.type)?.def.options.onFailure ?? opts.onFailure;
|
|
1463
|
+
if (!decide) return;
|
|
1464
|
+
try {
|
|
1465
|
+
return await decide(err, {
|
|
1466
|
+
id: env.id,
|
|
1467
|
+
type: env.type,
|
|
1468
|
+
attempt: env.attempt,
|
|
1469
|
+
attempts: attemptsOf(env.retry)
|
|
1470
|
+
}) ?? void 0;
|
|
1471
|
+
} catch (e) {
|
|
1472
|
+
report(e, "commit");
|
|
1473
|
+
return;
|
|
1474
|
+
}
|
|
1475
|
+
};
|
|
1476
|
+
/**
|
|
1477
|
+
* Decide what a handler/batch failure does — the single owner of failure routing for both single and
|
|
1478
|
+
* batched items. Explicit throws win (`WorkDelayError` → reschedule, `RetryAbort` → dead-letter now);
|
|
1479
|
+
* otherwise the {@link classifyFailure} hook may `'abort'` or reschedule a `{ delay }`/`{ runAt }`
|
|
1480
|
+
* (e.g. a rate limit — re-queued **without** advancing `attempt`); the default is retry/dead-letter.
|
|
1481
|
+
*/
|
|
1482
|
+
const handleFailure = async (p, env, runAt, err, q) => {
|
|
1483
|
+
if (err instanceof WorkDelayError) {
|
|
1484
|
+
await deferItem(p, env, q, resolveRunAt(err.when));
|
|
1485
|
+
return;
|
|
1486
|
+
}
|
|
1487
|
+
if (err instanceof RetryAbort$1) {
|
|
1488
|
+
await deadLetterItem(p, env, errString(err.cause ?? err), q, runAt);
|
|
1489
|
+
return;
|
|
1490
|
+
}
|
|
1491
|
+
const decision = await classifyFailure(err, env);
|
|
1492
|
+
if (decision === "abort") {
|
|
1493
|
+
await deadLetterItem(p, env, errString(err), q, runAt);
|
|
1494
|
+
return;
|
|
1495
|
+
}
|
|
1496
|
+
if (decision !== void 0 && decision !== "retry") {
|
|
1497
|
+
await deferItem(p, env, q, "runAt" in decision ? decision.runAt : now() + decision.delay);
|
|
1498
|
+
return;
|
|
1499
|
+
}
|
|
1500
|
+
await finishFailure(p, env, runAt, err, q);
|
|
1501
|
+
};
|
|
1502
|
+
const queuedContext = (env, pending) => makeContext({
|
|
1503
|
+
id: env.id,
|
|
1504
|
+
groupId: env.groupId,
|
|
1505
|
+
attempt: env.attempt,
|
|
1506
|
+
enqueueOne: (work, options) => {
|
|
1507
|
+
pending.push(submitQueued(work.id, work.type, work.input, env.groupId, resolveOptions(work.type, work.input, options)));
|
|
1508
|
+
return work.id;
|
|
1509
|
+
},
|
|
1510
|
+
setResult: (r) => {
|
|
1511
|
+
pending.push(port(() => Promise.resolve(store.set(k(`group:${env.groupId}:result`), globalCodec.stringify(r), RESULT_TTL)).then(() => void 0)));
|
|
1512
|
+
},
|
|
1513
|
+
states: storeStates,
|
|
1514
|
+
claim: storeClaim
|
|
1515
|
+
});
|
|
1516
|
+
/**
|
|
1517
|
+
* The task handed to a doer: run a single item (leased from queue `q`, or `skipQueue` with `p`/`q`
|
|
1518
|
+
* null). `release` is the {@link claim} teardown — {@link scoped} runs it in `finally`, so the
|
|
1519
|
+
* active-set entry + heartbeat are always cleaned up regardless of how the run ends.
|
|
1520
|
+
*/
|
|
1521
|
+
const execute = (p, env, input, def, q, release) => scoped(release, async () => {
|
|
1522
|
+
const runAt = now();
|
|
1523
|
+
markRunningSync(env, runAt);
|
|
1524
|
+
const running = persistRunning(env, runAt).catch((err) => report(err, "commit"));
|
|
1525
|
+
const pending = [];
|
|
1526
|
+
const ctx = queuedContext(env, pending);
|
|
1527
|
+
let output;
|
|
1528
|
+
try {
|
|
1529
|
+
output = await logWith(merge(opts.logContext?.(input, env.type), def.options.logContext?.(input)), () => Promise.resolve(def.handler(input, ctx)));
|
|
1530
|
+
await Promise.all(pending);
|
|
1531
|
+
} catch (err) {
|
|
1532
|
+
await running;
|
|
1533
|
+
await handleFailure(p, env, runAt, err, q);
|
|
1534
|
+
return;
|
|
1535
|
+
}
|
|
1536
|
+
try {
|
|
1537
|
+
await running;
|
|
1538
|
+
await finishSuccess(p, env, runAt, output, q);
|
|
1539
|
+
} catch (err) {
|
|
1540
|
+
report(err, "commit");
|
|
1541
|
+
}
|
|
1542
|
+
});
|
|
1543
|
+
const executeBatch = (items, batch) => scoped(() => {
|
|
1544
|
+
for (const it of items) it.release();
|
|
1545
|
+
}, async () => {
|
|
1546
|
+
const runAt = now();
|
|
1547
|
+
await Promise.all(items.map((it) => markRunning(it.env, runAt)));
|
|
1548
|
+
let outputs;
|
|
1549
|
+
try {
|
|
1550
|
+
const ran = await Promise.resolve(batch.run(items.map((it) => it.input)));
|
|
1551
|
+
if (!Array.isArray(ran) || ran.length !== items.length) throw new Error(`batch "${items[0]?.env.type}" returned ${Array.isArray(ran) ? ran.length : "a non-array"} outputs for ${items.length} inputs`);
|
|
1552
|
+
outputs = ran;
|
|
1553
|
+
} catch (err) {
|
|
1554
|
+
await Promise.all(items.map((it) => handleFailure(it.p, it.env, runAt, err, it.q)));
|
|
1555
|
+
return;
|
|
1556
|
+
}
|
|
1557
|
+
await Promise.all(items.map((it, i) => Promise.resolve(finishSuccess(it.p, it.env, runAt, outputs[i], it.q)).catch((err) => report(err, "commit"))));
|
|
1558
|
+
});
|
|
1559
|
+
const batchers = /* @__PURE__ */ new Map();
|
|
1560
|
+
const batcherFor = (type) => {
|
|
1561
|
+
let b = batchers.get(type);
|
|
1562
|
+
if (!b) {
|
|
1563
|
+
const config = registry.get(type).def.batch;
|
|
1564
|
+
const doer = doerFor(type);
|
|
1565
|
+
const buffer = [];
|
|
1566
|
+
let timer = null;
|
|
1567
|
+
const flush = () => {
|
|
1568
|
+
if (timer) {
|
|
1569
|
+
clearTimeout(timer);
|
|
1570
|
+
timer = null;
|
|
1571
|
+
}
|
|
1572
|
+
if (buffer.length === 0) return;
|
|
1573
|
+
const items = buffer.splice(0);
|
|
1574
|
+
try {
|
|
1575
|
+
doer.do(() => executeBatch(items, config), { createdAt: items[0].env.queueAt });
|
|
1576
|
+
} catch (err) {
|
|
1577
|
+
for (const it of items) it.release();
|
|
1578
|
+
report(err, "queue");
|
|
1579
|
+
}
|
|
1580
|
+
};
|
|
1581
|
+
b = {
|
|
1582
|
+
available: () => doer.available() <= 0 ? 0 : Math.max(0, config.size - buffer.length),
|
|
1583
|
+
add: (item) => {
|
|
1584
|
+
buffer.push(item);
|
|
1585
|
+
if (buffer.length >= config.size) flush();
|
|
1586
|
+
else if (!timer) {
|
|
1587
|
+
timer = setTimeout(flush, config.maxWait);
|
|
1588
|
+
unref(timer);
|
|
1589
|
+
}
|
|
1590
|
+
},
|
|
1591
|
+
flushAll: flush
|
|
1592
|
+
};
|
|
1593
|
+
batchers.set(type, b);
|
|
1594
|
+
}
|
|
1595
|
+
return b;
|
|
1596
|
+
};
|
|
1597
|
+
/**
|
|
1598
|
+
* Parse + validate an item leased from queue `q` and route it to a batcher or doer. Returns `true`
|
|
1599
|
+
* if it **started** the item, `false` if it put it back (early/not-due, affinity decline, saturated
|
|
1600
|
+
* doer/batcher, unknown type) or dropped it — the loop uses this to keep pulling vs back off.
|
|
1601
|
+
*/
|
|
1602
|
+
const accept = async (p, q) => {
|
|
1603
|
+
let env;
|
|
1604
|
+
try {
|
|
1605
|
+
env = JSON.parse(p.body);
|
|
1606
|
+
} catch {
|
|
1607
|
+
await Promise.resolve(q.ack(p));
|
|
1608
|
+
return false;
|
|
1609
|
+
}
|
|
1610
|
+
const due = now();
|
|
1611
|
+
if (env.startAt - due > SCHED_TOLERANCE) {
|
|
1612
|
+
await reschedule(p, env, q, env.startAt);
|
|
1613
|
+
stats.rescheduled(env.type, env.startAt - due);
|
|
1614
|
+
return false;
|
|
1615
|
+
}
|
|
1616
|
+
const { id, type, groupId } = env;
|
|
1617
|
+
const builder = registry.get(type);
|
|
1618
|
+
if (!builder) {
|
|
1619
|
+
if (env.attempt >= attemptsOf(env.retry)) await deadLetterItem(p, env, `unknown work type: ${type}`, q);
|
|
1620
|
+
else await Promise.resolve(q.fail(p, UNKNOWN_TYPE_DELAY));
|
|
1621
|
+
return false;
|
|
1622
|
+
}
|
|
1623
|
+
let input;
|
|
1624
|
+
try {
|
|
1625
|
+
input = codecFor(type).parse(env.input);
|
|
1626
|
+
} catch (e) {
|
|
1627
|
+
await deadLetterItem(p, env, `bad input: ${errString(e)}`, q);
|
|
1628
|
+
return false;
|
|
1629
|
+
}
|
|
1630
|
+
if (opts.accept && !opts.accept({
|
|
1631
|
+
id,
|
|
1632
|
+
type,
|
|
1633
|
+
groupId,
|
|
1634
|
+
attempt: env.attempt,
|
|
1635
|
+
input
|
|
1636
|
+
})) {
|
|
1637
|
+
await Promise.resolve(q.fail(p, pollInterval));
|
|
1638
|
+
return false;
|
|
1639
|
+
}
|
|
1640
|
+
if (builder.def.batch) {
|
|
1641
|
+
const batcher = batcherFor(type);
|
|
1642
|
+
if (batcher.available() <= 0) {
|
|
1643
|
+
await Promise.resolve(q.fail(p, pollInterval));
|
|
1644
|
+
return false;
|
|
1645
|
+
}
|
|
1646
|
+
batcher.add({
|
|
1647
|
+
p,
|
|
1648
|
+
env,
|
|
1649
|
+
input,
|
|
1650
|
+
release: claim(env, p, q),
|
|
1651
|
+
q
|
|
1652
|
+
});
|
|
1653
|
+
return true;
|
|
1654
|
+
}
|
|
1655
|
+
const doer = doerFor(type);
|
|
1656
|
+
if (doer.available() <= 0) {
|
|
1657
|
+
await Promise.resolve(q.fail(p, pollInterval));
|
|
1658
|
+
return false;
|
|
1659
|
+
}
|
|
1660
|
+
const release = claim(env, p, q);
|
|
1661
|
+
try {
|
|
1662
|
+
doer.do(() => execute(p, env, input, builder.def, q, release), {
|
|
1663
|
+
group: env.group,
|
|
1664
|
+
priority: env.priority,
|
|
1665
|
+
createdAt: env.queueAt
|
|
1666
|
+
});
|
|
1667
|
+
} catch (err) {
|
|
1668
|
+
release();
|
|
1669
|
+
throw err;
|
|
1670
|
+
}
|
|
1671
|
+
return true;
|
|
1672
|
+
};
|
|
1673
|
+
/**
|
|
1674
|
+
* Redrive up to `redriveCount` bodies from the configured {@link WorkSystemOptions.dlq} back onto
|
|
1675
|
+
* their type's queue as **fresh** work — `attempt` reset to 1 (full retry budget), `queueAt`/`startAt`
|
|
1676
|
+
* = now, a fresh group hold re-opened — then ack each off the DLQ. Called only when the normal queues
|
|
1677
|
+
* are idle and capacity is free. An unparseable body is dropped (acked); a re-queue failure leaves the
|
|
1678
|
+
* body leased on the DLQ to retry on a later idle tick. Returns how many were moved.
|
|
1679
|
+
*/
|
|
1680
|
+
const redriveCount = opts.redriveCount ?? REDRIVE_DEFAULT;
|
|
1681
|
+
const redriveFromDLQ = async () => {
|
|
1682
|
+
const dlq = opts.dlq;
|
|
1683
|
+
if (!dlq || redriveCount <= 0) return 0;
|
|
1684
|
+
const pulled = await Promise.resolve(dlq.pop(redriveCount, visibility));
|
|
1685
|
+
let moved = 0;
|
|
1686
|
+
for (const p of pulled) {
|
|
1687
|
+
let env;
|
|
1688
|
+
try {
|
|
1689
|
+
env = JSON.parse(p.body);
|
|
1690
|
+
} catch {
|
|
1691
|
+
await Promise.resolve(dlq.ack(p)).catch((err) => report(err, "queue"));
|
|
1692
|
+
continue;
|
|
1693
|
+
}
|
|
1694
|
+
const at = now();
|
|
1695
|
+
const fresh = {
|
|
1696
|
+
...env,
|
|
1697
|
+
attempt: 1,
|
|
1698
|
+
queueAt: at,
|
|
1699
|
+
startAt: at
|
|
1700
|
+
};
|
|
1701
|
+
try {
|
|
1702
|
+
await groupIncr(env.groupId, 1);
|
|
1703
|
+
try {
|
|
1704
|
+
await setState({
|
|
1705
|
+
id: env.id,
|
|
1706
|
+
type: env.type,
|
|
1707
|
+
status: "pending",
|
|
1708
|
+
attempt: 1,
|
|
1709
|
+
queueAt: at,
|
|
1710
|
+
startAt: at,
|
|
1711
|
+
priority: env.priority,
|
|
1712
|
+
group: env.group
|
|
1713
|
+
}, env.groupId);
|
|
1714
|
+
await pushEnvelope(fresh, 0);
|
|
1715
|
+
} catch (err) {
|
|
1716
|
+
await undoHold(env.groupId);
|
|
1717
|
+
throw err;
|
|
1718
|
+
}
|
|
1719
|
+
await port(() => Promise.resolve(dlq.ack(p)));
|
|
1720
|
+
stats.queued(env.type, at);
|
|
1721
|
+
emit({
|
|
1722
|
+
kind: "queued",
|
|
1723
|
+
id: env.id,
|
|
1724
|
+
type: env.type,
|
|
1725
|
+
groupId: env.groupId,
|
|
1726
|
+
at
|
|
1727
|
+
});
|
|
1728
|
+
moved += 1;
|
|
1729
|
+
} catch (err) {
|
|
1730
|
+
report(err, "queue");
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
return moved;
|
|
1734
|
+
};
|
|
1735
|
+
let running = false;
|
|
1736
|
+
let loopPromise = null;
|
|
1737
|
+
const scheduleCancels = /* @__PURE__ */ new Set();
|
|
1738
|
+
const pollCount = () => {
|
|
1739
|
+
const sources = new Set([globalDoer]);
|
|
1740
|
+
for (const b of registry.values()) if (b.def.batch) sources.add(batcherFor(b.type));
|
|
1741
|
+
else if (b.def.options.doer) sources.add(b.def.options.doer);
|
|
1742
|
+
let n = 0;
|
|
1743
|
+
for (const s of sources) n += s.available();
|
|
1744
|
+
return Math.min(POLL_BATCH_CAP, n);
|
|
1745
|
+
};
|
|
1746
|
+
let pollCursor = 0;
|
|
1747
|
+
const loop = async () => {
|
|
1748
|
+
while (running) try {
|
|
1749
|
+
const pause = (opts.backpressure ? await opts.backpressure({
|
|
1750
|
+
metrics: stats.metrics,
|
|
1751
|
+
active: activeMap.size
|
|
1752
|
+
}) : 0) ?? 0;
|
|
1753
|
+
if (pause > 0) {
|
|
1754
|
+
await sleep(pause);
|
|
1755
|
+
continue;
|
|
1756
|
+
}
|
|
1757
|
+
const n = pollCount();
|
|
1758
|
+
if (n <= 0) {
|
|
1759
|
+
await sleep(pollInterval);
|
|
1760
|
+
continue;
|
|
1761
|
+
}
|
|
1762
|
+
const qs = distinctQueues();
|
|
1763
|
+
const share = Math.max(1, Math.ceil(n / qs.length));
|
|
1764
|
+
let accepted = 0;
|
|
1765
|
+
let anyFull = false;
|
|
1766
|
+
let pulledTotal = 0;
|
|
1767
|
+
for (let i = 0; i < qs.length; i++) {
|
|
1768
|
+
const q = qs[(pollCursor + i) % qs.length];
|
|
1769
|
+
const pulled = await Promise.resolve(q.pop(share, visibility));
|
|
1770
|
+
pulledTotal += pulled.length;
|
|
1771
|
+
if (pulled.length >= share) anyFull = true;
|
|
1772
|
+
const started = await Promise.all(pulled.map((p) => accept(p, q).catch((err) => (report(err, "queue"), false))));
|
|
1773
|
+
accepted += started.filter(Boolean).length;
|
|
1774
|
+
}
|
|
1775
|
+
pollCursor = (pollCursor + 1) % qs.length;
|
|
1776
|
+
if (pulledTotal === 0 && opts.dlq && await redriveFromDLQ() > 0) continue;
|
|
1777
|
+
if (accepted === 0 || !anyFull) await sleep(pollInterval);
|
|
1778
|
+
} catch (err) {
|
|
1779
|
+
report(err, "queue");
|
|
1780
|
+
await sleep(pollInterval);
|
|
1781
|
+
}
|
|
1782
|
+
};
|
|
1783
|
+
registry.set(DEPENDENCY_TYPE, defineWork(DEPENDENCY_TYPE, dependencyHandler, { retry: { attempts: DEP_RETRY_ATTEMPTS } }));
|
|
1784
|
+
const start = () => {
|
|
1785
|
+
if (running) return;
|
|
1786
|
+
running = true;
|
|
1787
|
+
loopPromise = loop();
|
|
1788
|
+
};
|
|
1789
|
+
const stop = async () => {
|
|
1790
|
+
running = false;
|
|
1791
|
+
for (const cancel of scheduleCancels) cancel();
|
|
1792
|
+
scheduleCancels.clear();
|
|
1793
|
+
await loopPromise?.catch(() => {});
|
|
1794
|
+
loopPromise = null;
|
|
1795
|
+
for (const b of batchers.values()) b.flushAll();
|
|
1796
|
+
const drained = Promise.all([...allDoers()].map((d) => d.done())).then(() => void 0);
|
|
1797
|
+
await Promise.race([drained, sleep(STOP_DRAIN)]);
|
|
1798
|
+
};
|
|
1799
|
+
const allDoers = () => {
|
|
1800
|
+
const set = new Set([globalDoer]);
|
|
1801
|
+
for (const b of registry.values()) {
|
|
1802
|
+
const d = b.def.options.doer;
|
|
1803
|
+
if (d) set.add(d);
|
|
1804
|
+
}
|
|
1805
|
+
return set;
|
|
1806
|
+
};
|
|
1807
|
+
const schedule = (config) => {
|
|
1808
|
+
const cancel = startSchedule(config, {
|
|
1809
|
+
store,
|
|
1810
|
+
now,
|
|
1811
|
+
enqueueRaw,
|
|
1812
|
+
prefix,
|
|
1813
|
+
tick: SCHED_TICK,
|
|
1814
|
+
leaseTtl: SCHED_LEASE_TTL
|
|
1815
|
+
});
|
|
1816
|
+
scheduleCancels.add(cancel);
|
|
1817
|
+
return () => {
|
|
1818
|
+
cancel();
|
|
1819
|
+
scheduleCancels.delete(cancel);
|
|
1820
|
+
};
|
|
1821
|
+
};
|
|
1822
|
+
const active = () => [...activeMap.values()].map((a) => ({ ...a }));
|
|
1823
|
+
const api = {
|
|
1824
|
+
enqueue: enqueueImpl,
|
|
1825
|
+
schedule,
|
|
1826
|
+
start,
|
|
1827
|
+
stop,
|
|
1828
|
+
list: () => Promise.resolve([...allStates.values()]),
|
|
1829
|
+
active,
|
|
1830
|
+
stats: () => stats.metrics.list(),
|
|
1831
|
+
metrics: stats.metrics,
|
|
1832
|
+
backend
|
|
1833
|
+
};
|
|
1834
|
+
if (opts.autoStart !== false) start();
|
|
1835
|
+
return api;
|
|
1836
|
+
}
|
|
1837
|
+
//#endregion
|
|
1838
|
+
//#region src/adaptive.ts
|
|
1839
|
+
/**
|
|
1840
|
+
* Build an {@link AdaptiveDelay} controller for {@link WorkSystemOptions.backpressure}. It holds a
|
|
1841
|
+
* little state (the current pause + the last cumulative counts) across calls, so create **one** per
|
|
1842
|
+
* work system. Watching a subset of `types` lets one system protect a specific downstream.
|
|
1843
|
+
*/
|
|
1844
|
+
function adaptiveDelay(opts = {}) {
|
|
1845
|
+
const maxFailRate = opts.maxFailRate ?? 0;
|
|
1846
|
+
const min = opts.min ?? 0;
|
|
1847
|
+
const max = opts.max ?? 3e4;
|
|
1848
|
+
const base = opts.base ?? 100;
|
|
1849
|
+
const factor = opts.factor ?? 2;
|
|
1850
|
+
const step = opts.step ?? base;
|
|
1851
|
+
let delay = min;
|
|
1852
|
+
let prevCompleted = 0;
|
|
1853
|
+
let prevFailed = 0;
|
|
1854
|
+
const watch = opts.types ? new Set(opts.types) : null;
|
|
1855
|
+
return (ctx) => {
|
|
1856
|
+
let succeeded = 0;
|
|
1857
|
+
let failed = 0;
|
|
1858
|
+
for (const s of ctx.metrics.list()) {
|
|
1859
|
+
const type = s.labels.type;
|
|
1860
|
+
if (watch && (type === void 0 || !watch.has(type))) continue;
|
|
1861
|
+
if (s.meta.name === WORK_METRICS.succeeded) succeeded += s.value;
|
|
1862
|
+
else if (s.meta.name === WORK_METRICS.failed) failed += s.value;
|
|
1863
|
+
}
|
|
1864
|
+
const completed = succeeded + failed;
|
|
1865
|
+
const dCompleted = completed - prevCompleted;
|
|
1866
|
+
const dFailed = failed - prevFailed;
|
|
1867
|
+
prevCompleted = completed;
|
|
1868
|
+
prevFailed = failed;
|
|
1869
|
+
if ((dCompleted > 0 ? dFailed / dCompleted : 0) > maxFailRate) delay = Math.min(max, Math.max(base, delay === 0 ? base : delay * factor));
|
|
1870
|
+
else delay = Math.max(min, delay - step);
|
|
1871
|
+
return delay;
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
//#endregion
|
|
1875
|
+
//#region src/index.ts
|
|
1876
|
+
/**
|
|
1877
|
+
* # @ayepi/work
|
|
1878
|
+
*
|
|
1879
|
+
* Type-safe distributed work / job-queue + workflow engine. Define work types with
|
|
1880
|
+
* {@link defineWork} (each yields a typed, queueable builder), pass them to
|
|
1881
|
+
* {@link createWork} as a `const` registry, and `enqueue` is fully checked — by
|
|
1882
|
+
* instance or by name. Awaiting a handle resolves to the **group result**; work queued
|
|
1883
|
+
* inside a handler joins the same group.
|
|
1884
|
+
*
|
|
1885
|
+
* Built on three pluggable ports ({@link Queue}/{@link PubSub}/{@link Store}) with an
|
|
1886
|
+
* in-memory implementation bundled, so it runs zero-config and scales out by swapping
|
|
1887
|
+
* the ports for Redis/SQS/etc. Retries (backoff + jitter, by re-enqueue), batching,
|
|
1888
|
+
* non-blocking {@link dependency} gates, cron/fn {@link ScheduleConfig}, distributed
|
|
1889
|
+
* wait-for-result, in-process `skipQueue` execution, doers for concurrency/ordering,
|
|
1890
|
+
* an orphan-group hook, a JSON codec, and a `logWith` hook are all included.
|
|
1891
|
+
*
|
|
1892
|
+
* ```ts
|
|
1893
|
+
* import { defineWork, createWork } from '@ayepi/work'
|
|
1894
|
+
*
|
|
1895
|
+
* const add = defineWork('add', (i: { a: number; b: number }) => i.a + i.b)
|
|
1896
|
+
* const w = createWork({ work: [add] as const })
|
|
1897
|
+
*
|
|
1898
|
+
* const sum = await w.enqueue(add({ a: 1, b: 2 })).result() // 3, typed as number
|
|
1899
|
+
* await w.stop()
|
|
1900
|
+
* ```
|
|
1901
|
+
*
|
|
1902
|
+
* Bare `import` has **no side effects** — the default instance does not auto-start.
|
|
1903
|
+
*
|
|
1904
|
+
* @module
|
|
1905
|
+
*/
|
|
1906
|
+
const instance = createWork({ autoStart: false });
|
|
1907
|
+
/** The default (registry-less) work system. Most apps call {@link createWork} with their own registry instead. */
|
|
1908
|
+
const work = instance;
|
|
1909
|
+
/** Enqueue on the default system (instance form). */
|
|
1910
|
+
const enqueue = instance.enqueue;
|
|
1911
|
+
/** Register a recurring schedule on the default system. */
|
|
1912
|
+
const schedule = instance.schedule;
|
|
1913
|
+
/** Start the default system's worker loop. */
|
|
1914
|
+
const start = () => instance.start();
|
|
1915
|
+
/** Stop the default system. */
|
|
1916
|
+
const stop = () => instance.stop();
|
|
1917
|
+
/** Snapshot the default system's known work states. */
|
|
1918
|
+
const list = () => instance.list();
|
|
1919
|
+
//#endregion
|
|
1920
|
+
export { DEFAULT_BUCKETS, DEFAULT_RETRY_OPTIONS, DEPENDENCY_TYPE, RetryAbort, WORK_METRICS, WorkDelayError, adaptiveDelay, ageDoer, backoff, balancedDoer, conditionMet, createMetrics, createWork, defaultCodec, defineBatchWork, defineWork, dependency, enqueue, formatPrometheus, getDefaultRetryOptions, list, memoryBackend, memoryPubSub, memoryQueue, memoryStore, nextAfter, parseCron, priorityDoer, retry, schedule, setDefaultRetryOptions, start, stop, unlimitedDoer, work };
|