@ayepi/work 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,1920 @@
1
+ import { ageDoer, balancedDoer, priorityDoer, unlimitedDoer, unlimitedDoer as unlimitedDoer$1 } from "@ayepi/core/doer";
2
+ import { DEFAULT_RETRY_OPTIONS, RetryAbort, RetryAbort as RetryAbort$1, backoff, backoff as backoff$1, getDefaultRetryOptions, getDefaultRetryOptions as getDefaultRetryOptions$1, retry, retry as retry$1, setDefaultRetryOptions } from "@ayepi/core/retry";
3
+ import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
4
+ import { dirname } from "node:path";
5
+ import { randomUUID } from "node:crypto";
6
+ import { DEFAULT_BUCKETS, createMetrics, createMetrics as createMetrics$1, formatPrometheus } from "@ayepi/core/stats";
7
+ //#region src/json.ts
8
+ /** Wrapper key marking a tagged value (`{ [TAG]: 'Date', value: … }`). */
9
+ const TAG = "$ayepi";
10
+ const isTagged = (v) => v !== null && typeof v === "object" && TAG in v;
11
+ /**
12
+ * The default {@link JsonCodec}. Tags values JSON can't represent natively so they
13
+ * survive a `stringify` → `parse` round-trip:
14
+ *
15
+ * | Value | Encoded as |
16
+ * |-------------|-------------------------------------|
17
+ * | `undefined` | `{ $ayepi:'undefined' }` |
18
+ * | `bigint` | `{ $ayepi:'BigInt', value:'123' }` |
19
+ * | `Date` | `{ $ayepi:'Date', value:<iso> }` |
20
+ * | `Map` | `{ $ayepi:'Map', value:[[k,v]…] }` |
21
+ * | `Set` | `{ $ayepi:'Set', value:[…] }` |
22
+ * | `Error` | `{ $ayepi:'Error', value:{name,message,stack} }` |
23
+ */
24
+ const defaultCodec = {
25
+ stringify(value) {
26
+ return JSON.stringify(value, function(key, encoded) {
27
+ const raw = this[key];
28
+ if (raw === void 0) return { [TAG]: "undefined" };
29
+ if (typeof raw === "bigint") return {
30
+ [TAG]: "BigInt",
31
+ value: raw.toString()
32
+ };
33
+ if (raw instanceof Date) return {
34
+ [TAG]: "Date",
35
+ value: raw.toISOString()
36
+ };
37
+ if (raw instanceof Map) return {
38
+ [TAG]: "Map",
39
+ value: [...raw.entries()]
40
+ };
41
+ if (raw instanceof Set) return {
42
+ [TAG]: "Set",
43
+ value: [...raw.values()]
44
+ };
45
+ if (raw instanceof Error) return {
46
+ [TAG]: "Error",
47
+ value: {
48
+ name: raw.name,
49
+ message: raw.message,
50
+ stack: raw.stack
51
+ }
52
+ };
53
+ return encoded;
54
+ });
55
+ },
56
+ parse(text) {
57
+ return JSON.parse(text, (_key, value) => {
58
+ if (!isTagged(value)) return value;
59
+ switch (value[TAG]) {
60
+ case "undefined": return;
61
+ case "BigInt": return BigInt(value.value);
62
+ case "Date": return new Date(value.value);
63
+ case "Map": return new Map(value.value);
64
+ case "Set": return new Set(value.value);
65
+ case "Error": {
66
+ const e = value.value;
67
+ const err = new Error(e.message);
68
+ err.name = e.name;
69
+ if (e.stack) err.stack = e.stack;
70
+ return err;
71
+ }
72
+ default: return value;
73
+ }
74
+ });
75
+ }
76
+ };
77
+ //#endregion
78
+ //#region src/internal.ts
79
+ /**
80
+ * # Internals
81
+ *
82
+ * Dependency-free helpers shared across the engine: id generation, the
83
+ * backoff-with-jitter formula, an async sleep, a shallow merge, and the identity
84
+ * `logWith` (so `@ayepi/log` is injected, never imported).
85
+ *
86
+ * @module
87
+ */
88
+ /** A v4 UUID. Thin wrapper over `node:crypto` so callers don't import it directly. */
89
+ const uuid = () => randomUUID();
90
+ /** Resolve after `ms` (an unref'd timer, so it never keeps the process alive). */
91
+ function sleep(ms) {
92
+ return new Promise((resolve) => {
93
+ setTimeout(resolve, ms).unref?.();
94
+ });
95
+ }
96
+ /** Shallow merge of two objects into a new object (`b` wins on collisions). */
97
+ function merge(a, b) {
98
+ return {
99
+ ...a ?? {},
100
+ ...b ?? {}
101
+ };
102
+ }
103
+ /** The default {@link LogWith}: runs `inner` with no added context. */
104
+ const identityLogWith = (_add, inner) => inner();
105
+ //#endregion
106
+ //#region src/memory.ts
107
+ /**
108
+ * # In-memory backend
109
+ *
110
+ * A zero-dependency implementation of all three {@link Backend} ports that
111
+ * **simulates the distributed protocol** — visibility-timeout leases with
112
+ * heartbeat-driven redelivery, TTL'd store with an atomic `setIfNotExists`, and
113
+ * in-process fanout. Share one backend between several `createWork` instances to model
114
+ * a multi-pod deployment in tests, and inject `now` to drive time deterministically.
115
+ *
116
+ * The {@link memoryQueue} can additionally be **file-backed** (`file: './work-queue.json'`)
117
+ * so pending work survives a process restart — single-process durability without standing up
118
+ * Redis/SQS. State is written atomically (temp file + rename) after every mutation; on startup
119
+ * the file is reloaded and any in-flight (leased) item is redelivered, since the worker that
120
+ * held its lease is gone.
121
+ *
122
+ * @module
123
+ */
124
+ /** The default {@link QueueFsLike}, backed by synchronous `node:fs`. */
125
+ const nodeQueueFs = {
126
+ readFile: (p) => existsSync(p) ? readFileSync(p, "utf8") : void 0,
127
+ writeFile: (p, d) => writeFileSync(p, d),
128
+ rename: (a, b) => renameSync(a, b),
129
+ mkdir: (p) => void mkdirSync(p, { recursive: true })
130
+ };
131
+ /**
132
+ * In-process {@link PubSub} (the `localBroker` shape). Share one instance across
133
+ * engines to fan a publish out to every "pod".
134
+ */
135
+ function memoryPubSub() {
136
+ const listeners = /* @__PURE__ */ new Set();
137
+ return {
138
+ publish(m) {
139
+ for (const l of [...listeners]) l(m);
140
+ },
141
+ subscribe(l) {
142
+ listeners.add(l);
143
+ return () => void listeners.delete(l);
144
+ }
145
+ };
146
+ }
147
+ /**
148
+ * In-memory {@link Store} with lazy TTL expiry and an atomic {@link Store.setIfNotExists}.
149
+ * The compare-and-set every distributed claim relies on.
150
+ */
151
+ function memoryStore(opts = {}) {
152
+ const now = opts.now ?? Date.now;
153
+ const map = /* @__PURE__ */ new Map();
154
+ const live = (key) => {
155
+ const e = map.get(key);
156
+ if (!e) return;
157
+ if (e.expires !== void 0 && e.expires <= now()) {
158
+ map.delete(key);
159
+ return;
160
+ }
161
+ return e;
162
+ };
163
+ return {
164
+ get: (key) => live(key)?.value,
165
+ set: (key, value, ttl) => void map.set(key, {
166
+ value,
167
+ expires: ttl ? now() + ttl : void 0
168
+ }),
169
+ delete: (key) => void map.delete(key),
170
+ setIfNotExists: (key, value, ttl) => {
171
+ if (live(key)) return false;
172
+ map.set(key, {
173
+ value,
174
+ expires: ttl ? now() + ttl : void 0
175
+ });
176
+ return true;
177
+ },
178
+ increment: (key, by, ttl) => {
179
+ const cur = Number(live(key)?.value ?? "0") + by;
180
+ map.set(key, {
181
+ value: String(cur),
182
+ expires: ttl ? now() + ttl : void 0
183
+ });
184
+ return cur;
185
+ }
186
+ };
187
+ }
188
+ /**
189
+ * In-memory {@link Queue} with visibility-timeout leasing. `pop` first **reclaims**
190
+ * items whose lease expired (a dead worker → redelivery, `attempt++`), then leases
191
+ * fresh visible items. `ack`/`heartbeat`/`fail` are token-gated, so a stale worker
192
+ * whose lease already lapsed cannot ack work another worker now owns.
193
+ *
194
+ * Pass `file` to make it **durable**: state is reloaded on construction and rewritten
195
+ * atomically after every mutation. A heartbeat is *not* persisted (lease expiry is
196
+ * reset on reload anyway), so steady-state heartbeating doesn't touch the disk.
197
+ */
198
+ function memoryQueue(opts = {}) {
199
+ const now = opts.now ?? Date.now;
200
+ const items = [];
201
+ const dead = [];
202
+ const find = (token) => items.find((i) => i.leaseToken === token);
203
+ const file = opts.file;
204
+ const fs = opts.fs ?? nodeQueueFs;
205
+ const tmpFile = file !== void 0 ? `${file}.tmp` : "";
206
+ let dirEnsured = false;
207
+ const report = (err) => {
208
+ try {
209
+ opts.onError?.(err);
210
+ } catch {}
211
+ };
212
+ const persist = () => {
213
+ if (file === void 0) return;
214
+ try {
215
+ if (!dirEnsured) {
216
+ const dir = dirname(file);
217
+ if (dir !== ".") fs.mkdir(dir);
218
+ dirEnsured = true;
219
+ }
220
+ fs.writeFile(tmpFile, JSON.stringify({
221
+ items,
222
+ dead
223
+ }));
224
+ fs.rename(tmpFile, file);
225
+ } catch (err) {
226
+ report(err);
227
+ }
228
+ };
229
+ const load = () => {
230
+ if (file === void 0) return;
231
+ let raw;
232
+ try {
233
+ raw = fs.readFile(file);
234
+ } catch (err) {
235
+ report(err);
236
+ return;
237
+ }
238
+ if (raw === void 0) return;
239
+ try {
240
+ const parsed = JSON.parse(raw);
241
+ for (const i of parsed.items ?? []) {
242
+ if (i.leaseToken !== void 0) {
243
+ i.leaseToken = void 0;
244
+ i.leaseUntil = void 0;
245
+ i.attempt += 1;
246
+ i.visibleAt = now();
247
+ }
248
+ items.push(i);
249
+ }
250
+ for (const d of parsed.dead ?? []) dead.push(d);
251
+ } catch (err) {
252
+ report(err);
253
+ }
254
+ };
255
+ load();
256
+ return {
257
+ dead,
258
+ size: () => items.length,
259
+ push(body, o) {
260
+ if (o?.dedupeKey && items.some((i) => i.dedupeKey === o.dedupeKey)) return;
261
+ items.push({
262
+ body,
263
+ visibleAt: now() + (o?.delay ?? 0),
264
+ attempt: 0,
265
+ dedupeKey: o?.dedupeKey
266
+ });
267
+ persist();
268
+ },
269
+ pop(max, visibility) {
270
+ const t = now();
271
+ let changed = false;
272
+ for (const i of items) if (i.leaseToken && i.leaseUntil !== void 0 && i.leaseUntil <= t) {
273
+ i.leaseToken = void 0;
274
+ i.leaseUntil = void 0;
275
+ i.attempt += 1;
276
+ changed = true;
277
+ }
278
+ const out = [];
279
+ for (const i of items) {
280
+ if (out.length >= max) break;
281
+ if (i.leaseToken || i.visibleAt > t) continue;
282
+ const token = uuid();
283
+ i.leaseToken = token;
284
+ i.leaseUntil = t + visibility;
285
+ out.push({
286
+ body: i.body,
287
+ handle: token,
288
+ attempt: i.attempt + 1
289
+ });
290
+ changed = true;
291
+ }
292
+ if (changed) persist();
293
+ return out;
294
+ },
295
+ heartbeat(p, visibility) {
296
+ const i = find(p.handle);
297
+ if (i) i.leaseUntil = now() + visibility;
298
+ },
299
+ ack(p) {
300
+ const idx = items.findIndex((i) => i.leaseToken === p.handle);
301
+ if (idx >= 0) {
302
+ items.splice(idx, 1);
303
+ persist();
304
+ }
305
+ },
306
+ fail(p, delay) {
307
+ const i = find(p.handle);
308
+ if (!i) return;
309
+ i.leaseToken = void 0;
310
+ i.leaseUntil = void 0;
311
+ i.attempt = p.attempt;
312
+ i.visibleAt = now() + (delay ?? 0);
313
+ persist();
314
+ },
315
+ deadLetter(body, error) {
316
+ dead.push({
317
+ body,
318
+ error
319
+ });
320
+ persist();
321
+ }
322
+ };
323
+ }
324
+ /**
325
+ * The three in-memory ports together, sharing one clock. The default backend when
326
+ * `createWork()` is called with no ports.
327
+ *
328
+ * @example A two-pod test on one shared backend:
329
+ * ```ts
330
+ * const backend = memoryBackend()
331
+ * const podA = createWork({ ...backend, work: [add] })
332
+ * const podB = createWork({ ...backend, work: [add] })
333
+ * ```
334
+ *
335
+ * @example A single durable process — pending work survives a restart:
336
+ * ```ts
337
+ * const backend = memoryBackend({ queue: { file: './work-queue.json' } })
338
+ * const work = createWork({ ...backend, work: [add] })
339
+ * ```
340
+ */
341
+ function memoryBackend(opts = {}) {
342
+ return {
343
+ queue: memoryQueue({
344
+ now: opts.now,
345
+ ...opts.queue
346
+ }),
347
+ pubsub: memoryPubSub(),
348
+ store: memoryStore(opts)
349
+ };
350
+ }
351
+ //#endregion
352
+ //#region src/errors.ts
353
+ /**
354
+ * Throw from a work handler to **defer** the item to a future time (a reschedule, not a failure).
355
+ *
356
+ * ```ts
357
+ * defineWork('poll', async (input, ctx) => {
358
+ * if (!(await upstreamReady())) throw new WorkDelayError({ delay: 5 * 60_000 }); // retry in 5 min, attempt unchanged
359
+ * return doWork(input);
360
+ * });
361
+ * ```
362
+ */
363
+ var WorkDelayError = class extends Error {
364
+ when;
365
+ constructor(when, message = "work deferred") {
366
+ super(message);
367
+ this.when = when;
368
+ this.name = "WorkDelayError";
369
+ }
370
+ };
371
+ //#endregion
372
+ //#region src/dependency.ts
373
+ /**
374
+ * # Dependencies
375
+ *
376
+ * A dependency is **itself a work item** ({@link DEPENDENCY_TYPE}) — so it lives on the
377
+ * durable queue and survives a service disruption like any other work. Its handler is
378
+ * **non-blocking**: each run reads the state of the works it waits `on`, and either
379
+ *
380
+ * - fires (queues its `queue` dependents, once, under a distributed {@link WorkContext.claim}), or
381
+ * - **re-queues itself** with a small delay to check again later.
382
+ *
383
+ * It never holds a worker slot waiting, so a backlog of dependencies can't starve other
384
+ * work. Build one with {@link dependency} and enqueue it like anything else — typically
385
+ * alongside the works it depends on:
386
+ *
387
+ * ```ts
388
+ * const a = stepA(), b = stepB()
389
+ * ctx.queue([a, b, dependency({ on: [a, b], queue: [finalize()], config: 'all-success' })])
390
+ * ```
391
+ *
392
+ * @module
393
+ */
394
+ /** The built-in work type name for a dependency. */
395
+ const DEPENDENCY_TYPE = "@work/dependency";
396
+ /** Default re-check interval (ms). */
397
+ const DEFAULT_POLL = 1e3;
398
+ const TERMINAL = new Set([
399
+ "success",
400
+ "failed",
401
+ "dead"
402
+ ]);
403
+ const isTerminal = (s) => s !== void 0 && TERMINAL.has(s.status);
404
+ const isSuccess = (s) => s?.status === "success";
405
+ /**
406
+ * Evaluate a {@link DependencyCondition} against the watched items' states. A missing
407
+ * state counts as "not yet done". Pure and JSON-driven, so every instance agrees.
408
+ */
409
+ function conditionMet(condition, states) {
410
+ if (condition === "all-done") return states.every(isTerminal);
411
+ if (condition === "all-success") return states.every(isSuccess);
412
+ const pred = condition.of === "success" ? isSuccess : isTerminal;
413
+ return states.filter(pred).length >= condition.count;
414
+ }
415
+ const toId = (w) => typeof w === "string" ? w : w.id;
416
+ const toSerialized = (w) => ({
417
+ id: w.id,
418
+ type: w.type,
419
+ input: w.input
420
+ });
421
+ const rehydrate = (s) => ({
422
+ id: s.id,
423
+ type: s.type,
424
+ input: s.input
425
+ });
426
+ /** Build a {@link DEPENDENCY_TYPE} work item from a (re-usable) input — a fresh queue id, same key. */
427
+ const buildDependency = (input) => ({
428
+ id: uuid(),
429
+ type: DEPENDENCY_TYPE,
430
+ input
431
+ });
432
+ /**
433
+ * Build a dependency: when the works it waits `on` satisfy `config`, it queues its
434
+ * `queue` dependents (once). Enqueue it like any work.
435
+ */
436
+ function dependency(opts) {
437
+ return buildDependency({
438
+ key: uuid(),
439
+ on: opts.on.map(toId),
440
+ queue: opts.queue.map(toSerialized),
441
+ config: opts.config ?? "all-success",
442
+ poll: opts.poll ?? DEFAULT_POLL,
443
+ deadline: opts.timeout !== void 0 ? Date.now() + opts.timeout : void 0
444
+ });
445
+ }
446
+ /**
447
+ * The non-blocking handler for {@link DEPENDENCY_TYPE}: check once, then fire or
448
+ * re-queue. Registered automatically by every work system. Remembers terminal statuses
449
+ * (`resolved`) so it neither re-reads settled works nor treats an evicted state as a
450
+ * failure.
451
+ */
452
+ const dependencyHandler = async (input, ctx) => {
453
+ const resolved = { ...input.resolved };
454
+ const unknownIds = input.on.filter((id) => !(id in resolved));
455
+ const fresh = await ctx.states(unknownIds);
456
+ fresh.forEach((s, i) => {
457
+ if (s && TERMINAL.has(s.status)) resolved[unknownIds[i]] = s.status;
458
+ });
459
+ const freshById = new Map(unknownIds.map((id, i) => [id, fresh[i]]));
460
+ const states = input.on.map((id) => resolved[id] ? { status: resolved[id] } : freshById.get(id));
461
+ if (conditionMet(input.config, states)) {
462
+ if (await ctx.claim(`dep:${input.key}:fired`)) ctx.queue(input.queue.map(rehydrate));
463
+ return;
464
+ }
465
+ if (input.deadline !== void 0 && Date.now() >= input.deadline) throw new Error(`dependency: timed out waiting for [${input.on.join(", ")}]`);
466
+ ctx.queue(buildDependency({
467
+ ...input,
468
+ resolved
469
+ }), { delay: input.poll });
470
+ };
471
+ //#endregion
472
+ //#region src/schedule.ts
473
+ const MS_PER_MINUTE = 6e4;
474
+ /** Cap the forward scan so a never-matching expression can't loop forever (~1 year). */
475
+ const MAX_SCAN_MINUTES = 366 * 24 * 60;
476
+ /** `[min, max]` inclusive range for each of the five fields. */
477
+ const FIELD_BOUNDS = [
478
+ [0, 59],
479
+ [0, 23],
480
+ [1, 31],
481
+ [1, 12],
482
+ [0, 6]
483
+ ];
484
+ /** Expand one cron field — `*`, a number, an `a-b` range, a `<range>/<step>`, or a comma list — into the set of matching numbers. */
485
+ function parseField(field, min, max) {
486
+ const out = /* @__PURE__ */ new Set();
487
+ for (const part of field.split(",")) {
488
+ const [rangePart, stepPart] = part.split("/");
489
+ const step = stepPart ? Number(stepPart) : 1;
490
+ if (!Number.isInteger(step) || step < 1) throw new Error(`cron: bad step in "${part}"`);
491
+ let lo = min;
492
+ let hi = max;
493
+ if (rangePart !== "*" && rangePart !== "") {
494
+ const [a, b] = rangePart.split("-");
495
+ lo = Number(a);
496
+ hi = b !== void 0 ? Number(b) : lo;
497
+ if (!Number.isInteger(lo) || !Number.isInteger(hi) || lo < min || hi > max || lo > hi) throw new Error(`cron: bad range "${part}"`);
498
+ }
499
+ for (let v = lo; v <= hi; v += step) out.add(v);
500
+ }
501
+ return out;
502
+ }
503
+ /** Parse a 5-field cron expression (`min hour dom mon dow`). */
504
+ function parseCron(expr) {
505
+ const fields = expr.trim().split(/\s+/);
506
+ if (fields.length !== 5) throw new Error(`cron: expected 5 fields, got ${fields.length} in "${expr}"`);
507
+ const [minute, hour, dom, month, dow] = fields.map((f, i) => parseField(f, FIELD_BOUNDS[i][0], FIELD_BOUNDS[i][1]));
508
+ return {
509
+ minute,
510
+ hour,
511
+ dom,
512
+ month,
513
+ dow,
514
+ domRestricted: fields[2] !== "*",
515
+ dowRestricted: fields[4] !== "*"
516
+ };
517
+ }
518
+ /** Does `date` (local time) satisfy the cron fields? Standard dom/dow OR semantics when both are restricted. */
519
+ function matches(c, date) {
520
+ if (!c.minute.has(date.getMinutes()) || !c.hour.has(date.getHours()) || !c.month.has(date.getMonth() + 1)) return false;
521
+ const domOk = c.dom.has(date.getDate());
522
+ const dowOk = c.dow.has(date.getDay());
523
+ if (c.domRestricted && c.dowRestricted) return domOk || dowOk;
524
+ if (c.domRestricted) return domOk;
525
+ if (c.dowRestricted) return dowOk;
526
+ return true;
527
+ }
528
+ /**
529
+ * The next epoch-ms strictly after `fromMs` that matches `expr`, or `undefined` if
530
+ * none within ~a year. Minute-granular (cron's resolution).
531
+ */
532
+ function nextAfter(expr, fromMs) {
533
+ const c = parseCron(expr);
534
+ const start = /* @__PURE__ */ new Date(Math.floor(fromMs / MS_PER_MINUTE) * MS_PER_MINUTE + MS_PER_MINUTE);
535
+ start.setSeconds(0, 0);
536
+ for (let i = 0; i < MAX_SCAN_MINUTES; i++) {
537
+ const candidate = new Date(start.getTime() + i * MS_PER_MINUTE);
538
+ if (matches(c, candidate)) return candidate.getTime();
539
+ }
540
+ }
541
+ const unref$1 = (t) => void t.unref?.();
542
+ /** Compute the next fire time (epoch ms) for a schedule, or `undefined` to stop. */
543
+ function computeNext(config, from) {
544
+ if (config.cron !== void 0) return nextAfter(config.cron, from);
545
+ if (config.next) {
546
+ const r = config.next(from);
547
+ if (r === void 0 || r === null) return;
548
+ return r instanceof Date ? r.getTime() : r;
549
+ }
550
+ throw new Error(`schedule "${config.name}": provide either "cron" or "next"`);
551
+ }
552
+ /**
553
+ * Start a schedule's tick loop. Returns a cancel function. One instance fires per
554
+ * occurrence (claimed via a `setNX` lease keyed by the fire's second-bucket).
555
+ */
556
+ function startSchedule(config, deps) {
557
+ let cancelled = false;
558
+ let nextAt = computeNext(config, deps.now());
559
+ let timer;
560
+ const tick = async () => {
561
+ if (cancelled) return;
562
+ const t = deps.now();
563
+ if (nextAt !== void 0 && t >= nextAt) {
564
+ const bucket = Math.floor(nextAt / 1e3);
565
+ if (await deps.store.setIfNotExists(`${deps.prefix}sched:${config.name}:${bucket}`, "1", deps.leaseTtl)) {
566
+ const inst = config.run();
567
+ if (inst) deps.enqueueRaw(inst.type, inst.input);
568
+ }
569
+ nextAt = computeNext(config, deps.now());
570
+ }
571
+ if (!cancelled) {
572
+ timer = setTimeout(() => void tick(), deps.tick);
573
+ unref$1(timer);
574
+ }
575
+ };
576
+ timer = setTimeout(() => void tick(), 0);
577
+ unref$1(timer);
578
+ return () => {
579
+ cancelled = true;
580
+ clearTimeout(timer);
581
+ };
582
+ }
583
+ //#endregion
584
+ //#region src/stats.ts
585
+ /**
586
+ * # Work stats — per-type metrics over `@ayepi/core`'s {@link Metrics}
587
+ *
588
+ * A thin recorder the engine drives at each lifecycle transition. It owns a
589
+ * {@link Metrics} registry (bring your own via {@link WorkSystemOptions.metrics} to enable
590
+ * quantiles or share one across systems) and records, **labelled by work type**:
591
+ *
592
+ * - **counters** — `queued` / `started` / `succeeded` / `failed` / `retried` / `deferred` / `rescheduled`
593
+ * - **gauges** — live `active` / `pending` / `running`, the `peak_active` high-water mark, and
594
+ * `last_*_at` transition timestamps (epoch ms)
595
+ * - **summaries** (ms) — `wait_time` (poll lag = `runAt − startAt`), `total_time` (end-to-end
596
+ * `endAt − queueAt`), `success_time` / `error_time` (run duration), `delay_time` /
597
+ * `reschedule_time` (re-queue horizons), and `attempts` (tries at terminal)
598
+ *
599
+ * Read it via {@link WorkSystem.metrics} (`list()` / `get()` / `subscribe()`) or the
600
+ * {@link WorkSystem.stats} list snapshot.
601
+ *
602
+ * @module
603
+ */
604
+ /** Metric names (dotted; `formatPrometheus` sanitizes the dots to underscores). Exported so consumers can reference series by name. */
605
+ const WORK_METRICS = {
606
+ queued: "work.queued",
607
+ started: "work.started",
608
+ succeeded: "work.succeeded",
609
+ failed: "work.failed",
610
+ retried: "work.retried",
611
+ deferred: "work.deferred",
612
+ rescheduled: "work.rescheduled",
613
+ active: "work.active",
614
+ pending: "work.pending",
615
+ running: "work.running",
616
+ peak: "work.peak_active",
617
+ lastQueued: "work.last_queued_at",
618
+ lastStarted: "work.last_started_at",
619
+ lastSucceeded: "work.last_succeeded_at",
620
+ lastFailed: "work.last_failed_at",
621
+ waitTime: "work.wait_time",
622
+ totalTime: "work.total_time",
623
+ successTime: "work.success_time",
624
+ errorTime: "work.error_time",
625
+ delayTime: "work.delay_time",
626
+ rescheduleTime: "work.reschedule_time",
627
+ attempts: "work.attempts"
628
+ };
629
+ /** Create the work stats recorder over `metrics` (a fresh registry by default). */
630
+ const createWorkStats = (metrics = createMetrics$1()) => {
631
+ const cache = /* @__PURE__ */ new Map();
632
+ const ms = { unit: "ms" };
633
+ const handles = (type) => {
634
+ let h = cache.get(type);
635
+ if (h) return h;
636
+ const l = { type };
637
+ h = {
638
+ queued: metrics.counter(WORK_METRICS.queued, l, { description: "items enqueued" }),
639
+ started: metrics.counter(WORK_METRICS.started, l, { description: "runs begun" }),
640
+ succeeded: metrics.counter(WORK_METRICS.succeeded, l, { description: "runs that succeeded" }),
641
+ failed: metrics.counter(WORK_METRICS.failed, l, { description: "items dead-lettered" }),
642
+ retried: metrics.counter(WORK_METRICS.retried, l, { description: "attempt-advancing retries" }),
643
+ deferred: metrics.counter(WORK_METRICS.deferred, l, { description: "handler/classifier reschedules" }),
644
+ rescheduled: metrics.counter(WORK_METRICS.rescheduled, l, { description: "early-arrival re-pushes" }),
645
+ active: metrics.gauge(WORK_METRICS.active, l, { description: "items in flight (pending + running)" }),
646
+ pending: metrics.gauge(WORK_METRICS.pending, l, { description: "items admitted, awaiting a doer slot" }),
647
+ running: metrics.gauge(WORK_METRICS.running, l, { description: "items executing" }),
648
+ peak: metrics.gauge(WORK_METRICS.peak, l, { description: "high-water mark of in-flight items" }),
649
+ lastQueued: metrics.gauge(WORK_METRICS.lastQueued, l, {
650
+ ...ms,
651
+ description: "last enqueue time (epoch ms)"
652
+ }),
653
+ lastStarted: metrics.gauge(WORK_METRICS.lastStarted, l, {
654
+ ...ms,
655
+ description: "last start time (epoch ms)"
656
+ }),
657
+ lastSucceeded: metrics.gauge(WORK_METRICS.lastSucceeded, l, {
658
+ ...ms,
659
+ description: "last success time (epoch ms)"
660
+ }),
661
+ lastFailed: metrics.gauge(WORK_METRICS.lastFailed, l, {
662
+ ...ms,
663
+ description: "last dead-letter time (epoch ms)"
664
+ }),
665
+ waitTime: metrics.summary(WORK_METRICS.waitTime, l, {
666
+ ...ms,
667
+ description: "poll lag: runAt - startAt"
668
+ }),
669
+ totalTime: metrics.summary(WORK_METRICS.totalTime, l, {
670
+ ...ms,
671
+ description: "end-to-end: endAt - queueAt"
672
+ }),
673
+ successTime: metrics.summary(WORK_METRICS.successTime, l, {
674
+ ...ms,
675
+ description: "successful run duration"
676
+ }),
677
+ errorTime: metrics.summary(WORK_METRICS.errorTime, l, {
678
+ ...ms,
679
+ description: "failed-to-terminal run duration"
680
+ }),
681
+ delayTime: metrics.summary(WORK_METRICS.delayTime, l, {
682
+ ...ms,
683
+ description: "reschedule horizon"
684
+ }),
685
+ rescheduleTime: metrics.summary(WORK_METRICS.rescheduleTime, l, {
686
+ ...ms,
687
+ description: "early-arrival re-push horizon"
688
+ }),
689
+ attempts: metrics.summary(WORK_METRICS.attempts, l, {
690
+ unit: "count",
691
+ description: "attempts used at terminal"
692
+ })
693
+ };
694
+ cache.set(type, h);
695
+ return h;
696
+ };
697
+ return {
698
+ metrics,
699
+ queued: (type, at) => {
700
+ const h = handles(type);
701
+ h.queued.inc();
702
+ h.lastQueued.set(at);
703
+ },
704
+ claimed: (type) => {
705
+ const h = handles(type);
706
+ h.pending.add(1);
707
+ h.active.add(1);
708
+ h.peak.max(h.active.value());
709
+ },
710
+ started: (type, at, pollLagMs) => {
711
+ const h = handles(type);
712
+ h.started.inc();
713
+ h.pending.add(-1);
714
+ h.running.add(1);
715
+ h.waitTime.observe(Math.max(0, pollLagMs));
716
+ h.lastStarted.set(at);
717
+ },
718
+ released: (type, wasRunning) => {
719
+ const h = handles(type);
720
+ (wasRunning ? h.running : h.pending).add(-1);
721
+ h.active.add(-1);
722
+ },
723
+ succeeded: (type, at, runMs, totalMs, attempt) => {
724
+ const h = handles(type);
725
+ h.succeeded.inc();
726
+ h.successTime.observe(Math.max(0, runMs));
727
+ h.totalTime.observe(Math.max(0, totalMs));
728
+ h.attempts.observe(attempt);
729
+ h.lastSucceeded.set(at);
730
+ },
731
+ failed: (type, at, runMs, totalMs, attempt) => {
732
+ const h = handles(type);
733
+ h.failed.inc();
734
+ if (runMs !== void 0) h.errorTime.observe(Math.max(0, runMs));
735
+ h.totalTime.observe(Math.max(0, totalMs));
736
+ h.attempts.observe(attempt);
737
+ h.lastFailed.set(at);
738
+ },
739
+ retried: (type) => void handles(type).retried.inc(),
740
+ deferred: (type, delayMs) => {
741
+ const h = handles(type);
742
+ h.deferred.inc();
743
+ h.delayTime.observe(Math.max(0, delayMs));
744
+ },
745
+ rescheduled: (type, delayMs) => {
746
+ const h = handles(type);
747
+ h.rescheduled.inc();
748
+ h.rescheduleTime.observe(Math.max(0, delayMs));
749
+ }
750
+ };
751
+ };
752
+ //#endregion
753
+ //#region src/types.ts
754
+ /**
755
+ * # Types — the type-safe definition surface
756
+ *
757
+ * Defining a work type with {@link defineWork} yields a **callable builder**: call it
758
+ * with the work's exact input and you get a type-checked, queueable {@link Work} (with
759
+ * a build-time id) that also carries its output type. {@link createWork} takes a
760
+ * `const` tuple of builders and produces a {@link WorkSystem} whose `enqueue` is fully
761
+ * checked — by instance (`enqueue(add({ a, b }))`) or by name (`enqueue('add', { a, b })`).
762
+ *
763
+ * The **group result type** is the union of every registered work's non-void output
764
+ * ({@link GroupResult}). All durations are **milliseconds**.
765
+ *
766
+ * @module
767
+ */
768
+ const build = (name, input) => ({
769
+ id: uuid(),
770
+ type: name,
771
+ input
772
+ });
773
+ /**
774
+ * Define a work type. Returns a {@link WorkBuilder} — a function that builds queueable
775
+ * instances — typed by its input `I` and output `O`.
776
+ */
777
+ function defineWork(name, handler, opts = {}) {
778
+ return Object.assign((input) => build(name, input), {
779
+ type: name,
780
+ def: {
781
+ name,
782
+ handler,
783
+ options: opts
784
+ }
785
+ });
786
+ }
787
+ /**
788
+ * Define a **batched** work type. Items still enqueue, retry, prioritize, and join
789
+ * groups individually, but execute together via {@link BatchConfig.run} once `size`
790
+ * accumulate or `maxWait` ms elapse — so each `.result()` resolves to its aligned
791
+ * output. The per-type {@link WorkOptions.doer} governs how many *batches* run at once.
792
+ */
793
+ function defineBatchWork(name, config) {
794
+ const { size, maxWait, run, ...options } = config;
795
+ const batch = {
796
+ size,
797
+ maxWait,
798
+ run
799
+ };
800
+ const handler = async (input) => (await run([input]))[0];
801
+ return Object.assign((input) => build(name, input), {
802
+ type: name,
803
+ def: {
804
+ name,
805
+ handler,
806
+ options,
807
+ batch
808
+ }
809
+ });
810
+ }
811
+ //#endregion
812
+ //#region src/engine.ts
813
+ /**
814
+ * # Engine
815
+ *
816
+ * `createWork` ties the ports together into a running {@link WorkSystem}: a typed
817
+ * registry, a **doer**-driven worker loop (the doer decides how many items to pull and
818
+ * which to run next), per-item heartbeats, retry by **re-enqueue**, group linking +
819
+ * result resolution, batching, distributed wait handles, a built-in non-blocking
820
+ * dependency type, and scheduling. A `skipQueue` work runs in-process (doer + retries,
821
+ * no queue/store/heartbeat). Distributed coordination uses the store's
822
+ * `setIfNotExists`/`increment` atoms and pub/sub fanout, so the same logic runs on one
823
+ * process or a fleet.
824
+ *
825
+ * @module
826
+ */
827
+ /** Default idle poll interval (ms). */
828
+ const DEFAULT_POLL_INTERVAL = 1e3;
829
+ /** Default lease/visibility timeout for a pulled item (ms). */
830
+ const DEFAULT_VISIBILITY = 3e4;
831
+ /** Heartbeat interval = visibility / this, so a lease is refreshed well before it lapses. */
832
+ const HEARTBEAT_DIVISOR = 3;
833
+ /** Default key namespace. */
834
+ const DEFAULT_PREFIX = "work:";
835
+ /** Hard cap on how many items one poll fetches, regardless of doer appetite. */
836
+ const POLL_BATCH_CAP = 512;
837
+ /** TTL for results / states / group bookkeeping (ms). */
838
+ const RESULT_TTL = 864e5;
839
+ /** TTL for the "someone is waiting" registry key (ms). */
840
+ const WAIT_TTL = 36e5;
841
+ /** How often a distributed waiter re-polls the store alongside pub/sub (ms). */
842
+ const WAIT_POLL = 250;
843
+ /** Grace before the orphan check, so an in-process awaiter can register first (ms). */
844
+ const UNHANDLED_GRACE = 100;
845
+ /** Re-delivery delay for a work type this instance doesn't know (ms). */
846
+ const UNKNOWN_TYPE_DELAY = 5e3;
847
+ /** A popped item this far (ms) before its `startAt` is put back rather than run (handles backends that can't honor a long delay). */
848
+ const SCHED_TOLERANCE = 1e3;
849
+ /** Scheduler tick interval (ms). */
850
+ const SCHED_TICK = 1e3;
851
+ /** TTL for a schedule's per-fire lease (ms). */
852
+ const SCHED_LEASE_TTL = 9e4;
853
+ /** Max time `stop()` waits for in-flight work to drain (ms). */
854
+ const STOP_DRAIN = 5e3;
855
+ /** A dependency dead-letters on timeout rather than retrying. */
856
+ const DEP_RETRY_ATTEMPTS = 1;
857
+ /** Default max messages redriven from the DLQ per idle poll. */
858
+ const REDRIVE_DEFAULT = 10;
859
+ const unref = (t) => void t.unref?.();
860
+ const errString = (err) => err instanceof Error ? `${err.name}: ${err.message}` : String(err);
861
+ /**
862
+ * Create a work system. Zero-config (`createWork()`) uses the bundled in-memory backend
863
+ * and an {@link unlimitedDoer}; pass `work: [...] as const` for a typed registry, a
864
+ * `doer` to govern concurrency, and/or `queue`/`pubsub`/`store` to go distributed.
865
+ */
866
+ function createWork(opts = {}) {
867
+ const mem = opts.queue && opts.pubsub && opts.store ? null : memoryBackend({ now: opts.now });
868
+ const backend = {
869
+ queue: opts.queue ?? mem.queue,
870
+ pubsub: opts.pubsub ?? mem.pubsub,
871
+ store: opts.store ?? mem.store
872
+ };
873
+ const { pubsub, store } = backend;
874
+ const globalCodec = opts.codec ?? defaultCodec;
875
+ const globalDoer = opts.doer ?? unlimitedDoer$1();
876
+ const pollInterval = opts.pollInterval ?? DEFAULT_POLL_INTERVAL;
877
+ const visibility = opts.visibility ?? DEFAULT_VISIBILITY;
878
+ const heartbeatEvery = opts.heartbeat ?? Math.floor(visibility / HEARTBEAT_DIVISOR);
879
+ const prefix = opts.prefix ?? DEFAULT_PREFIX;
880
+ const logWith = opts.logWith ?? identityLogWith;
881
+ const now = opts.now ?? Date.now;
882
+ const random = opts.random ?? Math.random;
883
+ const attemptsOf = (r) => r.attempts ?? getDefaultRetryOptions$1().attempts ?? 1;
884
+ const stats = createWorkStats(opts.metrics);
885
+ const k = (suffix) => prefix + suffix;
886
+ /** Report a swallowed non-critical error (best-effort — a throwing `onError` is itself ignored). */
887
+ const report = (err, phase) => {
888
+ try {
889
+ opts.onError?.(err, phase);
890
+ } catch {}
891
+ };
892
+ /**
893
+ * Run a load-bearing backend call (store/queue write) with optional {@link WorkSystemOptions.portRetry}
894
+ * resilience — a transient blip is absorbed before the error reaches the engine's commit/queue handling.
895
+ * (The bundled Redis/SQS backends also retry per call; this is an engine-level safety net for any backend.)
896
+ */
897
+ const port = (fn) => opts.portRetry ? retry$1(fn, { ...opts.portRetry }) : fn();
898
+ /**
899
+ * The lifecycle combinator: run `fn`, then **always** run `release` (the teardown for state
900
+ * acquired *before* the call — an active-set entry, a heartbeat, …). The `finally` guarantees the
901
+ * release on every path (success, throw, early return), so no failure can leak that state. Pair it
902
+ * with an acquire that returns its own `release` (e.g. {@link claim}) so start↔stop can't drift apart.
903
+ */
904
+ const scoped = async (release, fn) => {
905
+ try {
906
+ return await fn();
907
+ } finally {
908
+ release();
909
+ }
910
+ };
911
+ /** Publish a best-effort notification; never throws synchronously (so callers can fire-and-forget it). */
912
+ const publish = (obj) => {
913
+ try {
914
+ return Promise.resolve(pubsub.publish(JSON.stringify(obj))).then(() => void 0);
915
+ } catch (err) {
916
+ return Promise.reject(err instanceof Error ? err : new Error(String(err)));
917
+ }
918
+ };
919
+ /** Fire a best-effort pub/sub notification detached — waiters also reach it via the store poll. */
920
+ const notify = (obj) => void publish(obj).catch((err) => report(err, "commit"));
921
+ const emit = (event) => {
922
+ try {
923
+ opts.onEvent?.(event);
924
+ } catch {}
925
+ const type = event.type;
926
+ if (type !== void 0) try {
927
+ registry.get(type)?.def.options.onEvent?.(event);
928
+ } catch {}
929
+ };
930
+ const registry = /* @__PURE__ */ new Map();
931
+ for (const builder of opts.work ?? []) registry.set(builder.type, builder);
932
+ const codecFor = (type) => registry.get(type)?.def.options.codec ?? globalCodec;
933
+ const doerFor = (type) => registry.get(type)?.def.options.doer ?? globalDoer;
934
+ /** The queue a type's items live on (its own, or the system default). New pushes for `type` go here. */
935
+ const queueFor = (type) => registry.get(type)?.def.options.queue ?? backend.queue;
936
+ /** Every distinct queue the registry uses (the default + any per-type queues) — polled fairly by the loop. */
937
+ const distinctQueues = () => {
938
+ const set = new Set([backend.queue]);
939
+ for (const b of registry.values()) if (b.def.options.queue) set.add(b.def.options.queue);
940
+ return [...set];
941
+ };
942
+ /** Resolve an item's effective options: queue-time > type `options(input)` > type constants > defaults. */
943
+ const resolveOptions = (type, input, qOpts) => {
944
+ const tOpts = registry.get(type)?.def.options;
945
+ const computed = tOpts?.options?.(input) ?? {};
946
+ return {
947
+ delay: qOpts?.delay ?? computed.delay ?? 0,
948
+ runAt: qOpts?.runAt ?? computed.runAt,
949
+ priority: qOpts?.priority ?? computed.priority ?? tOpts?.priority ?? 0,
950
+ group: qOpts?.group ?? computed.group ?? tOpts?.group,
951
+ retry: {
952
+ ...getDefaultRetryOptions$1(),
953
+ ...opts.retry,
954
+ ...tOpts?.retry,
955
+ ...computed.retry,
956
+ ...qOpts?.retry
957
+ },
958
+ skipQueue: qOpts?.skipQueue ?? computed.skipQueue ?? tOpts?.skipQueue ?? false
959
+ };
960
+ };
961
+ const allStates = /* @__PURE__ */ new Map();
962
+ const groupIndex = /* @__PURE__ */ new Map();
963
+ const recordState = (state, groupId) => {
964
+ allStates.set(state.id, state);
965
+ let g = groupIndex.get(groupId);
966
+ if (!g) {
967
+ g = /* @__PURE__ */ new Map();
968
+ groupIndex.set(groupId, g);
969
+ }
970
+ g.set(state.id, state);
971
+ };
972
+ const setState = async (state, groupId) => {
973
+ const { result: _result, ...forStore } = state;
974
+ await port(() => Promise.resolve(store.set(k(`state:${state.id}`), JSON.stringify(forStore), RESULT_TTL)));
975
+ recordState(state, groupId);
976
+ };
977
+ const readState = async (id) => {
978
+ const s = await Promise.resolve(store.get(k(`state:${id}`)));
979
+ if (s === void 0) return allStates.get(id);
980
+ try {
981
+ return JSON.parse(s);
982
+ } catch {
983
+ return;
984
+ }
985
+ };
986
+ /** Atomic add via the store `increment` atom, falling back to a (single-process-safe) get+set. */
987
+ const storeIncrement = (key, by, ttl) => port(async () => {
988
+ if (store.increment) return await Promise.resolve(store.increment(key, by, ttl));
989
+ const cur = Number(await Promise.resolve(store.get(key)) ?? "0") + by;
990
+ await Promise.resolve(store.set(key, String(cur), ttl));
991
+ return cur;
992
+ });
993
+ const groupIncr = (groupId, by) => storeIncrement(k(`group:${groupId}:open`), by, RESULT_TTL);
994
+ /** Roll back a group hold (`groupIncr(+1)`) whose item never reached the queue / a terminal path — best-effort, so the group can still settle. */
995
+ const undoHold = (groupId) => groupIncr(groupId, -1).then(() => void 0).catch((err) => report(err, "commit"));
996
+ const settleGroup = async (groupId) => {
997
+ if (await groupIncr(groupId, -1) > 0) return;
998
+ await port(() => Promise.resolve(store.set(k(`group:${groupId}:done`), "1", RESULT_TTL)));
999
+ notify({
1000
+ kind: "group-done",
1001
+ groupId
1002
+ });
1003
+ const r = await Promise.resolve(store.get(k(`group:${groupId}:result`)));
1004
+ emit({
1005
+ kind: "group-done",
1006
+ groupId,
1007
+ result: r === void 0 ? void 0 : globalCodec.parse(r),
1008
+ at: now()
1009
+ });
1010
+ maybeUnhandled(groupId);
1011
+ };
1012
+ const makeContext = (impl) => {
1013
+ const queue = (works, options) => Array.isArray(works) ? works.map((w) => impl.enqueueOne(w, options)) : impl.enqueueOne(works, options);
1014
+ return {
1015
+ id: impl.id,
1016
+ groupId: impl.groupId,
1017
+ attempt: impl.attempt,
1018
+ queue,
1019
+ setResult: impl.setResult,
1020
+ states: impl.states,
1021
+ claim: impl.claim
1022
+ };
1023
+ };
1024
+ const storeStates = (ids) => Promise.all(ids.map(readState));
1025
+ const storeClaim = (key) => Promise.resolve(store.setIfNotExists(k(key), "1", RESULT_TTL));
1026
+ /** Push an envelope onto **its type's** queue (default or per-type), invisible for `delay` ms. */
1027
+ const pushEnvelope = (env, delay) => port(() => Promise.resolve(queueFor(env.type).push(JSON.stringify(env), { delay })).then(() => void 0));
1028
+ /** Resolve when a deferred/scheduled item should next run (absolute epoch ms). */
1029
+ const resolveRunAt = (when) => when.runAt ?? now() + (when.delay ?? 0);
1030
+ const submitQueued = (id, type, input, groupId, ro) => {
1031
+ const queueAt = now();
1032
+ const startAt = ro.runAt ?? queueAt + ro.delay;
1033
+ return (async () => {
1034
+ await groupIncr(groupId, 1);
1035
+ try {
1036
+ await setState({
1037
+ id,
1038
+ type,
1039
+ status: "pending",
1040
+ attempt: 1,
1041
+ queueAt,
1042
+ startAt,
1043
+ priority: ro.priority,
1044
+ group: ro.group
1045
+ }, groupId);
1046
+ await pushEnvelope({
1047
+ id,
1048
+ type,
1049
+ groupId,
1050
+ input: codecFor(type).stringify(input),
1051
+ queueAt,
1052
+ startAt,
1053
+ attempt: 1,
1054
+ priority: ro.priority,
1055
+ group: ro.group,
1056
+ retry: ro.retry
1057
+ }, Math.max(0, startAt - queueAt));
1058
+ } catch (err) {
1059
+ await undoHold(groupId);
1060
+ throw err;
1061
+ }
1062
+ stats.queued(type, queueAt);
1063
+ emit({
1064
+ kind: "queued",
1065
+ id,
1066
+ type,
1067
+ groupId,
1068
+ at: queueAt
1069
+ });
1070
+ })();
1071
+ };
1072
+ /** Re-enter the queue for a retry: a fresh delivery with `attempt + 1` and a recomputed `startAt`. */
1073
+ const rePush = async (env, delay) => {
1074
+ const startAt = now() + delay;
1075
+ const next = {
1076
+ ...env,
1077
+ attempt: env.attempt + 1,
1078
+ startAt
1079
+ };
1080
+ await setState({
1081
+ id: env.id,
1082
+ type: env.type,
1083
+ status: "pending",
1084
+ attempt: next.attempt,
1085
+ queueAt: env.queueAt,
1086
+ startAt,
1087
+ priority: env.priority,
1088
+ group: env.group
1089
+ }, env.groupId);
1090
+ await pushEnvelope(next, delay);
1091
+ };
1092
+ const enqueueImpl = (a, b, c) => {
1093
+ const fromName = typeof a === "string";
1094
+ const id = fromName ? uuid() : a.id;
1095
+ const type = fromName ? a : a.type;
1096
+ const input = fromName ? b : a.input;
1097
+ const qOpts = fromName ? c : b;
1098
+ const groupId = uuid();
1099
+ const ro = resolveOptions(type, input, qOpts);
1100
+ return makeHandle(id, type, groupId, ro.skipQueue ? runImmediate(id, type, input, groupId, ro) : submitQueued(id, type, input, groupId, ro));
1101
+ };
1102
+ const enqueueRaw = (type, input) => void enqueueImpl(type, input);
1103
+ const makeHandle = (id, type, groupId, ready) => {
1104
+ const markWaiting = () => Promise.resolve(store.setIfNotExists(k(`wait:${groupId}`), "1", WAIT_TTL));
1105
+ const result = async () => {
1106
+ await ready;
1107
+ await markWaiting();
1108
+ return waitForItem(id, type);
1109
+ };
1110
+ const group = async () => {
1111
+ await ready;
1112
+ await markWaiting();
1113
+ return waitForGroup(groupId);
1114
+ };
1115
+ return {
1116
+ id,
1117
+ groupId,
1118
+ result,
1119
+ group,
1120
+ then: (onF, onR) => group().then(onF, onR)
1121
+ };
1122
+ };
1123
+ const waitForItem = (id, type) => new Promise((resolve, reject) => {
1124
+ let settled = false;
1125
+ const teardown = [];
1126
+ const cleanup = () => {
1127
+ settled = true;
1128
+ for (const t of teardown) t();
1129
+ };
1130
+ const check = async () => {
1131
+ if (settled) return;
1132
+ const res = await Promise.resolve(store.get(k(`result:${id}`)));
1133
+ if (res !== void 0) {
1134
+ cleanup();
1135
+ resolve(codecFor(type).parse(res));
1136
+ return;
1137
+ }
1138
+ const err = await Promise.resolve(store.get(k(`item:${id}:error`)));
1139
+ if (err !== void 0) {
1140
+ cleanup();
1141
+ reject(new Error(err));
1142
+ }
1143
+ };
1144
+ teardown.push(pubsub.subscribe((m) => {
1145
+ try {
1146
+ const e = JSON.parse(m);
1147
+ if (e.kind === "done" && e.id === id) check();
1148
+ } catch {}
1149
+ }));
1150
+ const poll = setInterval(() => void check(), WAIT_POLL);
1151
+ unref(poll);
1152
+ teardown.push(() => clearInterval(poll));
1153
+ check();
1154
+ });
1155
+ const waitForGroup = (groupId) => new Promise((resolve) => {
1156
+ let settled = false;
1157
+ const teardown = [];
1158
+ const cleanup = () => {
1159
+ settled = true;
1160
+ for (const t of teardown) t();
1161
+ };
1162
+ const check = async () => {
1163
+ if (settled) return;
1164
+ if (await Promise.resolve(store.get(k(`group:${groupId}:done`))) === void 0) return;
1165
+ cleanup();
1166
+ const r = await Promise.resolve(store.get(k(`group:${groupId}:result`)));
1167
+ resolve(r === void 0 ? void 0 : globalCodec.parse(r));
1168
+ };
1169
+ teardown.push(pubsub.subscribe((m) => {
1170
+ try {
1171
+ const e = JSON.parse(m);
1172
+ if (e.kind === "group-done" && e.groupId === groupId) check();
1173
+ } catch {}
1174
+ }));
1175
+ const poll = setInterval(() => void check(), WAIT_POLL);
1176
+ unref(poll);
1177
+ teardown.push(() => clearInterval(poll));
1178
+ check();
1179
+ });
1180
+ const maybeUnhandled = (groupId) => {
1181
+ if (!opts.unhandledWorkGroup) return;
1182
+ unref(setTimeout(async () => {
1183
+ if (!await Promise.resolve(store.setIfNotExists(k(`group-handled:${groupId}`), "1", RESULT_TTL))) return;
1184
+ if (await Promise.resolve(store.get(k(`wait:${groupId}`))) !== void 0) return;
1185
+ const r = await Promise.resolve(store.get(k(`group:${groupId}:result`)));
1186
+ const states = [...groupIndex.get(groupId)?.values() ?? []];
1187
+ opts.unhandledWorkGroup?.({
1188
+ groupId,
1189
+ lastResult: r === void 0 ? void 0 : globalCodec.parse(r),
1190
+ states
1191
+ });
1192
+ }, UNHANDLED_GRACE));
1193
+ };
1194
+ const activeMap = /* @__PURE__ */ new Map();
1195
+ /**
1196
+ * `skipQueue`: hand the first attempt straight to the doer (no queue hop, no lease,
1197
+ * no heartbeat) for low latency. State/results/group still go through the store, and
1198
+ * a **failure re-enqueues** the item (attempt + 1) onto the durable queue — so the
1199
+ * retry survives a crash and any instance in the fleet can pick it up. (The first run
1200
+ * itself is best-effort: that's the latency-for-durability trade `skipQueue` makes.)
1201
+ */
1202
+ const runImmediate = (id, type, input, groupId, ro) => {
1203
+ const queueAt = now();
1204
+ const startAt = ro.runAt ?? queueAt + ro.delay;
1205
+ const builder = registry.get(type);
1206
+ const env = {
1207
+ id,
1208
+ type,
1209
+ groupId,
1210
+ input: codecFor(type).stringify(input),
1211
+ queueAt,
1212
+ startAt,
1213
+ attempt: 1,
1214
+ priority: ro.priority,
1215
+ group: ro.group,
1216
+ retry: ro.retry
1217
+ };
1218
+ return (async () => {
1219
+ await groupIncr(groupId, 1);
1220
+ try {
1221
+ await setState({
1222
+ id,
1223
+ type,
1224
+ status: "pending",
1225
+ attempt: 1,
1226
+ queueAt,
1227
+ startAt,
1228
+ priority: ro.priority,
1229
+ group: ro.group
1230
+ }, groupId);
1231
+ stats.queued(type, queueAt);
1232
+ emit({
1233
+ kind: "queued",
1234
+ id,
1235
+ type,
1236
+ groupId,
1237
+ at: queueAt
1238
+ });
1239
+ } catch (err) {
1240
+ await undoHold(groupId);
1241
+ throw err;
1242
+ }
1243
+ if (!builder) {
1244
+ await deadLetterItem(null, env, `unknown work type: ${type}`, null);
1245
+ return;
1246
+ }
1247
+ const release = claim(env, null, null);
1248
+ try {
1249
+ doerFor(type).do(() => execute(null, env, input, builder.def, null, release));
1250
+ } catch (err) {
1251
+ release();
1252
+ await undoHold(groupId);
1253
+ throw err;
1254
+ }
1255
+ })();
1256
+ };
1257
+ /** Keep a leased item's lease (and its heartbeat key) alive on the **queue it came from**. */
1258
+ const startHeartbeat = (p, id, q) => {
1259
+ const hb = setInterval(() => {
1260
+ Promise.resolve(q.heartbeat(p, visibility));
1261
+ Promise.resolve(store.set(k(`hb:${id}`), String(now()), visibility));
1262
+ }, heartbeatEvery);
1263
+ unref(hb);
1264
+ return hb;
1265
+ };
1266
+ /**
1267
+ * Acquire the per-item processing state — register it in the active set and (if leased) start its
1268
+ * heartbeat — and return the **paired teardown** that clears the heartbeat and drops it from the
1269
+ * active set. This closure is the *sole owner* of that state: whoever runs the item passes the
1270
+ * returned `release` to {@link scoped}, so it is torn down on every path and can never drift/leak.
1271
+ * (A `skipQueue` item has no lease `p`/`q`, so no heartbeat.)
1272
+ */
1273
+ const claim = (env, p, q) => {
1274
+ activeMap.set(env.id, {
1275
+ id: env.id,
1276
+ type: env.type,
1277
+ groupId: env.groupId,
1278
+ status: "pending",
1279
+ attempt: env.attempt,
1280
+ priority: env.priority,
1281
+ group: env.group,
1282
+ queueAt: env.queueAt,
1283
+ startAt: env.startAt
1284
+ });
1285
+ stats.claimed(env.type);
1286
+ const hb = p && q ? startHeartbeat(p, env.id, q) : null;
1287
+ return () => {
1288
+ if (hb) clearInterval(hb);
1289
+ const wasRunning = activeMap.get(env.id)?.status === "running";
1290
+ activeMap.delete(env.id);
1291
+ stats.released(env.type, wasRunning);
1292
+ };
1293
+ };
1294
+ /**
1295
+ * Re-enqueue an item to become runnable at `startAt` **without advancing its attempt** — a
1296
+ * reschedule, not a retry. It **acks the current delivery and pushes a fresh message**, so a
1297
+ * backend's per-message delivery count (e.g. SQS `ApproximateReceiveCount`) **resets** — a
1298
+ * scheduled item that bounces (re-deferred each poll until due) can never trip the backend's
1299
+ * native redrive/DLQ. Returns the resolved `startAt`.
1300
+ */
1301
+ const reschedule = async (p, env, q, startAt) => {
1302
+ const next = {
1303
+ ...env,
1304
+ startAt
1305
+ };
1306
+ await setState({
1307
+ id: env.id,
1308
+ type: env.type,
1309
+ status: "pending",
1310
+ attempt: env.attempt,
1311
+ queueAt: env.queueAt,
1312
+ startAt,
1313
+ priority: env.priority,
1314
+ group: env.group
1315
+ }, env.groupId);
1316
+ if (p && q) await port(() => Promise.resolve(q.ack(p)));
1317
+ await pushEnvelope(next, Math.max(0, startAt - now()));
1318
+ return startAt;
1319
+ };
1320
+ /** A handler/classifier-initiated deferral: {@link reschedule} the item to `runAt` and emit a `deferred` event. */
1321
+ const deferItem = async (p, env, q, runAt) => {
1322
+ const startAt = await reschedule(p, env, q, Math.max(runAt, now()));
1323
+ const at = now();
1324
+ stats.deferred(env.type, startAt - at);
1325
+ emit({
1326
+ kind: "deferred",
1327
+ id: env.id,
1328
+ type: env.type,
1329
+ groupId: env.groupId,
1330
+ runAt: startAt,
1331
+ at
1332
+ });
1333
+ };
1334
+ const deadLetterItem = async (p, env, error, q, runAt) => {
1335
+ const at = now();
1336
+ stats.failed(env.type, at, runAt === void 0 ? void 0 : at - runAt, at - env.queueAt, env.attempt);
1337
+ await port(() => Promise.resolve(store.set(k(`item:${env.id}:error`), error, RESULT_TTL)));
1338
+ await setState({
1339
+ id: env.id,
1340
+ type: env.type,
1341
+ status: "dead",
1342
+ attempt: env.attempt,
1343
+ error,
1344
+ queueAt: env.queueAt,
1345
+ startAt: env.startAt,
1346
+ runAt,
1347
+ endAt: at,
1348
+ priority: env.priority,
1349
+ group: env.group
1350
+ }, env.groupId);
1351
+ if (p && q) {
1352
+ await Promise.resolve(q.deadLetter?.(p.body, error));
1353
+ await port(() => Promise.resolve(q.ack(p)));
1354
+ }
1355
+ notify({
1356
+ kind: "done",
1357
+ id: env.id,
1358
+ groupId: env.groupId,
1359
+ error
1360
+ });
1361
+ emit({
1362
+ kind: "failed",
1363
+ id: env.id,
1364
+ type: env.type,
1365
+ groupId: env.groupId,
1366
+ attempt: env.attempt,
1367
+ error,
1368
+ willRetry: false,
1369
+ at
1370
+ });
1371
+ await settleGroup(env.groupId);
1372
+ };
1373
+ /** The synchronous part of marking running (the active-map flip) — kept on the critical path. */
1374
+ const markRunningSync = (env, runAt) => {
1375
+ stats.started(env.type, runAt, runAt - env.startAt);
1376
+ const a = activeMap.get(env.id);
1377
+ if (a) {
1378
+ a.status = "running";
1379
+ a.runAt = runAt;
1380
+ }
1381
+ };
1382
+ /** The async part (persist the `running` state + emit `started`) — can run **alongside** the handler. */
1383
+ const persistRunning = async (env, runAt) => {
1384
+ await setState({
1385
+ id: env.id,
1386
+ type: env.type,
1387
+ status: "running",
1388
+ attempt: env.attempt,
1389
+ queueAt: env.queueAt,
1390
+ startAt: env.startAt,
1391
+ runAt,
1392
+ priority: env.priority,
1393
+ group: env.group
1394
+ }, env.groupId);
1395
+ emit({
1396
+ kind: "started",
1397
+ id: env.id,
1398
+ type: env.type,
1399
+ groupId: env.groupId,
1400
+ attempt: env.attempt,
1401
+ at: runAt
1402
+ });
1403
+ };
1404
+ const markRunning = async (env, runAt) => {
1405
+ markRunningSync(env, runAt);
1406
+ await persistRunning(env, runAt);
1407
+ };
1408
+ const finishSuccess = async (p, env, runAt, output, q) => {
1409
+ const at = now();
1410
+ stats.succeeded(env.type, at, at - runAt, at - env.queueAt, env.attempt);
1411
+ await port(() => Promise.resolve(store.set(k(`result:${env.id}`), codecFor(env.type).stringify(output), RESULT_TTL)));
1412
+ await setState({
1413
+ id: env.id,
1414
+ type: env.type,
1415
+ status: "success",
1416
+ attempt: env.attempt,
1417
+ result: output,
1418
+ queueAt: env.queueAt,
1419
+ startAt: env.startAt,
1420
+ runAt,
1421
+ endAt: at,
1422
+ priority: env.priority,
1423
+ group: env.group
1424
+ }, env.groupId);
1425
+ if (p && q) await port(() => Promise.resolve(q.ack(p)));
1426
+ notify({
1427
+ kind: "done",
1428
+ id: env.id,
1429
+ groupId: env.groupId
1430
+ });
1431
+ emit({
1432
+ kind: "succeeded",
1433
+ id: env.id,
1434
+ type: env.type,
1435
+ groupId: env.groupId,
1436
+ attempt: env.attempt,
1437
+ result: output,
1438
+ at
1439
+ });
1440
+ await settleGroup(env.groupId);
1441
+ };
1442
+ const finishFailure = async (p, env, runAt, err, q) => {
1443
+ if (env.attempt < attemptsOf(env.retry)) {
1444
+ const delay = backoff$1(env.attempt, env.retry, random);
1445
+ stats.retried(env.type);
1446
+ emit({
1447
+ kind: "failed",
1448
+ id: env.id,
1449
+ type: env.type,
1450
+ groupId: env.groupId,
1451
+ attempt: env.attempt,
1452
+ error: errString(err),
1453
+ willRetry: true,
1454
+ at: now()
1455
+ });
1456
+ if (p && q) await port(() => Promise.resolve(q.ack(p)));
1457
+ await rePush(env, delay);
1458
+ } else await deadLetterItem(p, env, errString(err), q, runAt);
1459
+ };
1460
+ /** Ask the per-type (else system) {@link FailureClassifier} what a handler error means; a throwing classifier falls back to the default. */
1461
+ const classifyFailure = async (err, env) => {
1462
+ const decide = registry.get(env.type)?.def.options.onFailure ?? opts.onFailure;
1463
+ if (!decide) return;
1464
+ try {
1465
+ return await decide(err, {
1466
+ id: env.id,
1467
+ type: env.type,
1468
+ attempt: env.attempt,
1469
+ attempts: attemptsOf(env.retry)
1470
+ }) ?? void 0;
1471
+ } catch (e) {
1472
+ report(e, "commit");
1473
+ return;
1474
+ }
1475
+ };
1476
+ /**
1477
+ * Decide what a handler/batch failure does — the single owner of failure routing for both single and
1478
+ * batched items. Explicit throws win (`WorkDelayError` → reschedule, `RetryAbort` → dead-letter now);
1479
+ * otherwise the {@link classifyFailure} hook may `'abort'` or reschedule a `{ delay }`/`{ runAt }`
1480
+ * (e.g. a rate limit — re-queued **without** advancing `attempt`); the default is retry/dead-letter.
1481
+ */
1482
+ const handleFailure = async (p, env, runAt, err, q) => {
1483
+ if (err instanceof WorkDelayError) {
1484
+ await deferItem(p, env, q, resolveRunAt(err.when));
1485
+ return;
1486
+ }
1487
+ if (err instanceof RetryAbort$1) {
1488
+ await deadLetterItem(p, env, errString(err.cause ?? err), q, runAt);
1489
+ return;
1490
+ }
1491
+ const decision = await classifyFailure(err, env);
1492
+ if (decision === "abort") {
1493
+ await deadLetterItem(p, env, errString(err), q, runAt);
1494
+ return;
1495
+ }
1496
+ if (decision !== void 0 && decision !== "retry") {
1497
+ await deferItem(p, env, q, "runAt" in decision ? decision.runAt : now() + decision.delay);
1498
+ return;
1499
+ }
1500
+ await finishFailure(p, env, runAt, err, q);
1501
+ };
1502
+ const queuedContext = (env, pending) => makeContext({
1503
+ id: env.id,
1504
+ groupId: env.groupId,
1505
+ attempt: env.attempt,
1506
+ enqueueOne: (work, options) => {
1507
+ pending.push(submitQueued(work.id, work.type, work.input, env.groupId, resolveOptions(work.type, work.input, options)));
1508
+ return work.id;
1509
+ },
1510
+ setResult: (r) => {
1511
+ pending.push(port(() => Promise.resolve(store.set(k(`group:${env.groupId}:result`), globalCodec.stringify(r), RESULT_TTL)).then(() => void 0)));
1512
+ },
1513
+ states: storeStates,
1514
+ claim: storeClaim
1515
+ });
1516
+ /**
1517
+ * The task handed to a doer: run a single item (leased from queue `q`, or `skipQueue` with `p`/`q`
1518
+ * null). `release` is the {@link claim} teardown — {@link scoped} runs it in `finally`, so the
1519
+ * active-set entry + heartbeat are always cleaned up regardless of how the run ends.
1520
+ */
1521
+ const execute = (p, env, input, def, q, release) => scoped(release, async () => {
1522
+ const runAt = now();
1523
+ markRunningSync(env, runAt);
1524
+ const running = persistRunning(env, runAt).catch((err) => report(err, "commit"));
1525
+ const pending = [];
1526
+ const ctx = queuedContext(env, pending);
1527
+ let output;
1528
+ try {
1529
+ output = await logWith(merge(opts.logContext?.(input, env.type), def.options.logContext?.(input)), () => Promise.resolve(def.handler(input, ctx)));
1530
+ await Promise.all(pending);
1531
+ } catch (err) {
1532
+ await running;
1533
+ await handleFailure(p, env, runAt, err, q);
1534
+ return;
1535
+ }
1536
+ try {
1537
+ await running;
1538
+ await finishSuccess(p, env, runAt, output, q);
1539
+ } catch (err) {
1540
+ report(err, "commit");
1541
+ }
1542
+ });
1543
+ const executeBatch = (items, batch) => scoped(() => {
1544
+ for (const it of items) it.release();
1545
+ }, async () => {
1546
+ const runAt = now();
1547
+ await Promise.all(items.map((it) => markRunning(it.env, runAt)));
1548
+ let outputs;
1549
+ try {
1550
+ const ran = await Promise.resolve(batch.run(items.map((it) => it.input)));
1551
+ if (!Array.isArray(ran) || ran.length !== items.length) throw new Error(`batch "${items[0]?.env.type}" returned ${Array.isArray(ran) ? ran.length : "a non-array"} outputs for ${items.length} inputs`);
1552
+ outputs = ran;
1553
+ } catch (err) {
1554
+ await Promise.all(items.map((it) => handleFailure(it.p, it.env, runAt, err, it.q)));
1555
+ return;
1556
+ }
1557
+ await Promise.all(items.map((it, i) => Promise.resolve(finishSuccess(it.p, it.env, runAt, outputs[i], it.q)).catch((err) => report(err, "commit"))));
1558
+ });
1559
+ const batchers = /* @__PURE__ */ new Map();
1560
+ const batcherFor = (type) => {
1561
+ let b = batchers.get(type);
1562
+ if (!b) {
1563
+ const config = registry.get(type).def.batch;
1564
+ const doer = doerFor(type);
1565
+ const buffer = [];
1566
+ let timer = null;
1567
+ const flush = () => {
1568
+ if (timer) {
1569
+ clearTimeout(timer);
1570
+ timer = null;
1571
+ }
1572
+ if (buffer.length === 0) return;
1573
+ const items = buffer.splice(0);
1574
+ try {
1575
+ doer.do(() => executeBatch(items, config), { createdAt: items[0].env.queueAt });
1576
+ } catch (err) {
1577
+ for (const it of items) it.release();
1578
+ report(err, "queue");
1579
+ }
1580
+ };
1581
+ b = {
1582
+ available: () => doer.available() <= 0 ? 0 : Math.max(0, config.size - buffer.length),
1583
+ add: (item) => {
1584
+ buffer.push(item);
1585
+ if (buffer.length >= config.size) flush();
1586
+ else if (!timer) {
1587
+ timer = setTimeout(flush, config.maxWait);
1588
+ unref(timer);
1589
+ }
1590
+ },
1591
+ flushAll: flush
1592
+ };
1593
+ batchers.set(type, b);
1594
+ }
1595
+ return b;
1596
+ };
1597
+ /**
1598
+ * Parse + validate an item leased from queue `q` and route it to a batcher or doer. Returns `true`
1599
+ * if it **started** the item, `false` if it put it back (early/not-due, affinity decline, saturated
1600
+ * doer/batcher, unknown type) or dropped it — the loop uses this to keep pulling vs back off.
1601
+ */
1602
+ const accept = async (p, q) => {
1603
+ let env;
1604
+ try {
1605
+ env = JSON.parse(p.body);
1606
+ } catch {
1607
+ await Promise.resolve(q.ack(p));
1608
+ return false;
1609
+ }
1610
+ const due = now();
1611
+ if (env.startAt - due > SCHED_TOLERANCE) {
1612
+ await reschedule(p, env, q, env.startAt);
1613
+ stats.rescheduled(env.type, env.startAt - due);
1614
+ return false;
1615
+ }
1616
+ const { id, type, groupId } = env;
1617
+ const builder = registry.get(type);
1618
+ if (!builder) {
1619
+ if (env.attempt >= attemptsOf(env.retry)) await deadLetterItem(p, env, `unknown work type: ${type}`, q);
1620
+ else await Promise.resolve(q.fail(p, UNKNOWN_TYPE_DELAY));
1621
+ return false;
1622
+ }
1623
+ let input;
1624
+ try {
1625
+ input = codecFor(type).parse(env.input);
1626
+ } catch (e) {
1627
+ await deadLetterItem(p, env, `bad input: ${errString(e)}`, q);
1628
+ return false;
1629
+ }
1630
+ if (opts.accept && !opts.accept({
1631
+ id,
1632
+ type,
1633
+ groupId,
1634
+ attempt: env.attempt,
1635
+ input
1636
+ })) {
1637
+ await Promise.resolve(q.fail(p, pollInterval));
1638
+ return false;
1639
+ }
1640
+ if (builder.def.batch) {
1641
+ const batcher = batcherFor(type);
1642
+ if (batcher.available() <= 0) {
1643
+ await Promise.resolve(q.fail(p, pollInterval));
1644
+ return false;
1645
+ }
1646
+ batcher.add({
1647
+ p,
1648
+ env,
1649
+ input,
1650
+ release: claim(env, p, q),
1651
+ q
1652
+ });
1653
+ return true;
1654
+ }
1655
+ const doer = doerFor(type);
1656
+ if (doer.available() <= 0) {
1657
+ await Promise.resolve(q.fail(p, pollInterval));
1658
+ return false;
1659
+ }
1660
+ const release = claim(env, p, q);
1661
+ try {
1662
+ doer.do(() => execute(p, env, input, builder.def, q, release), {
1663
+ group: env.group,
1664
+ priority: env.priority,
1665
+ createdAt: env.queueAt
1666
+ });
1667
+ } catch (err) {
1668
+ release();
1669
+ throw err;
1670
+ }
1671
+ return true;
1672
+ };
1673
+ /**
1674
+ * Redrive up to `redriveCount` bodies from the configured {@link WorkSystemOptions.dlq} back onto
1675
+ * their type's queue as **fresh** work — `attempt` reset to 1 (full retry budget), `queueAt`/`startAt`
1676
+ * = now, a fresh group hold re-opened — then ack each off the DLQ. Called only when the normal queues
1677
+ * are idle and capacity is free. An unparseable body is dropped (acked); a re-queue failure leaves the
1678
+ * body leased on the DLQ to retry on a later idle tick. Returns how many were moved.
1679
+ */
1680
+ const redriveCount = opts.redriveCount ?? REDRIVE_DEFAULT;
1681
+ const redriveFromDLQ = async () => {
1682
+ const dlq = opts.dlq;
1683
+ if (!dlq || redriveCount <= 0) return 0;
1684
+ const pulled = await Promise.resolve(dlq.pop(redriveCount, visibility));
1685
+ let moved = 0;
1686
+ for (const p of pulled) {
1687
+ let env;
1688
+ try {
1689
+ env = JSON.parse(p.body);
1690
+ } catch {
1691
+ await Promise.resolve(dlq.ack(p)).catch((err) => report(err, "queue"));
1692
+ continue;
1693
+ }
1694
+ const at = now();
1695
+ const fresh = {
1696
+ ...env,
1697
+ attempt: 1,
1698
+ queueAt: at,
1699
+ startAt: at
1700
+ };
1701
+ try {
1702
+ await groupIncr(env.groupId, 1);
1703
+ try {
1704
+ await setState({
1705
+ id: env.id,
1706
+ type: env.type,
1707
+ status: "pending",
1708
+ attempt: 1,
1709
+ queueAt: at,
1710
+ startAt: at,
1711
+ priority: env.priority,
1712
+ group: env.group
1713
+ }, env.groupId);
1714
+ await pushEnvelope(fresh, 0);
1715
+ } catch (err) {
1716
+ await undoHold(env.groupId);
1717
+ throw err;
1718
+ }
1719
+ await port(() => Promise.resolve(dlq.ack(p)));
1720
+ stats.queued(env.type, at);
1721
+ emit({
1722
+ kind: "queued",
1723
+ id: env.id,
1724
+ type: env.type,
1725
+ groupId: env.groupId,
1726
+ at
1727
+ });
1728
+ moved += 1;
1729
+ } catch (err) {
1730
+ report(err, "queue");
1731
+ }
1732
+ }
1733
+ return moved;
1734
+ };
1735
+ let running = false;
1736
+ let loopPromise = null;
1737
+ const scheduleCancels = /* @__PURE__ */ new Set();
1738
+ const pollCount = () => {
1739
+ const sources = new Set([globalDoer]);
1740
+ for (const b of registry.values()) if (b.def.batch) sources.add(batcherFor(b.type));
1741
+ else if (b.def.options.doer) sources.add(b.def.options.doer);
1742
+ let n = 0;
1743
+ for (const s of sources) n += s.available();
1744
+ return Math.min(POLL_BATCH_CAP, n);
1745
+ };
1746
+ let pollCursor = 0;
1747
+ const loop = async () => {
1748
+ while (running) try {
1749
+ const pause = (opts.backpressure ? await opts.backpressure({
1750
+ metrics: stats.metrics,
1751
+ active: activeMap.size
1752
+ }) : 0) ?? 0;
1753
+ if (pause > 0) {
1754
+ await sleep(pause);
1755
+ continue;
1756
+ }
1757
+ const n = pollCount();
1758
+ if (n <= 0) {
1759
+ await sleep(pollInterval);
1760
+ continue;
1761
+ }
1762
+ const qs = distinctQueues();
1763
+ const share = Math.max(1, Math.ceil(n / qs.length));
1764
+ let accepted = 0;
1765
+ let anyFull = false;
1766
+ let pulledTotal = 0;
1767
+ for (let i = 0; i < qs.length; i++) {
1768
+ const q = qs[(pollCursor + i) % qs.length];
1769
+ const pulled = await Promise.resolve(q.pop(share, visibility));
1770
+ pulledTotal += pulled.length;
1771
+ if (pulled.length >= share) anyFull = true;
1772
+ const started = await Promise.all(pulled.map((p) => accept(p, q).catch((err) => (report(err, "queue"), false))));
1773
+ accepted += started.filter(Boolean).length;
1774
+ }
1775
+ pollCursor = (pollCursor + 1) % qs.length;
1776
+ if (pulledTotal === 0 && opts.dlq && await redriveFromDLQ() > 0) continue;
1777
+ if (accepted === 0 || !anyFull) await sleep(pollInterval);
1778
+ } catch (err) {
1779
+ report(err, "queue");
1780
+ await sleep(pollInterval);
1781
+ }
1782
+ };
1783
+ registry.set(DEPENDENCY_TYPE, defineWork(DEPENDENCY_TYPE, dependencyHandler, { retry: { attempts: DEP_RETRY_ATTEMPTS } }));
1784
+ const start = () => {
1785
+ if (running) return;
1786
+ running = true;
1787
+ loopPromise = loop();
1788
+ };
1789
+ const stop = async () => {
1790
+ running = false;
1791
+ for (const cancel of scheduleCancels) cancel();
1792
+ scheduleCancels.clear();
1793
+ await loopPromise?.catch(() => {});
1794
+ loopPromise = null;
1795
+ for (const b of batchers.values()) b.flushAll();
1796
+ const drained = Promise.all([...allDoers()].map((d) => d.done())).then(() => void 0);
1797
+ await Promise.race([drained, sleep(STOP_DRAIN)]);
1798
+ };
1799
+ const allDoers = () => {
1800
+ const set = new Set([globalDoer]);
1801
+ for (const b of registry.values()) {
1802
+ const d = b.def.options.doer;
1803
+ if (d) set.add(d);
1804
+ }
1805
+ return set;
1806
+ };
1807
+ const schedule = (config) => {
1808
+ const cancel = startSchedule(config, {
1809
+ store,
1810
+ now,
1811
+ enqueueRaw,
1812
+ prefix,
1813
+ tick: SCHED_TICK,
1814
+ leaseTtl: SCHED_LEASE_TTL
1815
+ });
1816
+ scheduleCancels.add(cancel);
1817
+ return () => {
1818
+ cancel();
1819
+ scheduleCancels.delete(cancel);
1820
+ };
1821
+ };
1822
+ const active = () => [...activeMap.values()].map((a) => ({ ...a }));
1823
+ const api = {
1824
+ enqueue: enqueueImpl,
1825
+ schedule,
1826
+ start,
1827
+ stop,
1828
+ list: () => Promise.resolve([...allStates.values()]),
1829
+ active,
1830
+ stats: () => stats.metrics.list(),
1831
+ metrics: stats.metrics,
1832
+ backend
1833
+ };
1834
+ if (opts.autoStart !== false) start();
1835
+ return api;
1836
+ }
1837
+ //#endregion
1838
+ //#region src/adaptive.ts
1839
+ /**
1840
+ * Build an {@link AdaptiveDelay} controller for {@link WorkSystemOptions.backpressure}. It holds a
1841
+ * little state (the current pause + the last cumulative counts) across calls, so create **one** per
1842
+ * work system. Watching a subset of `types` lets one system protect a specific downstream.
1843
+ */
1844
+ function adaptiveDelay(opts = {}) {
1845
+ const maxFailRate = opts.maxFailRate ?? 0;
1846
+ const min = opts.min ?? 0;
1847
+ const max = opts.max ?? 3e4;
1848
+ const base = opts.base ?? 100;
1849
+ const factor = opts.factor ?? 2;
1850
+ const step = opts.step ?? base;
1851
+ let delay = min;
1852
+ let prevCompleted = 0;
1853
+ let prevFailed = 0;
1854
+ const watch = opts.types ? new Set(opts.types) : null;
1855
+ return (ctx) => {
1856
+ let succeeded = 0;
1857
+ let failed = 0;
1858
+ for (const s of ctx.metrics.list()) {
1859
+ const type = s.labels.type;
1860
+ if (watch && (type === void 0 || !watch.has(type))) continue;
1861
+ if (s.meta.name === WORK_METRICS.succeeded) succeeded += s.value;
1862
+ else if (s.meta.name === WORK_METRICS.failed) failed += s.value;
1863
+ }
1864
+ const completed = succeeded + failed;
1865
+ const dCompleted = completed - prevCompleted;
1866
+ const dFailed = failed - prevFailed;
1867
+ prevCompleted = completed;
1868
+ prevFailed = failed;
1869
+ if ((dCompleted > 0 ? dFailed / dCompleted : 0) > maxFailRate) delay = Math.min(max, Math.max(base, delay === 0 ? base : delay * factor));
1870
+ else delay = Math.max(min, delay - step);
1871
+ return delay;
1872
+ };
1873
+ }
1874
+ //#endregion
1875
+ //#region src/index.ts
1876
+ /**
1877
+ * # @ayepi/work
1878
+ *
1879
+ * Type-safe distributed work / job-queue + workflow engine. Define work types with
1880
+ * {@link defineWork} (each yields a typed, queueable builder), pass them to
1881
+ * {@link createWork} as a `const` registry, and `enqueue` is fully checked — by
1882
+ * instance or by name. Awaiting a handle resolves to the **group result**; work queued
1883
+ * inside a handler joins the same group.
1884
+ *
1885
+ * Built on three pluggable ports ({@link Queue}/{@link PubSub}/{@link Store}) with an
1886
+ * in-memory implementation bundled, so it runs zero-config and scales out by swapping
1887
+ * the ports for Redis/SQS/etc. Retries (backoff + jitter, by re-enqueue), batching,
1888
+ * non-blocking {@link dependency} gates, cron/fn {@link ScheduleConfig}, distributed
1889
+ * wait-for-result, in-process `skipQueue` execution, doers for concurrency/ordering,
1890
+ * an orphan-group hook, a JSON codec, and a `logWith` hook are all included.
1891
+ *
1892
+ * ```ts
1893
+ * import { defineWork, createWork } from '@ayepi/work'
1894
+ *
1895
+ * const add = defineWork('add', (i: { a: number; b: number }) => i.a + i.b)
1896
+ * const w = createWork({ work: [add] as const })
1897
+ *
1898
+ * const sum = await w.enqueue(add({ a: 1, b: 2 })).result() // 3, typed as number
1899
+ * await w.stop()
1900
+ * ```
1901
+ *
1902
+ * Bare `import` has **no side effects** — the default instance does not auto-start.
1903
+ *
1904
+ * @module
1905
+ */
1906
+ const instance = createWork({ autoStart: false });
1907
+ /** The default (registry-less) work system. Most apps call {@link createWork} with their own registry instead. */
1908
+ const work = instance;
1909
+ /** Enqueue on the default system (instance form). */
1910
+ const enqueue = instance.enqueue;
1911
+ /** Register a recurring schedule on the default system. */
1912
+ const schedule = instance.schedule;
1913
+ /** Start the default system's worker loop. */
1914
+ const start = () => instance.start();
1915
+ /** Stop the default system. */
1916
+ const stop = () => instance.stop();
1917
+ /** Snapshot the default system's known work states. */
1918
+ const list = () => instance.list();
1919
+ //#endregion
1920
+ export { DEFAULT_BUCKETS, DEFAULT_RETRY_OPTIONS, DEPENDENCY_TYPE, RetryAbort, WORK_METRICS, WorkDelayError, adaptiveDelay, ageDoer, backoff, balancedDoer, conditionMet, createMetrics, createWork, defaultCodec, defineBatchWork, defineWork, dependency, enqueue, formatPrometheus, getDefaultRetryOptions, list, memoryBackend, memoryPubSub, memoryQueue, memoryStore, nextAfter, parseCron, priorityDoer, retry, schedule, setDefaultRetryOptions, start, stop, unlimitedDoer, work };