@ayepi/work 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -18
- package/ayepi-work-deps-schedule.md +312 -0
- package/ayepi-work-ports.md +408 -0
- package/ayepi-work.md +926 -0
- package/dist/index.cjs +234 -56
- package/dist/index.d.cts +151 -50
- package/dist/index.d.ts +151 -50
- package/dist/index.js +234 -57
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -87,6 +87,16 @@ const defaultCodec = {
|
|
|
87
87
|
*/
|
|
88
88
|
/** A v4 UUID. Thin wrapper over `node:crypto` so callers don't import it directly. */
|
|
89
89
|
const uuid = () => randomUUID();
|
|
90
|
+
/** The active work-id generator (default {@link uuid}); swapped by {@link setIdGenerator}. */
|
|
91
|
+
let idGenerator = uuid;
|
|
92
|
+
/**
|
|
93
|
+
* Override how work/group ids are generated process-wide — e.g. sortable or prefixed ids. Affects
|
|
94
|
+
* **build-time** work ids (builders are minted outside a system) and any engine ids without their own
|
|
95
|
+
* `generateId`. Pass nothing to reset to the default UUID generator.
|
|
96
|
+
*/
|
|
97
|
+
const setIdGenerator = (fn) => void (idGenerator = fn ?? uuid);
|
|
98
|
+
/** Generate an id via the active generator. */
|
|
99
|
+
const genId = () => idGenerator();
|
|
90
100
|
/** Resolve after `ms` (an unref'd timer, so it never keeps the process alive). */
|
|
91
101
|
function sleep(ms) {
|
|
92
102
|
return new Promise((resolve) => {
|
|
@@ -425,7 +435,7 @@ const rehydrate = (s) => ({
|
|
|
425
435
|
});
|
|
426
436
|
/** Build a {@link DEPENDENCY_TYPE} work item from a (re-usable) input — a fresh queue id, same key. */
|
|
427
437
|
const buildDependency = (input) => ({
|
|
428
|
-
id:
|
|
438
|
+
id: genId(),
|
|
429
439
|
type: DEPENDENCY_TYPE,
|
|
430
440
|
input
|
|
431
441
|
});
|
|
@@ -435,7 +445,7 @@ const buildDependency = (input) => ({
|
|
|
435
445
|
*/
|
|
436
446
|
function dependency(opts) {
|
|
437
447
|
return buildDependency({
|
|
438
|
-
key:
|
|
448
|
+
key: genId(),
|
|
439
449
|
on: opts.on.map(toId),
|
|
440
450
|
queue: opts.queue.map(toSerialized),
|
|
441
451
|
config: opts.config ?? "all-success",
|
|
@@ -458,12 +468,9 @@ const dependencyHandler = async (input, ctx) => {
|
|
|
458
468
|
});
|
|
459
469
|
const freshById = new Map(unknownIds.map((id, i) => [id, fresh[i]]));
|
|
460
470
|
const states = input.on.map((id) => resolved[id] ? { status: resolved[id] } : freshById.get(id));
|
|
461
|
-
if (conditionMet(input.config, states)) {
|
|
462
|
-
if (await ctx.claim(`dep:${input.key}:fired`)) ctx.queue(input.queue.map(rehydrate));
|
|
463
|
-
return;
|
|
464
|
-
}
|
|
471
|
+
if (conditionMet(input.config, states)) return await ctx.claim(`dep:${input.key}:fired`) ? ctx.queue(input.queue.map(rehydrate)) : ctx.void();
|
|
465
472
|
if (input.deadline !== void 0 && Date.now() >= input.deadline) throw new Error(`dependency: timed out waiting for [${input.on.join(", ")}]`);
|
|
466
|
-
ctx.queue(buildDependency({
|
|
473
|
+
return ctx.queue(buildDependency({
|
|
467
474
|
...input,
|
|
468
475
|
resolved
|
|
469
476
|
}), { delay: input.poll });
|
|
@@ -610,6 +617,7 @@ const WORK_METRICS = {
|
|
|
610
617
|
retried: "work.retried",
|
|
611
618
|
deferred: "work.deferred",
|
|
612
619
|
rescheduled: "work.rescheduled",
|
|
620
|
+
expired: "work.expired",
|
|
613
621
|
active: "work.active",
|
|
614
622
|
pending: "work.pending",
|
|
615
623
|
running: "work.running",
|
|
@@ -642,6 +650,7 @@ const createWorkStats = (metrics = createMetrics$1()) => {
|
|
|
642
650
|
retried: metrics.counter(WORK_METRICS.retried, l, { description: "attempt-advancing retries" }),
|
|
643
651
|
deferred: metrics.counter(WORK_METRICS.deferred, l, { description: "handler/classifier reschedules" }),
|
|
644
652
|
rescheduled: metrics.counter(WORK_METRICS.rescheduled, l, { description: "early-arrival re-pushes" }),
|
|
653
|
+
expired: metrics.counter(WORK_METRICS.expired, l, { description: "items expired past their deadline" }),
|
|
645
654
|
active: metrics.gauge(WORK_METRICS.active, l, { description: "items in flight (pending + running)" }),
|
|
646
655
|
pending: metrics.gauge(WORK_METRICS.pending, l, { description: "items admitted, awaiting a doer slot" }),
|
|
647
656
|
running: metrics.gauge(WORK_METRICS.running, l, { description: "items executing" }),
|
|
@@ -737,6 +746,7 @@ const createWorkStats = (metrics = createMetrics$1()) => {
|
|
|
737
746
|
h.lastFailed.set(at);
|
|
738
747
|
},
|
|
739
748
|
retried: (type) => void handles(type).retried.inc(),
|
|
749
|
+
expired: (type) => void handles(type).expired.inc(),
|
|
740
750
|
deferred: (type, delayMs) => {
|
|
741
751
|
const h = handles(type);
|
|
742
752
|
h.deferred.inc();
|
|
@@ -756,23 +766,26 @@ const createWorkStats = (metrics = createMetrics$1()) => {
|
|
|
756
766
|
*
|
|
757
767
|
* Defining a work type with {@link defineWork} yields a **callable builder**: call it
|
|
758
768
|
* with the work's exact input and you get a type-checked, queueable {@link Work} (with
|
|
759
|
-
* a build-time id)
|
|
760
|
-
* `
|
|
761
|
-
*
|
|
769
|
+
* a build-time id). A handler **returns a {@link WorkResult}** — `ctx.result(value)`,
|
|
770
|
+
* `ctx.queue(...)`, `ctx.void()`, or a `.next(...)` dependency chain — so each work carries
|
|
771
|
+
* two inferred types: its *awaited-alone* result `S` and its *group* contribution `G`.
|
|
772
|
+
* {@link createWork} takes a `const` tuple of builders and produces a {@link WorkSystem} whose
|
|
773
|
+
* `enqueue` is fully checked — by instance or by name.
|
|
762
774
|
*
|
|
763
|
-
*
|
|
764
|
-
*
|
|
775
|
+
* `enqueue(root).group()` resolves to `root`'s `G` — a **precise union from the workflow
|
|
776
|
+
* structure**, not the whole registry. All durations are **milliseconds**.
|
|
765
777
|
*
|
|
766
778
|
* @module
|
|
767
779
|
*/
|
|
768
780
|
const build = (name, input) => ({
|
|
769
|
-
id:
|
|
781
|
+
id: genId(),
|
|
770
782
|
type: name,
|
|
771
783
|
input
|
|
772
784
|
});
|
|
773
785
|
/**
|
|
774
|
-
* Define a work type.
|
|
775
|
-
*
|
|
786
|
+
* Define a work type. The handler **returns a {@link WorkResult}** — `ctx.result(value)`,
|
|
787
|
+
* `ctx.queue(...)`, `ctx.void()`, or a `.next(...)` chain — and its `S`/`G` types are inferred
|
|
788
|
+
* from that return: `S` is what `.result()` resolves to alone, `G` what it contributes to the group.
|
|
776
789
|
*/
|
|
777
790
|
function defineWork(name, handler, opts = {}) {
|
|
778
791
|
return Object.assign((input) => build(name, input), {
|
|
@@ -785,10 +798,10 @@ function defineWork(name, handler, opts = {}) {
|
|
|
785
798
|
});
|
|
786
799
|
}
|
|
787
800
|
/**
|
|
788
|
-
* Define a **batched** work type. Items still enqueue, retry, prioritize, and join
|
|
789
|
-
*
|
|
790
|
-
*
|
|
791
|
-
*
|
|
801
|
+
* Define a **batched** work type. Items still enqueue, retry, prioritize, and join groups
|
|
802
|
+
* individually, but execute together via {@link BatchConfig.run} once `size` accumulate or `maxWait`
|
|
803
|
+
* ms elapse — so each `.result()` resolves to its aligned output `O` (which is also its group
|
|
804
|
+
* contribution). The per-type {@link WorkOptions.doer} governs how many *batches* run at once.
|
|
792
805
|
*/
|
|
793
806
|
function defineBatchWork(name, config) {
|
|
794
807
|
const { size, maxWait, run, ...options } = config;
|
|
@@ -797,7 +810,7 @@ function defineBatchWork(name, config) {
|
|
|
797
810
|
maxWait,
|
|
798
811
|
run
|
|
799
812
|
};
|
|
800
|
-
const handler = async (input) => (await run([input]))[0];
|
|
813
|
+
const handler = async (input, ctx) => ctx.result((await run([input]))[0]);
|
|
801
814
|
return Object.assign((input) => build(name, input), {
|
|
802
815
|
type: name,
|
|
803
816
|
def: {
|
|
@@ -858,6 +871,8 @@ const DEP_RETRY_ATTEMPTS = 1;
|
|
|
858
871
|
const REDRIVE_DEFAULT = 10;
|
|
859
872
|
const unref = (t) => void t.unref?.();
|
|
860
873
|
const errString = (err) => err instanceof Error ? `${err.name}: ${err.message}` : String(err);
|
|
874
|
+
/** Brand stamped on a runtime `WorkResult` node — distinguishes a handler instruction from a raw value. */
|
|
875
|
+
const WR_BRAND = Symbol.for("ayepi.work.result");
|
|
861
876
|
/**
|
|
862
877
|
* Create a work system. Zero-config (`createWork()`) uses the bundled in-memory backend
|
|
863
878
|
* and an {@link unlimitedDoer}; pass `work: [...] as const` for a typed registry, a
|
|
@@ -880,6 +895,8 @@ function createWork(opts = {}) {
|
|
|
880
895
|
const logWith = opts.logWith ?? identityLogWith;
|
|
881
896
|
const now = opts.now ?? Date.now;
|
|
882
897
|
const random = opts.random ?? Math.random;
|
|
898
|
+
const newId = opts.generateId ?? genId;
|
|
899
|
+
const strictReturnDefault = opts.strictReturn ?? true;
|
|
883
900
|
const attemptsOf = (r) => r.attempts ?? getDefaultRetryOptions$1().attempts ?? 1;
|
|
884
901
|
const stats = createWorkStats(opts.metrics);
|
|
885
902
|
const k = (suffix) => prefix + suffix;
|
|
@@ -943,11 +960,13 @@ function createWork(opts = {}) {
|
|
|
943
960
|
const resolveOptions = (type, input, qOpts) => {
|
|
944
961
|
const tOpts = registry.get(type)?.def.options;
|
|
945
962
|
const computed = tOpts?.options?.(input) ?? {};
|
|
963
|
+
const timeout = qOpts?.timeout ?? computed.timeout ?? tOpts?.timeout;
|
|
946
964
|
return {
|
|
947
965
|
delay: qOpts?.delay ?? computed.delay ?? 0,
|
|
948
966
|
runAt: qOpts?.runAt ?? computed.runAt,
|
|
949
967
|
priority: qOpts?.priority ?? computed.priority ?? tOpts?.priority ?? 0,
|
|
950
968
|
group: qOpts?.group ?? computed.group ?? tOpts?.group,
|
|
969
|
+
deadline: qOpts?.deadline ?? computed.deadline ?? (timeout !== void 0 ? now() + timeout : void 0),
|
|
951
970
|
retry: {
|
|
952
971
|
...getDefaultRetryOptions$1(),
|
|
953
972
|
...opts.retry,
|
|
@@ -1009,16 +1028,56 @@ function createWork(opts = {}) {
|
|
|
1009
1028
|
});
|
|
1010
1029
|
maybeUnhandled(groupId);
|
|
1011
1030
|
};
|
|
1012
|
-
const
|
|
1013
|
-
|
|
1031
|
+
const isWR = (x) => typeof x === "object" && x !== null && x[WR_BRAND] === true;
|
|
1032
|
+
const toItems = (x) => Array.isArray(x) ? [...x] : [x];
|
|
1033
|
+
const makeNode = (data) => {
|
|
1034
|
+
const node = {
|
|
1035
|
+
[WR_BRAND]: true,
|
|
1036
|
+
data,
|
|
1037
|
+
next(nextItems, condition = "all-success", options) {
|
|
1038
|
+
if (data.kind !== "queue") throw new Error(".next() is only valid on ctx.queue(...)");
|
|
1039
|
+
const q = toItems(nextItems);
|
|
1040
|
+
data.chains.push({
|
|
1041
|
+
on: data.frontier,
|
|
1042
|
+
queue: q,
|
|
1043
|
+
condition,
|
|
1044
|
+
options
|
|
1045
|
+
});
|
|
1046
|
+
data.frontier = q;
|
|
1047
|
+
return node;
|
|
1048
|
+
}
|
|
1049
|
+
};
|
|
1050
|
+
return node;
|
|
1051
|
+
};
|
|
1052
|
+
/** Collect the Work ids to wait on within items (recursing into nested `queue` nodes). */
|
|
1053
|
+
const workIdsOf = (items) => items.flatMap((it) => isWR(it) ? it.data.kind === "queue" ? workIdsOf(it.data.items) : [] : [it.id]);
|
|
1054
|
+
const makeContext = (meta, created) => {
|
|
1055
|
+
const reg = (n) => (created.add(n), n);
|
|
1014
1056
|
return {
|
|
1015
|
-
id:
|
|
1016
|
-
groupId:
|
|
1017
|
-
attempt:
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1057
|
+
id: meta.id,
|
|
1058
|
+
groupId: meta.groupId,
|
|
1059
|
+
attempt: meta.attempt,
|
|
1060
|
+
parent: meta.parent,
|
|
1061
|
+
dependents: meta.dependents,
|
|
1062
|
+
result: (value, options) => reg(makeNode({
|
|
1063
|
+
kind: "result",
|
|
1064
|
+
value,
|
|
1065
|
+
final: !!options?.final,
|
|
1066
|
+
append: options?.append
|
|
1067
|
+
})),
|
|
1068
|
+
queue: (items, options) => {
|
|
1069
|
+
const arr = toItems(items);
|
|
1070
|
+
return reg(makeNode({
|
|
1071
|
+
kind: "queue",
|
|
1072
|
+
items: arr,
|
|
1073
|
+
options,
|
|
1074
|
+
chains: [],
|
|
1075
|
+
frontier: arr
|
|
1076
|
+
}));
|
|
1077
|
+
},
|
|
1078
|
+
void: () => reg(makeNode({ kind: "void" })),
|
|
1079
|
+
states: storeStates,
|
|
1080
|
+
claim: storeClaim
|
|
1022
1081
|
};
|
|
1023
1082
|
};
|
|
1024
1083
|
const storeStates = (ids) => Promise.all(ids.map(readState));
|
|
@@ -1027,7 +1086,7 @@ function createWork(opts = {}) {
|
|
|
1027
1086
|
const pushEnvelope = (env, delay) => port(() => Promise.resolve(queueFor(env.type).push(JSON.stringify(env), { delay })).then(() => void 0));
|
|
1028
1087
|
/** Resolve when a deferred/scheduled item should next run (absolute epoch ms). */
|
|
1029
1088
|
const resolveRunAt = (when) => when.runAt ?? now() + (when.delay ?? 0);
|
|
1030
|
-
const submitQueued = (id, type, input, groupId, ro) => {
|
|
1089
|
+
const submitQueued = (id, type, input, groupId, ro, meta) => {
|
|
1031
1090
|
const queueAt = now();
|
|
1032
1091
|
const startAt = ro.runAt ?? queueAt + ro.delay;
|
|
1033
1092
|
return (async () => {
|
|
@@ -1053,7 +1112,10 @@ function createWork(opts = {}) {
|
|
|
1053
1112
|
attempt: 1,
|
|
1054
1113
|
priority: ro.priority,
|
|
1055
1114
|
group: ro.group,
|
|
1056
|
-
retry: ro.retry
|
|
1115
|
+
retry: ro.retry,
|
|
1116
|
+
deadline: ro.deadline,
|
|
1117
|
+
parent: meta?.parent,
|
|
1118
|
+
dependents: meta?.dependents
|
|
1057
1119
|
}, Math.max(0, startAt - queueAt));
|
|
1058
1120
|
} catch (err) {
|
|
1059
1121
|
await undoHold(groupId);
|
|
@@ -1065,10 +1127,14 @@ function createWork(opts = {}) {
|
|
|
1065
1127
|
id,
|
|
1066
1128
|
type,
|
|
1067
1129
|
groupId,
|
|
1130
|
+
parent: meta?.parent,
|
|
1131
|
+
dependents: meta?.dependents,
|
|
1068
1132
|
at: queueAt
|
|
1069
1133
|
});
|
|
1070
1134
|
})();
|
|
1071
1135
|
};
|
|
1136
|
+
/** Submit a built {@link Work} (used by the result-apply walk), carrying its parent/dependents metadata. */
|
|
1137
|
+
const submitWork = (work, groupId, options, meta) => submitQueued(work.id, work.type, work.input, groupId, resolveOptions(work.type, work.input, options), meta);
|
|
1072
1138
|
/** Re-enter the queue for a retry: a fresh delivery with `attempt + 1` and a recomputed `startAt`. */
|
|
1073
1139
|
const rePush = async (env, delay) => {
|
|
1074
1140
|
const startAt = now() + delay;
|
|
@@ -1091,11 +1157,11 @@ function createWork(opts = {}) {
|
|
|
1091
1157
|
};
|
|
1092
1158
|
const enqueueImpl = (a, b, c) => {
|
|
1093
1159
|
const fromName = typeof a === "string";
|
|
1094
|
-
const id = fromName ?
|
|
1160
|
+
const id = fromName ? newId() : a.id;
|
|
1095
1161
|
const type = fromName ? a : a.type;
|
|
1096
1162
|
const input = fromName ? b : a.input;
|
|
1097
1163
|
const qOpts = fromName ? c : b;
|
|
1098
|
-
const groupId =
|
|
1164
|
+
const groupId = newId();
|
|
1099
1165
|
const ro = resolveOptions(type, input, qOpts);
|
|
1100
1166
|
return makeHandle(id, type, groupId, ro.skipQueue ? runImmediate(id, type, input, groupId, ro) : submitQueued(id, type, input, groupId, ro));
|
|
1101
1167
|
};
|
|
@@ -1366,6 +1432,50 @@ function createWork(opts = {}) {
|
|
|
1366
1432
|
attempt: env.attempt,
|
|
1367
1433
|
error,
|
|
1368
1434
|
willRetry: false,
|
|
1435
|
+
parent: env.parent,
|
|
1436
|
+
dependents: env.dependents,
|
|
1437
|
+
at
|
|
1438
|
+
});
|
|
1439
|
+
await settleGroup(env.groupId);
|
|
1440
|
+
};
|
|
1441
|
+
/** Whether an item is past its deadline (so it should expire rather than run or retry). */
|
|
1442
|
+
const expired = (env) => env.deadline !== void 0 && now() > env.deadline;
|
|
1443
|
+
/** Terminal expiry: like a dead-letter, but emits `'expired'` (the item didn't start+finish by its deadline). */
|
|
1444
|
+
const expireItem = async (p, env, q) => {
|
|
1445
|
+
const at = now();
|
|
1446
|
+
const error = "deadline exceeded";
|
|
1447
|
+
stats.expired(env.type);
|
|
1448
|
+
await port(() => Promise.resolve(store.set(k(`item:${env.id}:error`), error, RESULT_TTL)));
|
|
1449
|
+
await setState({
|
|
1450
|
+
id: env.id,
|
|
1451
|
+
type: env.type,
|
|
1452
|
+
status: "dead",
|
|
1453
|
+
attempt: env.attempt,
|
|
1454
|
+
error,
|
|
1455
|
+
queueAt: env.queueAt,
|
|
1456
|
+
startAt: env.startAt,
|
|
1457
|
+
endAt: at,
|
|
1458
|
+
priority: env.priority,
|
|
1459
|
+
group: env.group
|
|
1460
|
+
}, env.groupId);
|
|
1461
|
+
if (p && q) {
|
|
1462
|
+
await Promise.resolve(q.deadLetter?.(p.body, error));
|
|
1463
|
+
await port(() => Promise.resolve(q.ack(p)));
|
|
1464
|
+
}
|
|
1465
|
+
notify({
|
|
1466
|
+
kind: "done",
|
|
1467
|
+
id: env.id,
|
|
1468
|
+
groupId: env.groupId,
|
|
1469
|
+
error
|
|
1470
|
+
});
|
|
1471
|
+
emit({
|
|
1472
|
+
kind: "expired",
|
|
1473
|
+
id: env.id,
|
|
1474
|
+
type: env.type,
|
|
1475
|
+
groupId: env.groupId,
|
|
1476
|
+
deadline: env.deadline,
|
|
1477
|
+
parent: env.parent,
|
|
1478
|
+
dependents: env.dependents,
|
|
1369
1479
|
at
|
|
1370
1480
|
});
|
|
1371
1481
|
await settleGroup(env.groupId);
|
|
@@ -1398,6 +1508,8 @@ function createWork(opts = {}) {
|
|
|
1398
1508
|
type: env.type,
|
|
1399
1509
|
groupId: env.groupId,
|
|
1400
1510
|
attempt: env.attempt,
|
|
1511
|
+
parent: env.parent,
|
|
1512
|
+
dependents: env.dependents,
|
|
1401
1513
|
at: runAt
|
|
1402
1514
|
});
|
|
1403
1515
|
};
|
|
@@ -1435,6 +1547,8 @@ function createWork(opts = {}) {
|
|
|
1435
1547
|
groupId: env.groupId,
|
|
1436
1548
|
attempt: env.attempt,
|
|
1437
1549
|
result: output,
|
|
1550
|
+
parent: env.parent,
|
|
1551
|
+
dependents: env.dependents,
|
|
1438
1552
|
at
|
|
1439
1553
|
});
|
|
1440
1554
|
await settleGroup(env.groupId);
|
|
@@ -1442,6 +1556,10 @@ function createWork(opts = {}) {
|
|
|
1442
1556
|
const finishFailure = async (p, env, runAt, err, q) => {
|
|
1443
1557
|
if (env.attempt < attemptsOf(env.retry)) {
|
|
1444
1558
|
const delay = backoff$1(env.attempt, env.retry, random);
|
|
1559
|
+
if (env.deadline !== void 0 && now() + delay > env.deadline) {
|
|
1560
|
+
await expireItem(p, env, q);
|
|
1561
|
+
return;
|
|
1562
|
+
}
|
|
1445
1563
|
stats.retried(env.type);
|
|
1446
1564
|
emit({
|
|
1447
1565
|
kind: "failed",
|
|
@@ -1451,6 +1569,8 @@ function createWork(opts = {}) {
|
|
|
1451
1569
|
attempt: env.attempt,
|
|
1452
1570
|
error: errString(err),
|
|
1453
1571
|
willRetry: true,
|
|
1572
|
+
parent: env.parent,
|
|
1573
|
+
dependents: env.dependents,
|
|
1454
1574
|
at: now()
|
|
1455
1575
|
});
|
|
1456
1576
|
if (p && q) await port(() => Promise.resolve(q.ack(p)));
|
|
@@ -1499,35 +1619,85 @@ function createWork(opts = {}) {
|
|
|
1499
1619
|
}
|
|
1500
1620
|
await finishFailure(p, env, runAt, err, q);
|
|
1501
1621
|
};
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
groupId:
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1622
|
+
/** Write a group-result contribution: `final` locks it; `append` folds into the existing value (read-modify-write, best-effort under concurrency). */
|
|
1623
|
+
const writeGroupResult = async (groupId, value, final, append) => {
|
|
1624
|
+
const resultKey = k(`group:${groupId}:result`);
|
|
1625
|
+
const finalKey = k(`group:${groupId}:final`);
|
|
1626
|
+
if (await Promise.resolve(store.get(finalKey)) !== void 0) return;
|
|
1627
|
+
let toWrite = value;
|
|
1628
|
+
if (append) {
|
|
1629
|
+
const ex = await Promise.resolve(store.get(resultKey));
|
|
1630
|
+
toWrite = append(ex === void 0 ? void 0 : globalCodec.parse(ex));
|
|
1631
|
+
}
|
|
1632
|
+
await port(() => Promise.resolve(store.set(resultKey, globalCodec.stringify(toWrite), RESULT_TTL)).then(() => void 0));
|
|
1633
|
+
if (final) await port(() => Promise.resolve(store.set(finalKey, "1", RESULT_TTL)).then(() => void 0));
|
|
1634
|
+
};
|
|
1635
|
+
/**
|
|
1636
|
+
* Run the returned WorkResult tree: enqueue works, build native dependencies (`.next`), write group
|
|
1637
|
+
* contributions. `assignDeps` (the `on` list when this is a fired dependency) tags non-dependency
|
|
1638
|
+
* children's `dependents`.
|
|
1639
|
+
*/
|
|
1640
|
+
const applyNode = (node, env, pending, assignDeps) => {
|
|
1641
|
+
const d = node.data;
|
|
1642
|
+
if (d.kind === "void") return;
|
|
1643
|
+
if (d.kind === "result") {
|
|
1644
|
+
pending.push(writeGroupResult(env.groupId, d.value, d.final, d.append));
|
|
1645
|
+
return;
|
|
1646
|
+
}
|
|
1647
|
+
for (const item of d.items) if (isWR(item)) applyNode(item, env, pending, assignDeps);
|
|
1648
|
+
else pending.push(submitWork(item, env.groupId, d.options, {
|
|
1649
|
+
parent: env.id,
|
|
1650
|
+
dependents: item.type === "@work/dependency" ? void 0 : assignDeps
|
|
1651
|
+
}));
|
|
1652
|
+
for (const chain of d.chains) {
|
|
1653
|
+
const dep = dependency({
|
|
1654
|
+
on: workIdsOf(chain.on),
|
|
1655
|
+
queue: chain.queue,
|
|
1656
|
+
config: chain.condition
|
|
1657
|
+
});
|
|
1658
|
+
pending.push(submitWork(dep, env.groupId, chain.options, { parent: env.id }));
|
|
1659
|
+
}
|
|
1660
|
+
};
|
|
1661
|
+
/** Strict-return: every created WorkResult must be reachable from the returned `root` (else it silently does nothing). */
|
|
1662
|
+
const reachableCheck = (root, created) => {
|
|
1663
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1664
|
+
const walk = (n) => {
|
|
1665
|
+
if (seen.has(n)) return;
|
|
1666
|
+
seen.add(n);
|
|
1667
|
+
if (n.data.kind === "queue") {
|
|
1668
|
+
for (const it of n.data.items) if (isWR(it)) walk(it);
|
|
1669
|
+
}
|
|
1670
|
+
};
|
|
1671
|
+
walk(root);
|
|
1672
|
+
for (const n of created) if (!seen.has(n)) throw new Error(`work "${root.data.kind}" created a ctx.queue()/result()/void() that was not returned (set strictReturn:false to allow detached instructions)`);
|
|
1673
|
+
};
|
|
1516
1674
|
/**
|
|
1517
1675
|
* The task handed to a doer: run a single item (leased from queue `q`, or `skipQueue` with `p`/`q`
|
|
1518
1676
|
* null). `release` is the {@link claim} teardown — {@link scoped} runs it in `finally`, so the
|
|
1519
|
-
* active-set entry + heartbeat are always cleaned up regardless of how the run ends.
|
|
1677
|
+
* active-set entry + heartbeat are always cleaned up regardless of how the run ends. The handler
|
|
1678
|
+
* returns a {@link WorkResult}; we then apply its tree (enqueue children, write group contributions).
|
|
1520
1679
|
*/
|
|
1521
1680
|
const execute = (p, env, input, def, q, release) => scoped(release, async () => {
|
|
1522
1681
|
const runAt = now();
|
|
1523
1682
|
markRunningSync(env, runAt);
|
|
1524
1683
|
const running = persistRunning(env, runAt).catch((err) => report(err, "commit"));
|
|
1525
|
-
const
|
|
1526
|
-
const ctx =
|
|
1527
|
-
|
|
1684
|
+
const created = /* @__PURE__ */ new Set();
|
|
1685
|
+
const ctx = makeContext({
|
|
1686
|
+
id: env.id,
|
|
1687
|
+
groupId: env.groupId,
|
|
1688
|
+
attempt: env.attempt,
|
|
1689
|
+
parent: env.parent,
|
|
1690
|
+
dependents: env.dependents
|
|
1691
|
+
}, created);
|
|
1692
|
+
let itemValue;
|
|
1528
1693
|
try {
|
|
1529
|
-
|
|
1694
|
+
const ret = await logWith(merge(opts.logContext?.(input, env.type), def.options.logContext?.(input)), () => Promise.resolve(def.handler(input, ctx)));
|
|
1695
|
+
if (!isWR(ret)) throw new Error(`work "${env.type}" handler must return a WorkResult — ctx.result(...), ctx.queue(...), or ctx.void()`);
|
|
1696
|
+
if (def.options.strictReturn ?? strictReturnDefault) reachableCheck(ret, created);
|
|
1697
|
+
const pending = [];
|
|
1698
|
+
applyNode(ret, env, pending, env.type === "@work/dependency" ? input.on : void 0);
|
|
1530
1699
|
await Promise.all(pending);
|
|
1700
|
+
itemValue = ret.data.kind === "result" ? ret.data.value : void 0;
|
|
1531
1701
|
} catch (err) {
|
|
1532
1702
|
await running;
|
|
1533
1703
|
await handleFailure(p, env, runAt, err, q);
|
|
@@ -1535,7 +1705,7 @@ function createWork(opts = {}) {
|
|
|
1535
1705
|
}
|
|
1536
1706
|
try {
|
|
1537
1707
|
await running;
|
|
1538
|
-
await finishSuccess(p, env, runAt,
|
|
1708
|
+
await finishSuccess(p, env, runAt, itemValue, q);
|
|
1539
1709
|
} catch (err) {
|
|
1540
1710
|
report(err, "commit");
|
|
1541
1711
|
}
|
|
@@ -1554,7 +1724,7 @@ function createWork(opts = {}) {
|
|
|
1554
1724
|
await Promise.all(items.map((it) => handleFailure(it.p, it.env, runAt, err, it.q)));
|
|
1555
1725
|
return;
|
|
1556
1726
|
}
|
|
1557
|
-
await Promise.all(items.map((it, i) => Promise.resolve(finishSuccess(it.p, it.env, runAt, outputs[i], it.q)).catch((err) => report(err, "commit"))));
|
|
1727
|
+
await Promise.all(items.map((it, i) => Promise.resolve(writeGroupResult(it.env.groupId, outputs[i], false)).then(() => finishSuccess(it.p, it.env, runAt, outputs[i], it.q)).catch((err) => report(err, "commit"))));
|
|
1558
1728
|
});
|
|
1559
1729
|
const batchers = /* @__PURE__ */ new Map();
|
|
1560
1730
|
const batcherFor = (type) => {
|
|
@@ -1607,6 +1777,10 @@ function createWork(opts = {}) {
|
|
|
1607
1777
|
await Promise.resolve(q.ack(p));
|
|
1608
1778
|
return false;
|
|
1609
1779
|
}
|
|
1780
|
+
if (expired(env)) {
|
|
1781
|
+
await expireItem(p, env, q);
|
|
1782
|
+
return false;
|
|
1783
|
+
}
|
|
1610
1784
|
const due = now();
|
|
1611
1785
|
if (env.startAt - due > SCHED_TOLERANCE) {
|
|
1612
1786
|
await reschedule(p, env, q, env.startAt);
|
|
@@ -1780,7 +1954,10 @@ function createWork(opts = {}) {
|
|
|
1780
1954
|
await sleep(pollInterval);
|
|
1781
1955
|
}
|
|
1782
1956
|
};
|
|
1783
|
-
registry.set(DEPENDENCY_TYPE, defineWork(DEPENDENCY_TYPE, dependencyHandler, {
|
|
1957
|
+
registry.set(DEPENDENCY_TYPE, defineWork(DEPENDENCY_TYPE, dependencyHandler, {
|
|
1958
|
+
retry: { attempts: DEP_RETRY_ATTEMPTS },
|
|
1959
|
+
strictReturn: false
|
|
1960
|
+
}));
|
|
1784
1961
|
const start = () => {
|
|
1785
1962
|
if (running) return;
|
|
1786
1963
|
running = true;
|
|
@@ -1892,7 +2069,7 @@ function adaptiveDelay(opts = {}) {
|
|
|
1892
2069
|
* ```ts
|
|
1893
2070
|
* import { defineWork, createWork } from '@ayepi/work'
|
|
1894
2071
|
*
|
|
1895
|
-
* const add = defineWork('add', (i: { a: number; b: number }) => i.a + i.b)
|
|
2072
|
+
* const add = defineWork('add', (i: { a: number; b: number }, ctx) => ctx.result(i.a + i.b))
|
|
1896
2073
|
* const w = createWork({ work: [add] as const })
|
|
1897
2074
|
*
|
|
1898
2075
|
* const sum = await w.enqueue(add({ a: 1, b: 2 })).result() // 3, typed as number
|
|
@@ -1917,4 +2094,4 @@ const stop = () => instance.stop();
|
|
|
1917
2094
|
/** Snapshot the default system's known work states. */
|
|
1918
2095
|
const list = () => instance.list();
|
|
1919
2096
|
//#endregion
|
|
1920
|
-
export { DEFAULT_BUCKETS, DEFAULT_RETRY_OPTIONS, DEPENDENCY_TYPE, RetryAbort, WORK_METRICS, WorkDelayError, adaptiveDelay, ageDoer, backoff, balancedDoer, conditionMet, createMetrics, createWork, defaultCodec, defineBatchWork, defineWork, dependency, enqueue, formatPrometheus, getDefaultRetryOptions, list, memoryBackend, memoryPubSub, memoryQueue, memoryStore, nextAfter, parseCron, priorityDoer, retry, schedule, setDefaultRetryOptions, start, stop, unlimitedDoer, work };
|
|
2097
|
+
export { DEFAULT_BUCKETS, DEFAULT_RETRY_OPTIONS, DEPENDENCY_TYPE, RetryAbort, WORK_METRICS, WorkDelayError, adaptiveDelay, ageDoer, backoff, balancedDoer, conditionMet, createMetrics, createWork, defaultCodec, defineBatchWork, defineWork, dependency, enqueue, formatPrometheus, getDefaultRetryOptions, list, memoryBackend, memoryPubSub, memoryQueue, memoryStore, nextAfter, parseCron, priorityDoer, retry, schedule, setDefaultRetryOptions, setIdGenerator, start, stop, unlimitedDoer, work };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ayepi/work",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Type-safe distributed work / job-queue + workflow engine for ayepi — pluggable queue/pubsub/kv ports, in-memory bundled, retries, dependencies, scheduling",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
"type": "module",
|
|
19
19
|
"sideEffects": false,
|
|
20
20
|
"files": [
|
|
21
|
-
"dist"
|
|
21
|
+
"dist",
|
|
22
|
+
"ayepi-*.md"
|
|
22
23
|
],
|
|
23
24
|
"exports": {
|
|
24
25
|
".": {
|
|
@@ -37,14 +38,14 @@
|
|
|
37
38
|
"node": ">=18"
|
|
38
39
|
},
|
|
39
40
|
"peerDependencies": {
|
|
40
|
-
"@ayepi/core": "^0.
|
|
41
|
+
"@ayepi/core": "^0.2.0"
|
|
41
42
|
},
|
|
42
43
|
"devDependencies": {
|
|
43
44
|
"@vitest/coverage-v8": "^2.1.8",
|
|
44
45
|
"publint": "^0.3.0",
|
|
45
46
|
"tsdown": "^0.12.0",
|
|
46
47
|
"vitest": "^2.1.8",
|
|
47
|
-
"@ayepi/core": "0.
|
|
48
|
+
"@ayepi/core": "0.2.0"
|
|
48
49
|
},
|
|
49
50
|
"keywords": [
|
|
50
51
|
"ayepi",
|