@dudousxd/nestjs-durable-core 0.78.0 → 0.79.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1063 -78
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +396 -21
- package/dist/index.d.ts +396 -21
- package/dist/index.js +1040 -78
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -22,6 +22,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
22
22
|
// src/index.ts
|
|
23
23
|
var index_exports = {};
|
|
24
24
|
__export(index_exports, {
|
|
25
|
+
CHILD_EDGE_PREFIXES: () => CHILD_EDGE_PREFIXES,
|
|
25
26
|
CONTROL_FLOW_SIGNAL: () => CONTROL_FLOW_SIGNAL,
|
|
26
27
|
CURRENT_PROTOCOL_VERSION: () => CURRENT_PROTOCOL_VERSION,
|
|
27
28
|
CodecStateStore: () => CodecStateStore,
|
|
@@ -42,6 +43,7 @@ __export(index_exports, {
|
|
|
42
43
|
FAN_SUMMARY_FAILURES: () => FAN_SUMMARY_FAILURES,
|
|
43
44
|
FatalError: () => FatalError,
|
|
44
45
|
GatherError: () => GatherError,
|
|
46
|
+
HANDOFF_SEQ_BASE: () => HANDOFF_SEQ_BASE,
|
|
45
47
|
InMemoryAdmissionBackend: () => InMemoryAdmissionBackend,
|
|
46
48
|
InMemoryStateStore: () => InMemoryStateStore,
|
|
47
49
|
InMemoryTransport: () => InMemoryTransport,
|
|
@@ -76,9 +78,11 @@ __export(index_exports, {
|
|
|
76
78
|
WorkflowEngine: () => WorkflowEngine,
|
|
77
79
|
WorkflowSuspended: () => WorkflowSuspended,
|
|
78
80
|
aggregateAnnouncements: () => aggregateAnnouncements,
|
|
81
|
+
aggregatesCheckpoints: () => aggregatesCheckpoints,
|
|
79
82
|
announcementsOf: () => announcementsOf,
|
|
80
83
|
applyAttributeQuery: () => applyAttributeQuery,
|
|
81
84
|
assertConcurrency: () => assertConcurrency,
|
|
85
|
+
assertRoutes: () => assertRoutes,
|
|
82
86
|
attributeColumnFor: () => attributeColumnFor,
|
|
83
87
|
attributeOperand: () => attributeOperand,
|
|
84
88
|
attributePredicateOperands: () => attributePredicateOperands,
|
|
@@ -88,10 +92,14 @@ __export(index_exports, {
|
|
|
88
92
|
buildWorkflowDirectory: () => buildWorkflowDirectory,
|
|
89
93
|
canRoute: () => canRoute,
|
|
90
94
|
capableWorkers: () => capableWorkers,
|
|
95
|
+
checkpointNameKind: () => checkpointNameKind,
|
|
96
|
+
checkpointStatsOf: () => checkpointStatsOf,
|
|
91
97
|
childIdsOf: () => childIdsOf,
|
|
92
98
|
classifyWaiterToken: () => classifyWaiterToken,
|
|
93
99
|
clearDurableExecutionWrappers: () => clearDurableExecutionWrappers,
|
|
94
100
|
clearTerminalSingletonAdmission: () => clearTerminalSingletonAdmission,
|
|
101
|
+
codeAndDetails: () => codeAndDetails,
|
|
102
|
+
codeOf: () => codeOf,
|
|
95
103
|
collectMetrics: () => collectMetrics,
|
|
96
104
|
concurrencyTag: () => concurrencyTag,
|
|
97
105
|
controlPlaneDescriptor: () => controlPlaneDescriptor,
|
|
@@ -104,13 +112,20 @@ __export(index_exports, {
|
|
|
104
112
|
debug: () => debug,
|
|
105
113
|
describeWorker: () => describeWorker,
|
|
106
114
|
descriptorHash: () => descriptorHash,
|
|
115
|
+
detailsOf: () => detailsOf,
|
|
107
116
|
error: () => error,
|
|
117
|
+
etagMatches: () => etagMatches,
|
|
108
118
|
eventMatchOf: () => eventMatchOf,
|
|
109
119
|
eventMatches: () => eventMatches,
|
|
110
120
|
eventNameOf: () => eventNameOf,
|
|
111
121
|
eventPrefix: () => eventPrefix,
|
|
112
122
|
eventToken: () => eventToken,
|
|
113
123
|
facetOrigin: () => facetOrigin,
|
|
124
|
+
fanPageBounds: () => fanPageBounds,
|
|
125
|
+
fanPageEtag: () => fanPageEtag,
|
|
126
|
+
foldedCount: () => foldedCount,
|
|
127
|
+
handoffHandlerId: () => handoffHandlerId,
|
|
128
|
+
handoffName: () => handoffName,
|
|
114
129
|
hasDurableExecutionWrappers: () => hasDurableExecutionWrappers,
|
|
115
130
|
heartbeatStatus: () => heartbeatStatus,
|
|
116
131
|
indexWaitersByRun: () => indexWaitersByRun,
|
|
@@ -122,6 +137,7 @@ __export(index_exports, {
|
|
|
122
137
|
isNonEmptyRunScope: () => isNonEmptyRunScope,
|
|
123
138
|
isWorkflowControlFlowSignal: () => isWorkflowControlFlowSignal,
|
|
124
139
|
listSignalWaitersOfRuns: () => listSignalWaitersOfRuns,
|
|
140
|
+
liveHandoff: () => liveHandoff,
|
|
125
141
|
log: () => log,
|
|
126
142
|
matchesAttributes: () => matchesAttributes,
|
|
127
143
|
mergeRunFacetRows: () => mergeRunFacetRows,
|
|
@@ -132,17 +148,23 @@ __export(index_exports, {
|
|
|
132
148
|
normalizeAttributeRows: () => normalizeAttributeRows,
|
|
133
149
|
normalizeDescriptor: () => normalizeDescriptor,
|
|
134
150
|
observedAnnouncements: () => observedAnnouncements,
|
|
151
|
+
onErrorRunId: () => onErrorRunId,
|
|
135
152
|
pageFan: () => pageFan,
|
|
153
|
+
pageFanFromStore: () => pageFanFromStore,
|
|
136
154
|
parentRunIdOf: () => parentRunIdOf,
|
|
137
155
|
parseDuration: () => parseDuration,
|
|
138
156
|
partitionOfToken: () => partitionOfToken,
|
|
139
157
|
planDispatch: () => planDispatch,
|
|
140
158
|
prevCronFireMs: () => prevCronFireMs,
|
|
159
|
+
queryCheckpointsOf: () => queryCheckpointsOf,
|
|
141
160
|
readSearchAttributes: () => readSearchAttributes,
|
|
142
161
|
requiredCapabilities: () => requiredCapabilities,
|
|
143
162
|
resolveConcurrency: () => resolveConcurrency,
|
|
144
163
|
resolveRouting: () => resolveRouting,
|
|
145
164
|
resolveRunWaiting: () => resolveRunWaiting,
|
|
165
|
+
retiredHandoffName: () => retiredHandoffName,
|
|
166
|
+
routeFor: () => routeFor,
|
|
167
|
+
runDetailEtag: () => runDetailEtag,
|
|
146
168
|
runDurableExecution: () => runDurableExecution,
|
|
147
169
|
runInStepLogger: () => runInStepLogger,
|
|
148
170
|
runInStepScope: () => runInStepScope,
|
|
@@ -163,6 +185,7 @@ __export(index_exports, {
|
|
|
163
185
|
subEvent: () => subEvent,
|
|
164
186
|
subProcess: () => subProcess,
|
|
165
187
|
summarizeFan: () => summarizeFan,
|
|
188
|
+
summarizeRunFromStore: () => summarizeRunFromStore,
|
|
166
189
|
summarizeTimeline: () => summarizeTimeline,
|
|
167
190
|
tenantGroup: () => tenantGroup,
|
|
168
191
|
useDurableExecution: () => useDurableExecution,
|
|
@@ -435,10 +458,16 @@ var FatalError = class extends Error {
|
|
|
435
458
|
__name(this, "FatalError");
|
|
436
459
|
}
|
|
437
460
|
code;
|
|
438
|
-
|
|
461
|
+
/** Structured, JSON-serializable detail (e.g. validation issues). It travels with the failure to
|
|
462
|
+
* the run's error, a parent's `allSettled` item and an onError handler's input. */
|
|
463
|
+
details;
|
|
464
|
+
/** Read by the step result builders, so a dispatched step's retry policy honours it too. */
|
|
465
|
+
retryable = false;
|
|
466
|
+
constructor(message, code, details) {
|
|
439
467
|
super(message);
|
|
440
468
|
this.name = "FatalError";
|
|
441
469
|
this.code = code;
|
|
470
|
+
this.details = details;
|
|
442
471
|
}
|
|
443
472
|
};
|
|
444
473
|
var ConcurrencyLimitError = class extends Error {
|
|
@@ -516,6 +545,8 @@ var NonDeterminismError = class extends Error {
|
|
|
516
545
|
static {
|
|
517
546
|
__name(this, "NonDeterminismError");
|
|
518
547
|
}
|
|
548
|
+
/** Marks the failure as a broken deploy, which an `onError` route never takes. */
|
|
549
|
+
code = "non_determinism";
|
|
519
550
|
runId;
|
|
520
551
|
seq;
|
|
521
552
|
constructor(runId, seq, expected, recorded) {
|
|
@@ -927,6 +958,9 @@ async function runStepHandler(task, handler) {
|
|
|
927
958
|
...typeof e?.code === "string" ? {
|
|
928
959
|
code: e.code
|
|
929
960
|
} : {},
|
|
961
|
+
...e?.details !== void 0 ? {
|
|
962
|
+
details: e.details
|
|
963
|
+
} : {},
|
|
930
964
|
...typeof e?.retryable === "boolean" ? {
|
|
931
965
|
retryable: e.retryable
|
|
932
966
|
} : {}
|
|
@@ -1003,11 +1037,36 @@ __name(isFailedCompletion, "isFailedCompletion");
|
|
|
1003
1037
|
function unwrapCompletion(payload, label) {
|
|
1004
1038
|
const c = payload;
|
|
1005
1039
|
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
1006
|
-
throw new FatalError(`${label} failed: ${c.error}
|
|
1040
|
+
throw new FatalError(`${label} failed: ${c.error}`, c.code, c.details);
|
|
1007
1041
|
}
|
|
1008
1042
|
return c?.value;
|
|
1009
1043
|
}
|
|
1010
1044
|
__name(unwrapCompletion, "unwrapCompletion");
|
|
1045
|
+
function failedCompletion(error2) {
|
|
1046
|
+
return {
|
|
1047
|
+
ok: false,
|
|
1048
|
+
error: error2.message,
|
|
1049
|
+
...error2.code !== void 0 ? {
|
|
1050
|
+
code: error2.code
|
|
1051
|
+
} : {},
|
|
1052
|
+
...error2.details !== void 0 ? {
|
|
1053
|
+
details: error2.details
|
|
1054
|
+
} : {}
|
|
1055
|
+
};
|
|
1056
|
+
}
|
|
1057
|
+
__name(failedCompletion, "failedCompletion");
|
|
1058
|
+
function completionError(c) {
|
|
1059
|
+
return {
|
|
1060
|
+
message: c.error,
|
|
1061
|
+
...c.code !== void 0 ? {
|
|
1062
|
+
code: c.code
|
|
1063
|
+
} : {},
|
|
1064
|
+
...c.details !== void 0 ? {
|
|
1065
|
+
details: c.details
|
|
1066
|
+
} : {}
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
__name(completionError, "completionError");
|
|
1011
1070
|
|
|
1012
1071
|
// src/concurrency.ts
|
|
1013
1072
|
var DEFAULT_CONCURRENCY_STATUSES = [
|
|
@@ -1328,7 +1387,7 @@ function allWaves(size, concurrency) {
|
|
|
1328
1387
|
};
|
|
1329
1388
|
}
|
|
1330
1389
|
__name(allWaves, "allWaves");
|
|
1331
|
-
var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+)$/;
|
|
1390
|
+
var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+|onError\.\d+)$/;
|
|
1332
1391
|
function parentRunIdOf(runId) {
|
|
1333
1392
|
return ENGINE_CHILD_ID.exec(runId)?.[1];
|
|
1334
1393
|
}
|
|
@@ -1413,7 +1472,11 @@ function canonicalizeRegistrations(regs) {
|
|
|
1413
1472
|
requires: r.requires ? [
|
|
1414
1473
|
...new Set(r.requires)
|
|
1415
1474
|
].sort() : null,
|
|
1416
|
-
origin: r.origin ?? null
|
|
1475
|
+
origin: r.origin ?? null,
|
|
1476
|
+
// Conditional, so a registration that states no routes keeps the hash it had before routes existed.
|
|
1477
|
+
...r.onError !== void 0 ? {
|
|
1478
|
+
onError: canonicalRoutes(r.onError)
|
|
1479
|
+
} : {}
|
|
1417
1480
|
}));
|
|
1418
1481
|
const byJson = /* @__PURE__ */ new Map();
|
|
1419
1482
|
for (const entry of canonical) byJson.set(JSON.stringify(entry), entry);
|
|
@@ -1422,6 +1485,20 @@ function canonicalizeRegistrations(regs) {
|
|
|
1422
1485
|
].sort().map((k) => byJson.get(k));
|
|
1423
1486
|
}
|
|
1424
1487
|
__name(canonicalizeRegistrations, "canonicalizeRegistrations");
|
|
1488
|
+
function canonicalRoutes(routes) {
|
|
1489
|
+
const byJson = /* @__PURE__ */ new Map();
|
|
1490
|
+
for (const r of routes) {
|
|
1491
|
+
const entry = {
|
|
1492
|
+
code: r.code ?? null,
|
|
1493
|
+
workflow: r.workflow
|
|
1494
|
+
};
|
|
1495
|
+
byJson.set(JSON.stringify(entry), entry);
|
|
1496
|
+
}
|
|
1497
|
+
return [
|
|
1498
|
+
...byJson.keys()
|
|
1499
|
+
].sort().map((k) => byJson.get(k));
|
|
1500
|
+
}
|
|
1501
|
+
__name(canonicalRoutes, "canonicalRoutes");
|
|
1425
1502
|
function canonicalizeForHash(d) {
|
|
1426
1503
|
const set = /* @__PURE__ */ __name((xs) => [
|
|
1427
1504
|
...new Set(xs)
|
|
@@ -1456,13 +1533,13 @@ function fnv1a64Hex(input) {
|
|
|
1456
1533
|
const OFFSET = 0xcbf29ce484222325n;
|
|
1457
1534
|
const PRIME = 0x100000001b3n;
|
|
1458
1535
|
const MASK = 0xffffffffffffffffn;
|
|
1459
|
-
let
|
|
1536
|
+
let hash2 = OFFSET;
|
|
1460
1537
|
for (let i = 0; i < input.length; i += 1) {
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1538
|
+
hash2 ^= BigInt(input.charCodeAt(i) & 255);
|
|
1539
|
+
hash2 ^= BigInt(input.charCodeAt(i) >> 8 & 255);
|
|
1540
|
+
hash2 = hash2 * PRIME & MASK;
|
|
1464
1541
|
}
|
|
1465
|
-
return
|
|
1542
|
+
return hash2.toString(16).padStart(16, "0");
|
|
1466
1543
|
}
|
|
1467
1544
|
__name(fnv1a64Hex, "fnv1a64Hex");
|
|
1468
1545
|
function descriptorHash(descriptor) {
|
|
@@ -1578,6 +1655,20 @@ function aggregateAnnouncements(descriptors) {
|
|
|
1578
1655
|
const requires = [
|
|
1579
1656
|
...requiresByJson.values()
|
|
1580
1657
|
].sort((x, y) => x.join(",").localeCompare(y.join(",")));
|
|
1658
|
+
const routesByJson = /* @__PURE__ */ new Map();
|
|
1659
|
+
for (const a of announcements) {
|
|
1660
|
+
if (a.onError === void 0) continue;
|
|
1661
|
+
const canonical = canonicalRoutes(a.onError).map((r) => r.code === null ? {
|
|
1662
|
+
workflow: r.workflow
|
|
1663
|
+
} : {
|
|
1664
|
+
code: r.code,
|
|
1665
|
+
workflow: r.workflow
|
|
1666
|
+
});
|
|
1667
|
+
routesByJson.set(JSON.stringify(canonical), canonical);
|
|
1668
|
+
}
|
|
1669
|
+
const onError = [
|
|
1670
|
+
...routesByJson.keys()
|
|
1671
|
+
].sort().map((k) => routesByJson.get(k) ?? []);
|
|
1581
1672
|
const disagreements = [];
|
|
1582
1673
|
if (groups.length > 1) disagreements.push({
|
|
1583
1674
|
axis: "group",
|
|
@@ -1593,6 +1684,14 @@ function aggregateAnnouncements(descriptors) {
|
|
|
1593
1684
|
values: requires.map((r) => r.join(","))
|
|
1594
1685
|
});
|
|
1595
1686
|
}
|
|
1687
|
+
if (onError.length > 1) {
|
|
1688
|
+
disagreements.push({
|
|
1689
|
+
axis: "onError",
|
|
1690
|
+
values: [
|
|
1691
|
+
...routesByJson.keys()
|
|
1692
|
+
].sort()
|
|
1693
|
+
});
|
|
1694
|
+
}
|
|
1596
1695
|
const first = announcements[0];
|
|
1597
1696
|
if (!first) continue;
|
|
1598
1697
|
out.push({
|
|
@@ -1605,6 +1704,7 @@ function aggregateAnnouncements(descriptors) {
|
|
|
1605
1704
|
groups,
|
|
1606
1705
|
origins,
|
|
1607
1706
|
requires,
|
|
1707
|
+
onError,
|
|
1608
1708
|
runtimes: [
|
|
1609
1709
|
...new Set(announcements.map((a) => a.runtime))
|
|
1610
1710
|
].sort(),
|
|
@@ -1638,6 +1738,7 @@ function observedAnnouncements(heartbeats, covered) {
|
|
|
1638
1738
|
],
|
|
1639
1739
|
origins: [],
|
|
1640
1740
|
requires: [],
|
|
1741
|
+
onError: [],
|
|
1641
1742
|
runtimes: [],
|
|
1642
1743
|
instances: [
|
|
1643
1744
|
...instances
|
|
@@ -1996,6 +2097,13 @@ function planDispatch(requires, liveDescriptors, cp, token) {
|
|
|
1996
2097
|
__name(planDispatch, "planDispatch");
|
|
1997
2098
|
|
|
1998
2099
|
// src/interfaces.ts
|
|
2100
|
+
function checkpointNameKind(name) {
|
|
2101
|
+
if (name.startsWith("signal:child:")) return "child";
|
|
2102
|
+
if (name.startsWith("fan:")) return "folded";
|
|
2103
|
+
if (name.startsWith("spawn:")) return "spawn";
|
|
2104
|
+
return "step";
|
|
2105
|
+
}
|
|
2106
|
+
__name(checkpointNameKind, "checkpointNameKind");
|
|
1999
2107
|
var TERMINAL_RUN_STATUSES = [
|
|
2000
2108
|
"completed",
|
|
2001
2109
|
"failed",
|
|
@@ -2008,6 +2116,71 @@ function isNonEmptyRunScope(scope) {
|
|
|
2008
2116
|
}
|
|
2009
2117
|
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
2010
2118
|
|
|
2119
|
+
// src/on-error.ts
|
|
2120
|
+
var UNROUTABLE_CODES = /* @__PURE__ */ new Set([
|
|
2121
|
+
"non_determinism",
|
|
2122
|
+
"nested_workflow_call"
|
|
2123
|
+
]);
|
|
2124
|
+
var HANDOFF_SEQ_BASE = 1e9;
|
|
2125
|
+
var onErrorRunId = /* @__PURE__ */ __name((runId, attempt) => `${runId}.onError.${attempt}`, "onErrorRunId");
|
|
2126
|
+
var handoffName = /* @__PURE__ */ __name((handlerId) => `signal:child:${handlerId}`, "handoffName");
|
|
2127
|
+
var retiredHandoffName = /* @__PURE__ */ __name((handlerId) => `onError:retried:${handlerId}`, "retiredHandoffName");
|
|
2128
|
+
function handoffHandlerId(runId, cp) {
|
|
2129
|
+
const prefix = `signal:child:${runId}.onError.`;
|
|
2130
|
+
return cp.seq >= HANDOFF_SEQ_BASE && cp.name.startsWith(prefix) ? cp.name.slice("signal:child:".length) : void 0;
|
|
2131
|
+
}
|
|
2132
|
+
__name(handoffHandlerId, "handoffHandlerId");
|
|
2133
|
+
function liveHandoff(runId, checkpoints) {
|
|
2134
|
+
let live;
|
|
2135
|
+
for (const cp of checkpoints) {
|
|
2136
|
+
if (handoffHandlerId(runId, cp) !== void 0 && (!live || cp.seq > live.seq)) live = cp;
|
|
2137
|
+
}
|
|
2138
|
+
return live;
|
|
2139
|
+
}
|
|
2140
|
+
__name(liveHandoff, "liveHandoff");
|
|
2141
|
+
function routeFor(routes, error2) {
|
|
2142
|
+
if (!routes?.length || error2.code && UNROUTABLE_CODES.has(error2.code)) return void 0;
|
|
2143
|
+
return (error2.code ? routes.find((r) => r.code === error2.code) : void 0) ?? routes.find((r) => r.code === void 0);
|
|
2144
|
+
}
|
|
2145
|
+
__name(routeFor, "routeFor");
|
|
2146
|
+
function assertRoutes(workflow, routes) {
|
|
2147
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2148
|
+
for (const r of routes) {
|
|
2149
|
+
if (seen.has(r.code)) {
|
|
2150
|
+
throw new Error(r.code === void 0 ? `workflow ${workflow}: more than one onError route without a code` : `workflow ${workflow}: more than one onError route for code "${r.code}"`);
|
|
2151
|
+
}
|
|
2152
|
+
seen.add(r.code);
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2155
|
+
__name(assertRoutes, "assertRoutes");
|
|
2156
|
+
function codeOf(err) {
|
|
2157
|
+
const e = err;
|
|
2158
|
+
return [
|
|
2159
|
+
e?.code,
|
|
2160
|
+
e?.stepError?.code,
|
|
2161
|
+
e?.error?.code
|
|
2162
|
+
].find((c) => typeof c === "string");
|
|
2163
|
+
}
|
|
2164
|
+
__name(codeOf, "codeOf");
|
|
2165
|
+
function detailsOf(err) {
|
|
2166
|
+
const e = err;
|
|
2167
|
+
return e?.details ?? e?.stepError?.details ?? e?.error?.details;
|
|
2168
|
+
}
|
|
2169
|
+
__name(detailsOf, "detailsOf");
|
|
2170
|
+
function codeAndDetails(err) {
|
|
2171
|
+
const code = codeOf(err);
|
|
2172
|
+
const details = detailsOf(err);
|
|
2173
|
+
return {
|
|
2174
|
+
...code !== void 0 ? {
|
|
2175
|
+
code
|
|
2176
|
+
} : {},
|
|
2177
|
+
...details !== void 0 ? {
|
|
2178
|
+
details
|
|
2179
|
+
} : {}
|
|
2180
|
+
};
|
|
2181
|
+
}
|
|
2182
|
+
__name(codeAndDetails, "codeAndDetails");
|
|
2183
|
+
|
|
2011
2184
|
// src/tenant-group.ts
|
|
2012
2185
|
function tenantGroup(baseGroup, tenant) {
|
|
2013
2186
|
return tenant !== void 0 && tenant !== "" && tenant !== "default" ? `${baseGroup}@${tenant}` : baseGroup;
|
|
@@ -3067,7 +3240,8 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3067
3240
|
kind: "local",
|
|
3068
3241
|
error: {
|
|
3069
3242
|
message: err instanceof Error ? err.message : String(err),
|
|
3070
|
-
stack: err instanceof Error ? err.stack : void 0
|
|
3243
|
+
stack: err instanceof Error ? err.stack : void 0,
|
|
3244
|
+
...codeAndDetails(err)
|
|
3071
3245
|
},
|
|
3072
3246
|
events,
|
|
3073
3247
|
attempts: attempt,
|
|
@@ -3323,12 +3497,12 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3323
3497
|
}
|
|
3324
3498
|
]));
|
|
3325
3499
|
}, "child");
|
|
3326
|
-
const
|
|
3500
|
+
const fan = /* @__PURE__ */ __name(async (workflow2, inputs, opts, settle) => {
|
|
3327
3501
|
if (inputs.length === 0) return [];
|
|
3328
3502
|
if (stepNameOf(workflow2) !== void 0) {
|
|
3329
|
-
return allSteps(workflow2, inputs, opts);
|
|
3503
|
+
return allSteps(workflow2, inputs, opts, settle);
|
|
3330
3504
|
}
|
|
3331
|
-
const mode = opts?.mode ?? "waitAll";
|
|
3505
|
+
const mode = settle ? "waitAll" : opts?.mode ?? "waitAll";
|
|
3332
3506
|
const name = workflowName(workflow2);
|
|
3333
3507
|
const waves = allWaves(inputs.length, opts?.concurrency);
|
|
3334
3508
|
const positions = inputs.map(() => pos.next());
|
|
@@ -3450,24 +3624,45 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3450
3624
|
throw new WorkflowSuspended(void 0, fanJoin(active.map((k) => allWait(runId, firstSeq, waves ? k : void 0))));
|
|
3451
3625
|
}
|
|
3452
3626
|
const outputs = [];
|
|
3627
|
+
const results = [];
|
|
3453
3628
|
const failures = [];
|
|
3454
3629
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
3455
3630
|
const c = existing[i]?.output;
|
|
3456
3631
|
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
3632
|
+
const { message, ...rest } = completionError({
|
|
3633
|
+
...c,
|
|
3634
|
+
error: c.error ?? "unknown"
|
|
3635
|
+
});
|
|
3636
|
+
const failure = {
|
|
3637
|
+
id: id(i),
|
|
3638
|
+
error: message,
|
|
3639
|
+
...rest
|
|
3640
|
+
};
|
|
3457
3641
|
failures.push({
|
|
3458
3642
|
index: i,
|
|
3459
|
-
|
|
3460
|
-
|
|
3643
|
+
...failure
|
|
3644
|
+
});
|
|
3645
|
+
results.push({
|
|
3646
|
+
status: "rejected",
|
|
3647
|
+
...failure
|
|
3461
3648
|
});
|
|
3462
3649
|
outputs.push(void 0);
|
|
3463
3650
|
} else {
|
|
3464
|
-
|
|
3651
|
+
const value = c?.value;
|
|
3652
|
+
results.push({
|
|
3653
|
+
status: "fulfilled",
|
|
3654
|
+
value
|
|
3655
|
+
});
|
|
3656
|
+
outputs.push(value);
|
|
3465
3657
|
}
|
|
3466
3658
|
}
|
|
3659
|
+
if (settle) return results;
|
|
3467
3660
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
3468
3661
|
return outputs;
|
|
3469
|
-
}, "
|
|
3470
|
-
const
|
|
3662
|
+
}, "fan");
|
|
3663
|
+
const all = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, false), "all");
|
|
3664
|
+
const allSettled = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, true), "allSettled");
|
|
3665
|
+
const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts, settle) => {
|
|
3471
3666
|
if (opts?.queue !== void 0) {
|
|
3472
3667
|
throw new Error("ctx.all: a flow-control queue is not supported on a step fan");
|
|
3473
3668
|
}
|
|
@@ -3478,7 +3673,8 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3478
3673
|
};
|
|
3479
3674
|
const positions = inputs.map(() => pos.next());
|
|
3480
3675
|
return host.callRemoteAll(runId, positions, def, inputs, {
|
|
3481
|
-
mode: opts?.mode ?? "waitAll",
|
|
3676
|
+
mode: settle ? "waitAll" : opts?.mode ?? "waitAll",
|
|
3677
|
+
settle,
|
|
3482
3678
|
concurrency: opts?.concurrency,
|
|
3483
3679
|
transport: opts?.transport,
|
|
3484
3680
|
admission: {
|
|
@@ -3796,6 +3992,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3796
3992
|
task: guard("task", task),
|
|
3797
3993
|
child: guard("child", child),
|
|
3798
3994
|
all: guard("all", all),
|
|
3995
|
+
allSettled: guard("allSettled", allSettled),
|
|
3799
3996
|
startChild: guard("startChild", startChild),
|
|
3800
3997
|
breakpoint: guard("breakpoint", breakpoint),
|
|
3801
3998
|
webhook: guardedWebhook,
|
|
@@ -3821,9 +4018,11 @@ function mergeTags(staticTags, runTags) {
|
|
|
3821
4018
|
__name(mergeTags, "mergeTags");
|
|
3822
4019
|
var BREAKPOINT = "breakpoint";
|
|
3823
4020
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
4021
|
+
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
3824
4022
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
3825
4023
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
3826
4024
|
var STEP_FAN_CHUNK = 200;
|
|
4025
|
+
var RESULT_BATCH_MAX = 256;
|
|
3827
4026
|
function shuffle(items) {
|
|
3828
4027
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
3829
4028
|
const j = Math.floor(Math.random() * (i + 1));
|
|
@@ -3931,6 +4130,8 @@ var WorkflowEngine = class {
|
|
|
3931
4130
|
updateValidators = /* @__PURE__ */ new Map();
|
|
3932
4131
|
/** Runs being cancelled WITH saga compensation — see `cancel({ compensate: true })`. */
|
|
3933
4132
|
cancelRequested = /* @__PURE__ */ new Set();
|
|
4133
|
+
/** {@link announcedRoutes} reads, by group + workflow + version. */
|
|
4134
|
+
announcedRoutesCache = /* @__PURE__ */ new Map();
|
|
3934
4135
|
/**
|
|
3935
4136
|
* Seqs of the dispatched remote steps a turn suspended on, by run, while that turn executes on
|
|
3936
4137
|
* THIS instance — so the turn can re-check them once its lease is released (see
|
|
@@ -3952,6 +4153,8 @@ var WorkflowEngine = class {
|
|
|
3952
4153
|
stepFansFiredHere = /* @__PURE__ */ new Map();
|
|
3953
4154
|
/** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
|
|
3954
4155
|
resultChains = /* @__PURE__ */ new Map();
|
|
4156
|
+
/** Each run's results waiting to be settled on this instance — see {@link completeRemoteResult}. */
|
|
4157
|
+
resultQueues = /* @__PURE__ */ new Map();
|
|
3955
4158
|
/** Flow-control admission backend for remote steps (see {@link registerQueue}). */
|
|
3956
4159
|
admission;
|
|
3957
4160
|
/** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
|
|
@@ -4158,6 +4361,7 @@ var WorkflowEngine = class {
|
|
|
4158
4361
|
name,
|
|
4159
4362
|
version,
|
|
4160
4363
|
fn,
|
|
4364
|
+
onError: this.checkedRoutes(name, opts?.onError),
|
|
4161
4365
|
tags: opts?.tags,
|
|
4162
4366
|
singleton: opts?.singleton,
|
|
4163
4367
|
concurrency: opts?.concurrency,
|
|
@@ -4227,6 +4431,7 @@ var WorkflowEngine = class {
|
|
|
4227
4431
|
executionTimeoutMs: opts.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
|
|
4228
4432
|
validateInput: opts.validateInput,
|
|
4229
4433
|
requires: opts.requires,
|
|
4434
|
+
onError: this.checkedRoutes(name, opts.onError),
|
|
4230
4435
|
remote: {
|
|
4231
4436
|
group: opts.group,
|
|
4232
4437
|
executor: opts.executor
|
|
@@ -5004,6 +5209,24 @@ var WorkflowEngine = class {
|
|
|
5004
5209
|
if (resurrecting) {
|
|
5005
5210
|
const isFailureCompletion = /* @__PURE__ */ __name((v) => typeof v === "object" && v !== null && v.ok === false, "isFailureCompletion");
|
|
5006
5211
|
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
5212
|
+
const handlerId = handoffHandlerId(runId, cp);
|
|
5213
|
+
if (handlerId !== void 0) {
|
|
5214
|
+
await this.store.takeSignalWaiter(`child:${handlerId}`);
|
|
5215
|
+
await this.store.saveCheckpoint({
|
|
5216
|
+
...cp,
|
|
5217
|
+
name: retiredHandoffName(handlerId),
|
|
5218
|
+
status: "completed"
|
|
5219
|
+
});
|
|
5220
|
+
const tags = run.tags?.filter((t) => !t.startsWith("onError:"));
|
|
5221
|
+
if (tags && tags.length !== run.tags?.length) {
|
|
5222
|
+
await this.store.updateRun(runId, {
|
|
5223
|
+
tags,
|
|
5224
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
5225
|
+
});
|
|
5226
|
+
run.tags = tags;
|
|
5227
|
+
}
|
|
5228
|
+
continue;
|
|
5229
|
+
}
|
|
5007
5230
|
if (cp.status === "failed") {
|
|
5008
5231
|
await this.store.saveCheckpoint({
|
|
5009
5232
|
...cp,
|
|
@@ -5148,13 +5371,13 @@ var WorkflowEngine = class {
|
|
|
5148
5371
|
* `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
|
|
5149
5372
|
*/
|
|
5150
5373
|
waitForRun(runId, opts) {
|
|
5151
|
-
const
|
|
5374
|
+
const TERMINAL2 = /* @__PURE__ */ new Set([
|
|
5152
5375
|
"completed",
|
|
5153
5376
|
"failed",
|
|
5154
5377
|
"cancelled",
|
|
5155
5378
|
"dead"
|
|
5156
5379
|
]);
|
|
5157
|
-
const isSettled = /* @__PURE__ */ __name((run) => opts?.until === "terminal" ?
|
|
5380
|
+
const isSettled = /* @__PURE__ */ __name((run) => opts?.until === "terminal" ? TERMINAL2.has(run.status) : run.status !== "pending" && run.status !== "running" && run.status !== "cancelling" && !(run.status === "suspended" && run.awaitingDecisionTaskId !== void 0), "isSettled");
|
|
5158
5381
|
const toResult = /* @__PURE__ */ __name((run) => ({
|
|
5159
5382
|
runId,
|
|
5160
5383
|
status: run.status,
|
|
@@ -5409,10 +5632,7 @@ var WorkflowEngine = class {
|
|
|
5409
5632
|
async redriveSettledStepFans(runId, fans) {
|
|
5410
5633
|
for (const fan of fans) {
|
|
5411
5634
|
if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
|
|
5412
|
-
|
|
5413
|
-
"pending"
|
|
5414
|
-
]) : (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= fan.from && c.seq <= fan.to && c.status === "pending").length;
|
|
5415
|
-
if (pending > 0) continue;
|
|
5635
|
+
if (await this.stepFanPending(runId, fan.from, fan.to) > 0) continue;
|
|
5416
5636
|
const nowMs = this.clock();
|
|
5417
5637
|
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5418
5638
|
return;
|
|
@@ -6122,6 +6342,209 @@ var WorkflowEngine = class {
|
|
|
6122
6342
|
* us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
|
|
6123
6343
|
*/
|
|
6124
6344
|
/**
|
|
6345
|
+
* The registered onError routes whose handler workflow nothing can run: not registered on this
|
|
6346
|
+
* engine, and no live worker group serves it. Meant for a boot-time warning: a typo in a handler's
|
|
6347
|
+
* name would otherwise surface only when the first run fails.
|
|
6348
|
+
*/
|
|
6349
|
+
async unservedOnErrorRoutes() {
|
|
6350
|
+
const unserved = [];
|
|
6351
|
+
let groups;
|
|
6352
|
+
for (const registered of this.workflows.values()) {
|
|
6353
|
+
for (const route of registered.onError ?? []) {
|
|
6354
|
+
if (this.latest.has(route.workflow)) continue;
|
|
6355
|
+
groups ??= await this.pool.listWorkerGroups().catch(() => []);
|
|
6356
|
+
const token = sanitizeQueueToken(route.workflow);
|
|
6357
|
+
if (groups.some((g) => g === token || g.startsWith(`${token}@`))) continue;
|
|
6358
|
+
unserved.push({
|
|
6359
|
+
workflow: registered.name,
|
|
6360
|
+
route
|
|
6361
|
+
});
|
|
6362
|
+
}
|
|
6363
|
+
}
|
|
6364
|
+
return unserved;
|
|
6365
|
+
}
|
|
6366
|
+
/** Validate a workflow's {@link OnErrorRoute}s at registration. */
|
|
6367
|
+
checkedRoutes(name, routes) {
|
|
6368
|
+
if (!routes?.length) return void 0;
|
|
6369
|
+
assertRoutes(name, routes);
|
|
6370
|
+
return routes;
|
|
6371
|
+
}
|
|
6372
|
+
/**
|
|
6373
|
+
* Route a failed run to its {@link OnErrorRoute} handler, read from the workflow's local registration
|
|
6374
|
+
* or, for one served by convention, from what its live workers announce. Starts the handler run and
|
|
6375
|
+
* waits for it the way a parent waits for a `ctx.child`: a `child:<handlerId>` waiter plus a
|
|
6376
|
+
* `running` placeholder checkpoint carrying the handler's name and input, so the handler's terminal
|
|
6377
|
+
* state wakes the run (buffered and reconciled like any child). From then on every execution of the
|
|
6378
|
+
* run only follows the handler ({@link followHandoff}); the body never runs again. Undefined when no
|
|
6379
|
+
* route matches: the run fails as usual.
|
|
6380
|
+
*
|
|
6381
|
+
* The placeholder sits past every seq the body used and is left out of {@link remoteHistory}.
|
|
6382
|
+
*/
|
|
6383
|
+
async routeFailure(run, error2) {
|
|
6384
|
+
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
|
|
6385
|
+
const routes = registered ? registered.onError : await this.announcedRoutes(run, error2);
|
|
6386
|
+
const route = routeFor(routes, error2);
|
|
6387
|
+
if (!route) return void 0;
|
|
6388
|
+
const checkpoints = await this.handoffCheckpoints(run.id);
|
|
6389
|
+
const live = liveHandoff(run.id, checkpoints);
|
|
6390
|
+
if (live) return this.followHandoff(run, live);
|
|
6391
|
+
const attempt = checkpoints.length;
|
|
6392
|
+
const seq = HANDOFF_SEQ_BASE + attempt;
|
|
6393
|
+
const handlerId = onErrorRunId(run.id, attempt);
|
|
6394
|
+
const handler = {
|
|
6395
|
+
workflow: route.workflow,
|
|
6396
|
+
input: {
|
|
6397
|
+
runId: run.id,
|
|
6398
|
+
workflow: run.workflow,
|
|
6399
|
+
input: run.input,
|
|
6400
|
+
error: error2
|
|
6401
|
+
}
|
|
6402
|
+
};
|
|
6403
|
+
await this.store.putSignalWaiter({
|
|
6404
|
+
token: `child:${handlerId}`,
|
|
6405
|
+
runId: run.id,
|
|
6406
|
+
seq
|
|
6407
|
+
});
|
|
6408
|
+
await this.store.saveCheckpoint({
|
|
6409
|
+
...instantCheckpoint({
|
|
6410
|
+
runId: run.id,
|
|
6411
|
+
seq,
|
|
6412
|
+
name: handoffName(handlerId),
|
|
6413
|
+
kind: "signal",
|
|
6414
|
+
status: "running"
|
|
6415
|
+
}),
|
|
6416
|
+
input: handler
|
|
6417
|
+
});
|
|
6418
|
+
const tag = `onError:${route.code ?? "*"}`;
|
|
6419
|
+
if (!run.tags?.includes(tag)) {
|
|
6420
|
+
const tags = [
|
|
6421
|
+
...run.tags ?? [],
|
|
6422
|
+
tag
|
|
6423
|
+
];
|
|
6424
|
+
await this.store.updateRun(run.id, {
|
|
6425
|
+
tags,
|
|
6426
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
6427
|
+
});
|
|
6428
|
+
run.tags = tags;
|
|
6429
|
+
}
|
|
6430
|
+
this.emit({
|
|
6431
|
+
type: "run.routed",
|
|
6432
|
+
runId: run.id,
|
|
6433
|
+
workflow: run.workflow,
|
|
6434
|
+
namespace: run.namespace,
|
|
6435
|
+
name: route.workflow,
|
|
6436
|
+
error: error2
|
|
6437
|
+
});
|
|
6438
|
+
this.startChildDeferred(handler.workflow, handler.input, handlerId, {
|
|
6439
|
+
namespace: run.namespace
|
|
6440
|
+
});
|
|
6441
|
+
return {
|
|
6442
|
+
kind: "suspended"
|
|
6443
|
+
};
|
|
6444
|
+
}
|
|
6445
|
+
/** A run's handoff checkpoints, live and retired: the only ones at {@link HANDOFF_SEQ_BASE} and up. */
|
|
6446
|
+
async handoffCheckpoints(runId) {
|
|
6447
|
+
const handoffs = await this.store.listCheckpointsByNamePrefix?.(runId, [
|
|
6448
|
+
`signal:child:${runId}.onError.`,
|
|
6449
|
+
`onError:retried:${runId}.onError.`
|
|
6450
|
+
]);
|
|
6451
|
+
return (handoffs ?? await this.store.listCheckpoints(runId)).filter((cp) => cp.seq >= HANDOFF_SEQ_BASE);
|
|
6452
|
+
}
|
|
6453
|
+
/**
|
|
6454
|
+
* Where a run that handed its failure off stands: the handler's outcome once it settled (from the
|
|
6455
|
+
* wake's checkpoint, or off the handler run when the wake never arrived), else still waiting. Starts
|
|
6456
|
+
* the handler if the turn that handed off crashed before its deferred start ran.
|
|
6457
|
+
*/
|
|
6458
|
+
async followHandoff(run, handoff) {
|
|
6459
|
+
if (handoff.status === "completed") {
|
|
6460
|
+
const outcome = handoff.output;
|
|
6461
|
+
return outcome.ok ? {
|
|
6462
|
+
kind: "completed",
|
|
6463
|
+
output: outcome.value
|
|
6464
|
+
} : {
|
|
6465
|
+
kind: "failed",
|
|
6466
|
+
error: completionError(outcome)
|
|
6467
|
+
};
|
|
6468
|
+
}
|
|
6469
|
+
const handlerId = handoffHandlerId(run.id, handoff);
|
|
6470
|
+
const handlerRun = await this.store.getRun(handlerId);
|
|
6471
|
+
if (handlerRun?.status === "completed") return {
|
|
6472
|
+
kind: "completed",
|
|
6473
|
+
output: handlerRun.output
|
|
6474
|
+
};
|
|
6475
|
+
if (handlerRun && TERMINAL_RUN_STATUSES.includes(handlerRun.status)) {
|
|
6476
|
+
return {
|
|
6477
|
+
kind: "failed",
|
|
6478
|
+
error: handlerRun.error ?? {
|
|
6479
|
+
message: `onError handler ${handlerId} ${handlerRun.status}`
|
|
6480
|
+
}
|
|
6481
|
+
};
|
|
6482
|
+
}
|
|
6483
|
+
if (!handlerRun) {
|
|
6484
|
+
const handler = handoff.input;
|
|
6485
|
+
this.startChildDeferred(handler.workflow, handler.input, handlerId, {
|
|
6486
|
+
namespace: run.namespace
|
|
6487
|
+
});
|
|
6488
|
+
}
|
|
6489
|
+
return {
|
|
6490
|
+
kind: "suspended"
|
|
6491
|
+
};
|
|
6492
|
+
}
|
|
6493
|
+
/**
|
|
6494
|
+
* The onError routes the live workers announce for a workflow the engine serves by convention: the
|
|
6495
|
+
* one set they agree on, or none when they state none or cannot be read. Workers that announce
|
|
6496
|
+
* different sets (a rollout in progress) route nothing, and the run's failure is reported as a
|
|
6497
|
+
* `run.unrouted` event. Announcers of another version of the workflow are ignored. The read is
|
|
6498
|
+
* cached for {@link ANNOUNCED_ROUTES_TTL_MS}, so a burst of failures costs one descriptor read.
|
|
6499
|
+
*/
|
|
6500
|
+
async announcedRoutes(run, error2) {
|
|
6501
|
+
const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
|
|
6502
|
+
const key = `${group}
|
|
6503
|
+
${run.workflow}
|
|
6504
|
+
${run.workflowVersion}`;
|
|
6505
|
+
const now = this.clock();
|
|
6506
|
+
let cached = this.announcedRoutesCache.get(key);
|
|
6507
|
+
if (!cached || now - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
|
|
6508
|
+
cached = {
|
|
6509
|
+
at: now,
|
|
6510
|
+
sets: await this.readAnnouncedRoutes(run, group)
|
|
6511
|
+
};
|
|
6512
|
+
this.announcedRoutesCache.set(key, cached);
|
|
6513
|
+
}
|
|
6514
|
+
if (cached.sets.length > 1) {
|
|
6515
|
+
this.emit({
|
|
6516
|
+
type: "run.unrouted",
|
|
6517
|
+
runId: run.id,
|
|
6518
|
+
workflow: run.workflow,
|
|
6519
|
+
namespace: run.namespace,
|
|
6520
|
+
error: error2,
|
|
6521
|
+
reason: `live workers announce ${cached.sets.length} different onError route sets`
|
|
6522
|
+
});
|
|
6523
|
+
return void 0;
|
|
6524
|
+
}
|
|
6525
|
+
return cached.sets[0];
|
|
6526
|
+
}
|
|
6527
|
+
/** Every distinct onError route set the live workers of `group` announce for the run's workflow. */
|
|
6528
|
+
async readAnnouncedRoutes(run, group) {
|
|
6529
|
+
const transport = await this.pool.transportWithLiveGroup(group);
|
|
6530
|
+
if (!transport?.readWorkerDescriptors) return [];
|
|
6531
|
+
let descriptors;
|
|
6532
|
+
try {
|
|
6533
|
+
descriptors = await transport.readWorkerDescriptors(group);
|
|
6534
|
+
} catch {
|
|
6535
|
+
return [];
|
|
6536
|
+
}
|
|
6537
|
+
const sets = /* @__PURE__ */ new Map();
|
|
6538
|
+
for (const announced of aggregateAnnouncements(descriptors)) {
|
|
6539
|
+
if (announced.name !== run.workflow) continue;
|
|
6540
|
+
if (announced.version !== void 0 && announced.version !== run.workflowVersion) continue;
|
|
6541
|
+
for (const routes of announced.onError) sets.set(JSON.stringify(routes), routes);
|
|
6542
|
+
}
|
|
6543
|
+
return [
|
|
6544
|
+
...sets.values()
|
|
6545
|
+
];
|
|
6546
|
+
}
|
|
6547
|
+
/**
|
|
6125
6548
|
* Apply a terminal/suspended transition once: persist the new run status, emit the matching
|
|
6126
6549
|
* lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
|
|
6127
6550
|
* executors funnel their completed/failed/suspended outcomes through here so a status update can
|
|
@@ -6155,9 +6578,12 @@ var WorkflowEngine = class {
|
|
|
6155
6578
|
};
|
|
6156
6579
|
}
|
|
6157
6580
|
if (outcome.kind === "failed") {
|
|
6581
|
+
const handed = await this.routeFailure(run, outcome.error);
|
|
6582
|
+
if (handed && handed.kind !== "failed") return this.settleRun(run, handed);
|
|
6583
|
+
const error2 = handed?.error ?? outcome.error;
|
|
6158
6584
|
await this.store.updateRun(run.id, {
|
|
6159
6585
|
status: "failed",
|
|
6160
|
-
error:
|
|
6586
|
+
error: error2,
|
|
6161
6587
|
updatedAt
|
|
6162
6588
|
});
|
|
6163
6589
|
this.emit({
|
|
@@ -6165,16 +6591,13 @@ var WorkflowEngine = class {
|
|
|
6165
6591
|
runId: run.id,
|
|
6166
6592
|
workflow: run.workflow,
|
|
6167
6593
|
namespace: run.namespace,
|
|
6168
|
-
error:
|
|
6169
|
-
});
|
|
6170
|
-
void this.notifyParent(run.id, {
|
|
6171
|
-
ok: false,
|
|
6172
|
-
error: outcome.error.message
|
|
6594
|
+
error: error2
|
|
6173
6595
|
});
|
|
6596
|
+
void this.notifyParent(run.id, failedCompletion(error2));
|
|
6174
6597
|
return {
|
|
6175
6598
|
runId: run.id,
|
|
6176
6599
|
status: "failed",
|
|
6177
|
-
error:
|
|
6600
|
+
error: error2
|
|
6178
6601
|
};
|
|
6179
6602
|
}
|
|
6180
6603
|
if (outcome.kind === "blocked") return this.parkBlocked(run, outcome.blocked);
|
|
@@ -6263,7 +6686,14 @@ var WorkflowEngine = class {
|
|
|
6263
6686
|
}
|
|
6264
6687
|
const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
|
|
6265
6688
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
6266
|
-
const
|
|
6689
|
+
const checkpoints = await this.store.listCheckpoints(run.id);
|
|
6690
|
+
const handoff = liveHandoff(run.id, checkpoints);
|
|
6691
|
+
if (handoff) {
|
|
6692
|
+
const result = await this.settleRun(run, await this.followHandoff(run, handoff));
|
|
6693
|
+
await this.store.releaseRunLock(run.id);
|
|
6694
|
+
return result;
|
|
6695
|
+
}
|
|
6696
|
+
const history = await this.remoteHistory(run.id, checkpoints);
|
|
6267
6697
|
if (remote.executor.dispatch) {
|
|
6268
6698
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
6269
6699
|
const silenceMs = this.remoteAdvanceSilenceMs;
|
|
@@ -6587,8 +7017,8 @@ var WorkflowEngine = class {
|
|
|
6587
7017
|
}
|
|
6588
7018
|
}
|
|
6589
7019
|
/** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
|
|
6590
|
-
async remoteHistory(runId) {
|
|
6591
|
-
const checkpoints = await this.store.listCheckpoints(runId);
|
|
7020
|
+
async remoteHistory(runId, listed) {
|
|
7021
|
+
const checkpoints = listed ?? await this.store.listCheckpoints(runId);
|
|
6592
7022
|
const kindOf = {
|
|
6593
7023
|
remote: "call",
|
|
6594
7024
|
local: "step",
|
|
@@ -6598,15 +7028,14 @@ var WorkflowEngine = class {
|
|
|
6598
7028
|
const events = [];
|
|
6599
7029
|
for (const cp of checkpoints) {
|
|
6600
7030
|
if (cp.status === "completed" || cp.status === "failed") {
|
|
7031
|
+
if (cp.seq >= HANDOFF_SEQ_BASE) continue;
|
|
6601
7032
|
if (cp.kind === "signal" && cp.name.startsWith("signal:child:")) {
|
|
6602
7033
|
const completion = cp.output;
|
|
6603
7034
|
events.push({
|
|
6604
7035
|
seq: cp.seq,
|
|
6605
7036
|
kind: "child",
|
|
6606
7037
|
output: completion?.ok ? completion.value : void 0,
|
|
6607
|
-
error: completion && completion.ok === false ?
|
|
6608
|
-
message: completion.error
|
|
6609
|
-
} : void 0
|
|
7038
|
+
error: completion && completion.ok === false ? completionError(completion) : void 0
|
|
6610
7039
|
});
|
|
6611
7040
|
continue;
|
|
6612
7041
|
}
|
|
@@ -6887,6 +7316,10 @@ var WorkflowEngine = class {
|
|
|
6887
7316
|
this.awaitedStepFans.set(run.id, awaitedFans);
|
|
6888
7317
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
|
|
6889
7318
|
try {
|
|
7319
|
+
const handoff = liveHandoff(run.id, snapshot);
|
|
7320
|
+
if (handoff && !this.cancelRequested.has(run.id)) {
|
|
7321
|
+
return await this.settleRun(run, await this.followHandoff(run, handoff));
|
|
7322
|
+
}
|
|
6890
7323
|
const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
|
|
6891
7324
|
if (this.cancelRequested.has(run.id)) {
|
|
6892
7325
|
return await this.compensateAndCancel(run, compensations, replay);
|
|
@@ -6949,7 +7382,8 @@ var WorkflowEngine = class {
|
|
|
6949
7382
|
}
|
|
6950
7383
|
const error2 = {
|
|
6951
7384
|
message: err instanceof Error ? err.message : String(err),
|
|
6952
|
-
stack: err instanceof Error ? err.stack : void 0
|
|
7385
|
+
stack: err instanceof Error ? err.stack : void 0,
|
|
7386
|
+
...codeAndDetails(err)
|
|
6953
7387
|
};
|
|
6954
7388
|
const pending = await this.unwindCompensations(run, compensations, replay);
|
|
6955
7389
|
if (pending) return await this.settleRun(run, {
|
|
@@ -7585,7 +8019,8 @@ var WorkflowEngine = class {
|
|
|
7585
8019
|
const failure = {
|
|
7586
8020
|
index: i,
|
|
7587
8021
|
id: stepId(runId, seq),
|
|
7588
|
-
error: err.message
|
|
8022
|
+
error: err.message,
|
|
8023
|
+
...codeAndDetails(err)
|
|
7589
8024
|
};
|
|
7590
8025
|
if (opts.mode === "failFast") throw new GatherError([
|
|
7591
8026
|
failure
|
|
@@ -7607,6 +8042,24 @@ var WorkflowEngine = class {
|
|
|
7607
8042
|
});
|
|
7608
8043
|
throw new WorkflowSuspended(wakeAt);
|
|
7609
8044
|
}
|
|
8045
|
+
if (opts.settle) {
|
|
8046
|
+
const failed = new Map(failures.map((f) => [
|
|
8047
|
+
f.index,
|
|
8048
|
+
f
|
|
8049
|
+
]));
|
|
8050
|
+
return Array.from(outputs, (value, i) => {
|
|
8051
|
+
const f = failed.get(i);
|
|
8052
|
+
if (!f) return {
|
|
8053
|
+
status: "fulfilled",
|
|
8054
|
+
value
|
|
8055
|
+
};
|
|
8056
|
+
const { index: _index, ...rejected } = f;
|
|
8057
|
+
return {
|
|
8058
|
+
status: "rejected",
|
|
8059
|
+
...rejected
|
|
8060
|
+
};
|
|
8061
|
+
});
|
|
8062
|
+
}
|
|
7610
8063
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
7611
8064
|
return outputs;
|
|
7612
8065
|
}
|
|
@@ -7678,6 +8131,25 @@ var WorkflowEngine = class {
|
|
|
7678
8131
|
await this.store.saveCheckpoint(checkpoint);
|
|
7679
8132
|
return true;
|
|
7680
8133
|
}
|
|
8134
|
+
/**
|
|
8135
|
+
* {@link savePendingCheckpoint} for checkpoints at distinct seqs of one run: one
|
|
8136
|
+
* {@link StateStore.saveCheckpointsIfPending} call when the store has it, else one write each.
|
|
8137
|
+
* Never rejects as a whole: a write that failed is reported as that entry's error, and the batch
|
|
8138
|
+
* call rejecting means it wrote nothing, so the entries are then written one by one.
|
|
8139
|
+
*/
|
|
8140
|
+
async savePendingCheckpoints(checkpoints) {
|
|
8141
|
+
if (checkpoints.length > 1 && this.store.saveCheckpointsIfPending) {
|
|
8142
|
+
try {
|
|
8143
|
+
const wrote = await this.store.saveCheckpointsIfPending(checkpoints);
|
|
8144
|
+
return checkpoints.map((_, i) => wrote[i] === true);
|
|
8145
|
+
} catch {
|
|
8146
|
+
}
|
|
8147
|
+
}
|
|
8148
|
+
const settled = await Promise.allSettled(checkpoints.map((cp) => this.savePendingCheckpoint(cp)));
|
|
8149
|
+
return settled.map((s) => s.status === "fulfilled" ? s.value : {
|
|
8150
|
+
error: s.reason
|
|
8151
|
+
});
|
|
8152
|
+
}
|
|
7681
8153
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
7682
8154
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
7683
8155
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -7716,57 +8188,161 @@ var WorkflowEngine = class {
|
|
|
7716
8188
|
* result (the dispatching one may be gone), so the run is crash/scale-safe. Resolves whether the
|
|
7717
8189
|
* run should now be resumed: not for a duplicate or late delivery (the checkpoint isn't `pending`),
|
|
7718
8190
|
* a run already finished, or a step fan whose barrier is still closed.
|
|
8191
|
+
*
|
|
8192
|
+
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
8193
|
+
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
8194
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked once
|
|
8195
|
+
* per batch, not once per result. A result resolves only once its own write has committed, so the
|
|
8196
|
+
* transport acknowledges nothing that is not durable yet.
|
|
8197
|
+
*/
|
|
8198
|
+
completeRemoteResult(result) {
|
|
8199
|
+
return new Promise((resolve, reject) => {
|
|
8200
|
+
const entry = {
|
|
8201
|
+
result,
|
|
8202
|
+
resolve,
|
|
8203
|
+
reject
|
|
8204
|
+
};
|
|
8205
|
+
const queued = this.resultQueues.get(result.runId);
|
|
8206
|
+
if (queued) {
|
|
8207
|
+
queued.push(entry);
|
|
8208
|
+
return;
|
|
8209
|
+
}
|
|
8210
|
+
const queue = [
|
|
8211
|
+
entry
|
|
8212
|
+
];
|
|
8213
|
+
this.resultQueues.set(result.runId, queue);
|
|
8214
|
+
void this.drainResultQueue(result.runId, queue);
|
|
8215
|
+
});
|
|
8216
|
+
}
|
|
8217
|
+
/** Settle `queue` (a run's queued results) batch by batch until it is empty, then retire it. */
|
|
8218
|
+
async drainResultQueue(runId, queue) {
|
|
8219
|
+
await Promise.resolve();
|
|
8220
|
+
while (queue.length > 0) {
|
|
8221
|
+
const batch = queue.splice(0, RESULT_BATCH_MAX);
|
|
8222
|
+
let outcomes;
|
|
8223
|
+
try {
|
|
8224
|
+
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
|
|
8225
|
+
} catch (err) {
|
|
8226
|
+
outcomes = batch.map(() => ({
|
|
8227
|
+
error: err
|
|
8228
|
+
}));
|
|
8229
|
+
}
|
|
8230
|
+
batch.forEach((q, i) => {
|
|
8231
|
+
const outcome = outcomes[i] ?? false;
|
|
8232
|
+
if (typeof outcome === "boolean") q.resolve(outcome);
|
|
8233
|
+
else q.reject(outcome.error);
|
|
8234
|
+
});
|
|
8235
|
+
}
|
|
8236
|
+
this.resultQueues.delete(runId);
|
|
8237
|
+
}
|
|
8238
|
+
/**
|
|
8239
|
+
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
8240
|
+
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
8241
|
+
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
7719
8242
|
*/
|
|
7720
|
-
async
|
|
7721
|
-
const
|
|
7722
|
-
|
|
7723
|
-
|
|
7724
|
-
|
|
7725
|
-
|
|
8243
|
+
async completeRemoteResults(runId, results) {
|
|
8244
|
+
const outcomes = results.map(() => false);
|
|
8245
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8246
|
+
const live = results.flatMap((r, i) => {
|
|
8247
|
+
if (seen.has(r.seq)) return [];
|
|
8248
|
+
seen.add(r.seq);
|
|
8249
|
+
return [
|
|
8250
|
+
i
|
|
8251
|
+
];
|
|
8252
|
+
});
|
|
8253
|
+
const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
|
|
8254
|
+
const pending = [];
|
|
8255
|
+
reads.forEach((read, k) => {
|
|
8256
|
+
const index = live[k];
|
|
8257
|
+
if (read.status === "rejected") outcomes[index] = {
|
|
8258
|
+
error: read.reason
|
|
8259
|
+
};
|
|
8260
|
+
else if (read.value?.status === "pending") pending.push({
|
|
8261
|
+
index,
|
|
8262
|
+
cp: read.value
|
|
8263
|
+
});
|
|
8264
|
+
});
|
|
8265
|
+
if (pending.length === 0) return outcomes;
|
|
8266
|
+
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
8267
|
+
const run = await this.store.getRun(runId);
|
|
8268
|
+
if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
|
|
7726
8269
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
7727
|
-
const
|
|
7728
|
-
|
|
7729
|
-
|
|
7730
|
-
|
|
7731
|
-
|
|
7732
|
-
|
|
7733
|
-
|
|
7734
|
-
|
|
7735
|
-
|
|
8270
|
+
const writes = pending.map(({ index, cp }) => {
|
|
8271
|
+
const result = results[index];
|
|
8272
|
+
return {
|
|
8273
|
+
...cp,
|
|
8274
|
+
status: result.status,
|
|
8275
|
+
output: result.status === "completed" ? result.output : cp.output,
|
|
8276
|
+
error: result.error,
|
|
8277
|
+
events: result.events ?? cp.events,
|
|
8278
|
+
startedAt: result.startedAt ? new Date(result.startedAt) : cp.startedAt,
|
|
8279
|
+
finishedAt
|
|
8280
|
+
};
|
|
7736
8281
|
});
|
|
7737
|
-
|
|
7738
|
-
|
|
7739
|
-
|
|
7740
|
-
|
|
7741
|
-
|
|
7742
|
-
|
|
7743
|
-
|
|
7744
|
-
|
|
7745
|
-
|
|
7746
|
-
|
|
7747
|
-
queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
7748
|
-
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
8282
|
+
const claims = await this.savePendingCheckpoints(writes);
|
|
8283
|
+
const settled = [];
|
|
8284
|
+
claims.forEach((claim, k) => {
|
|
8285
|
+
const { index, cp } = pending[k];
|
|
8286
|
+
if (claim === true) settled.push({
|
|
8287
|
+
index,
|
|
8288
|
+
cp,
|
|
8289
|
+
result: results[index]
|
|
8290
|
+
});
|
|
8291
|
+
else if (claim !== false) outcomes[index] = claim;
|
|
7749
8292
|
});
|
|
7750
|
-
|
|
8293
|
+
for (const { cp, result } of settled) {
|
|
8294
|
+
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
8295
|
+
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
8296
|
+
this.emit({
|
|
8297
|
+
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
8298
|
+
runId,
|
|
8299
|
+
seq: result.seq,
|
|
8300
|
+
name: cp.name,
|
|
8301
|
+
kind: cp.kind,
|
|
8302
|
+
output: result.output,
|
|
8303
|
+
error: result.error,
|
|
8304
|
+
queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
8305
|
+
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
8306
|
+
});
|
|
8307
|
+
}
|
|
8308
|
+
const barriers = /* @__PURE__ */ new Map();
|
|
8309
|
+
let carrier;
|
|
8310
|
+
for (const { index, cp, result } of settled) {
|
|
8311
|
+
if (!await this.stepFanStillRunning(cp, result, barriers)) carrier = index;
|
|
8312
|
+
}
|
|
8313
|
+
if (carrier !== void 0) outcomes[carrier] = true;
|
|
8314
|
+
return outcomes;
|
|
7751
8315
|
}
|
|
7752
8316
|
/**
|
|
7753
8317
|
* The join barrier of a step fan: a result for one of its steps resumes the parent only once no
|
|
7754
8318
|
* step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
|
|
7755
8319
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
7756
|
-
* which makes a fan of N steps cost O(N²) checkpoint reads.
|
|
8320
|
+
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
8321
|
+
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
8322
|
+
* re-suspends, a lost one strands the run.
|
|
7757
8323
|
*/
|
|
7758
|
-
async stepFanStillRunning(cp, result) {
|
|
8324
|
+
async stepFanStillRunning(cp, result, barriers) {
|
|
7759
8325
|
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
7760
8326
|
if (!fan) return false;
|
|
7761
8327
|
if (result.status === "failed") return false;
|
|
7762
|
-
|
|
7763
|
-
|
|
7764
|
-
|
|
7765
|
-
|
|
7766
|
-
|
|
8328
|
+
let closed = barriers.get(fan.firstSeq);
|
|
8329
|
+
if (!closed) {
|
|
8330
|
+
closed = this.stepFanPending(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1).then((pending) => pending > 0, () => false);
|
|
8331
|
+
barriers.set(fan.firstSeq, closed);
|
|
8332
|
+
}
|
|
8333
|
+
if (await closed) return true;
|
|
7767
8334
|
this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
|
|
7768
8335
|
return false;
|
|
7769
8336
|
}
|
|
8337
|
+
/** How many steps of a step fan's seq range are still `pending`. */
|
|
8338
|
+
async stepFanPending(runId, from, to) {
|
|
8339
|
+
if (this.store.countCheckpoints) {
|
|
8340
|
+
return this.store.countCheckpoints(runId, from, to, [
|
|
8341
|
+
"pending"
|
|
8342
|
+
]);
|
|
8343
|
+
}
|
|
8344
|
+
return (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= from && c.seq <= to && c.status === "pending").length;
|
|
8345
|
+
}
|
|
7770
8346
|
/**
|
|
7771
8347
|
* Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
|
|
7772
8348
|
* it over the broker (the dispatching one may be gone, or a point-to-point broker handed it to a
|
|
@@ -8213,14 +8789,339 @@ function summarizeTimeline(timeline, over) {
|
|
|
8213
8789
|
__name(summarizeTimeline, "summarizeTimeline");
|
|
8214
8790
|
function pageFan(timeline, group, query) {
|
|
8215
8791
|
const matching = timeline.filter((cp) => cp.parallelGroup === group && (!query.failedOnly || isFailedMember(cp)));
|
|
8216
|
-
const
|
|
8217
|
-
const limit = query.limit !== void 0 && Number.isFinite(query.limit) ? Math.min(FAN_PAGE_MAX, Math.max(1, Math.floor(query.limit))) : 100;
|
|
8792
|
+
const { offset, limit } = fanPageBounds(query);
|
|
8218
8793
|
return {
|
|
8219
8794
|
items: matching.slice(offset, offset + limit),
|
|
8220
8795
|
total: matching.length
|
|
8221
8796
|
};
|
|
8222
8797
|
}
|
|
8223
8798
|
__name(pageFan, "pageFan");
|
|
8799
|
+
function fanPageBounds(query) {
|
|
8800
|
+
const offset = query.offset !== void 0 && Number.isFinite(query.offset) ? Math.max(0, Math.floor(query.offset)) : 0;
|
|
8801
|
+
const limit = query.limit !== void 0 && Number.isFinite(query.limit) ? Math.min(FAN_PAGE_MAX, Math.max(1, Math.floor(query.limit))) : 100;
|
|
8802
|
+
return {
|
|
8803
|
+
offset,
|
|
8804
|
+
limit
|
|
8805
|
+
};
|
|
8806
|
+
}
|
|
8807
|
+
__name(fanPageBounds, "fanPageBounds");
|
|
8808
|
+
|
|
8809
|
+
// src/run-detail-store.ts
|
|
8810
|
+
function aggregatesCheckpoints(store) {
|
|
8811
|
+
return store.checkpointStats !== void 0 && store.queryCheckpoints !== void 0;
|
|
8812
|
+
}
|
|
8813
|
+
__name(aggregatesCheckpoints, "aggregatesCheckpoints");
|
|
8814
|
+
function checkpointStatsOf(checkpoints, opts) {
|
|
8815
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
8816
|
+
for (const cp of checkpoints) {
|
|
8817
|
+
const group = cp.parallelGroup ?? null;
|
|
8818
|
+
if (opts?.group !== void 0 && group !== opts.group) continue;
|
|
8819
|
+
const nameKind = checkpointNameKind(cp.name);
|
|
8820
|
+
const key = JSON.stringify([
|
|
8821
|
+
group,
|
|
8822
|
+
cp.status,
|
|
8823
|
+
nameKind
|
|
8824
|
+
]);
|
|
8825
|
+
const enqueuedAt = cp.enqueuedAt ?? cp.startedAt;
|
|
8826
|
+
const bucket = buckets.get(key);
|
|
8827
|
+
if (!bucket) {
|
|
8828
|
+
buckets.set(key, {
|
|
8829
|
+
group,
|
|
8830
|
+
status: cp.status,
|
|
8831
|
+
nameKind,
|
|
8832
|
+
count: 1,
|
|
8833
|
+
minSeq: cp.seq,
|
|
8834
|
+
maxSeq: cp.seq,
|
|
8835
|
+
minName: cp.name,
|
|
8836
|
+
maxName: cp.name,
|
|
8837
|
+
minEnqueuedAt: enqueuedAt,
|
|
8838
|
+
minStartedAt: cp.startedAt,
|
|
8839
|
+
maxStartedAt: cp.startedAt,
|
|
8840
|
+
maxFinishedAt: cp.finishedAt,
|
|
8841
|
+
sumAttempts: cp.attempts
|
|
8842
|
+
});
|
|
8843
|
+
continue;
|
|
8844
|
+
}
|
|
8845
|
+
bucket.count += 1;
|
|
8846
|
+
bucket.minSeq = Math.min(bucket.minSeq, cp.seq);
|
|
8847
|
+
bucket.maxSeq = Math.max(bucket.maxSeq, cp.seq);
|
|
8848
|
+
if (cp.name < bucket.minName) bucket.minName = cp.name;
|
|
8849
|
+
if (cp.name > bucket.maxName) bucket.maxName = cp.name;
|
|
8850
|
+
if (enqueuedAt < bucket.minEnqueuedAt) bucket.minEnqueuedAt = enqueuedAt;
|
|
8851
|
+
if (cp.startedAt < bucket.minStartedAt) bucket.minStartedAt = cp.startedAt;
|
|
8852
|
+
if (cp.startedAt > bucket.maxStartedAt) bucket.maxStartedAt = cp.startedAt;
|
|
8853
|
+
if (cp.finishedAt > bucket.maxFinishedAt) bucket.maxFinishedAt = cp.finishedAt;
|
|
8854
|
+
bucket.sumAttempts += cp.attempts;
|
|
8855
|
+
}
|
|
8856
|
+
return [
|
|
8857
|
+
...buckets.values()
|
|
8858
|
+
];
|
|
8859
|
+
}
|
|
8860
|
+
__name(checkpointStatsOf, "checkpointStatsOf");
|
|
8861
|
+
function queryCheckpointsOf(checkpoints, query) {
|
|
8862
|
+
const excluded = new Set(query.excludeGroups ?? []);
|
|
8863
|
+
const matching = checkpoints.filter((cp) => (query.group === void 0 || cp.parallelGroup === query.group) && (cp.parallelGroup === void 0 || !excluded.has(cp.parallelGroup)) && (query.statuses === void 0 || query.statuses.includes(cp.status)) && (query.namePrefixes === void 0 || query.namePrefixes.some((prefix) => cp.name.startsWith(prefix)))).sort((a, b) => a.seq - b.seq);
|
|
8864
|
+
const offset = query.offset ?? 0;
|
|
8865
|
+
return matching.slice(offset, query.limit === void 0 ? void 0 : offset + query.limit);
|
|
8866
|
+
}
|
|
8867
|
+
__name(queryCheckpointsOf, "queryCheckpointsOf");
|
|
8868
|
+
var TERMINAL = /* @__PURE__ */ __name((status) => status !== "pending" && status !== "running", "TERMINAL");
|
|
8869
|
+
function isWide(group, rows, over) {
|
|
8870
|
+
return rows > over || (parseFanGroup(group)?.size ?? 0) > over;
|
|
8871
|
+
}
|
|
8872
|
+
__name(isWide, "isWide");
|
|
8873
|
+
function mergeBySeq(a, b) {
|
|
8874
|
+
return [
|
|
8875
|
+
...a,
|
|
8876
|
+
...b
|
|
8877
|
+
].sort((x, y) => x.seq - y.seq);
|
|
8878
|
+
}
|
|
8879
|
+
__name(mergeBySeq, "mergeBySeq");
|
|
8880
|
+
async function failedChildren(store, runId, group, buckets) {
|
|
8881
|
+
if (!buckets.some((b) => b.nameKind === "child" && b.status === "completed")) {
|
|
8882
|
+
return {
|
|
8883
|
+
completed: [],
|
|
8884
|
+
failed: []
|
|
8885
|
+
};
|
|
8886
|
+
}
|
|
8887
|
+
const completed = await store.queryCheckpoints(runId, {
|
|
8888
|
+
group,
|
|
8889
|
+
statuses: [
|
|
8890
|
+
"completed"
|
|
8891
|
+
],
|
|
8892
|
+
namePrefixes: [
|
|
8893
|
+
"signal:child:"
|
|
8894
|
+
]
|
|
8895
|
+
});
|
|
8896
|
+
return {
|
|
8897
|
+
completed,
|
|
8898
|
+
failed: completed.filter(isFailedMember)
|
|
8899
|
+
};
|
|
8900
|
+
}
|
|
8901
|
+
__name(failedChildren, "failedChildren");
|
|
8902
|
+
async function summarizeFanFromStats(store, runId, group, buckets, folded) {
|
|
8903
|
+
const counts = {};
|
|
8904
|
+
const add = /* @__PURE__ */ __name((status, n) => {
|
|
8905
|
+
counts[status] = (counts[status] ?? 0) + n;
|
|
8906
|
+
}, "add");
|
|
8907
|
+
let size = 0;
|
|
8908
|
+
let startedAt;
|
|
8909
|
+
let finishedAt;
|
|
8910
|
+
let firstSeq = Number.POSITIVE_INFINITY;
|
|
8911
|
+
let lastSeq = Number.NEGATIVE_INFINITY;
|
|
8912
|
+
let minName;
|
|
8913
|
+
let maxName;
|
|
8914
|
+
let childRuns = false;
|
|
8915
|
+
for (const b of buckets) {
|
|
8916
|
+
firstSeq = Math.min(firstSeq, b.minSeq);
|
|
8917
|
+
lastSeq = Math.max(lastSeq, b.maxSeq);
|
|
8918
|
+
if (minName === void 0 || b.minName < minName) minName = b.minName;
|
|
8919
|
+
if (maxName === void 0 || b.maxName > maxName) maxName = b.maxName;
|
|
8920
|
+
if (b.nameKind === "child" || b.nameKind === "folded") childRuns = true;
|
|
8921
|
+
if (TERMINAL(b.status)) {
|
|
8922
|
+
if (!startedAt || b.minStartedAt < startedAt) startedAt = b.minStartedAt;
|
|
8923
|
+
if (!finishedAt || b.maxFinishedAt > finishedAt) finishedAt = b.maxFinishedAt;
|
|
8924
|
+
} else if (!startedAt || b.minEnqueuedAt < startedAt) {
|
|
8925
|
+
startedAt = b.minEnqueuedAt;
|
|
8926
|
+
}
|
|
8927
|
+
if (b.nameKind === "folded") continue;
|
|
8928
|
+
if (b.nameKind === "child" && b.status === "completed") continue;
|
|
8929
|
+
add(b.status, b.count);
|
|
8930
|
+
size += b.count;
|
|
8931
|
+
}
|
|
8932
|
+
for (const cp of folded) {
|
|
8933
|
+
const n = foldedCount(cp) ?? 1;
|
|
8934
|
+
add(isFailedMember(cp) ? "failed" : cp.status, n);
|
|
8935
|
+
size += n;
|
|
8936
|
+
}
|
|
8937
|
+
const children = await failedChildren(store, runId, group, buckets);
|
|
8938
|
+
const failedChildSet = new Set(children.failed);
|
|
8939
|
+
for (const cp of children.completed) add(failedChildSet.has(cp) ? "failed" : "completed", 1);
|
|
8940
|
+
size += children.completed.length;
|
|
8941
|
+
const failedStatusRows = buckets.some((b) => b.status === "failed") ? await store.queryCheckpoints(runId, {
|
|
8942
|
+
group,
|
|
8943
|
+
statuses: [
|
|
8944
|
+
"failed"
|
|
8945
|
+
],
|
|
8946
|
+
limit: FAN_SUMMARY_FAILURES
|
|
8947
|
+
}) : [];
|
|
8948
|
+
const failures = mergeBySeq(failedStatusRows, children.failed).slice(0, FAN_SUMMARY_FAILURES);
|
|
8949
|
+
const parsed = parseFanGroup(group);
|
|
8950
|
+
const total = parsed?.size;
|
|
8951
|
+
return {
|
|
8952
|
+
group,
|
|
8953
|
+
label: childRuns ? "child runs" : minName === maxName ? minName ?? "" : "steps",
|
|
8954
|
+
firstSeq,
|
|
8955
|
+
lastSeq,
|
|
8956
|
+
size,
|
|
8957
|
+
total,
|
|
8958
|
+
notStarted: total === void 0 ? void 0 : Math.max(0, total - size),
|
|
8959
|
+
failFast: parsed?.failFast ?? false,
|
|
8960
|
+
counts,
|
|
8961
|
+
startedAt,
|
|
8962
|
+
finishedAt,
|
|
8963
|
+
failures
|
|
8964
|
+
};
|
|
8965
|
+
}
|
|
8966
|
+
__name(summarizeFanFromStats, "summarizeFanFromStats");
|
|
8967
|
+
async function firstChildIds(store, runId) {
|
|
8968
|
+
const ids = [];
|
|
8969
|
+
for (let offset = 0; ids.length < SUMMARY_CHILD_LIMIT; offset += SUMMARY_CHILD_LIMIT) {
|
|
8970
|
+
const page = await store.queryCheckpoints(runId, {
|
|
8971
|
+
namePrefixes: CHILD_EDGE_PREFIXES,
|
|
8972
|
+
offset,
|
|
8973
|
+
limit: SUMMARY_CHILD_LIMIT
|
|
8974
|
+
});
|
|
8975
|
+
ids.push(...childIdsOf(page));
|
|
8976
|
+
if (page.length < SUMMARY_CHILD_LIMIT) break;
|
|
8977
|
+
}
|
|
8978
|
+
return ids.slice(0, SUMMARY_CHILD_LIMIT);
|
|
8979
|
+
}
|
|
8980
|
+
__name(firstChildIds, "firstChildIds");
|
|
8981
|
+
async function summarizeRunFromStore(store, runId, over, stats) {
|
|
8982
|
+
const all = stats ?? await store.checkpointStats(runId);
|
|
8983
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
8984
|
+
let edgeRows = 0;
|
|
8985
|
+
let hasFolded = false;
|
|
8986
|
+
for (const b of all) {
|
|
8987
|
+
if (b.nameKind === "child" || b.nameKind === "spawn") edgeRows += b.count;
|
|
8988
|
+
if (b.nameKind === "folded") hasFolded = true;
|
|
8989
|
+
if (b.group === null) continue;
|
|
8990
|
+
const buckets = byGroup.get(b.group);
|
|
8991
|
+
if (buckets) buckets.push(b);
|
|
8992
|
+
else byGroup.set(b.group, [
|
|
8993
|
+
b
|
|
8994
|
+
]);
|
|
8995
|
+
}
|
|
8996
|
+
const wide = [
|
|
8997
|
+
...byGroup
|
|
8998
|
+
].filter(([group, buckets]) => isWide(group, buckets.reduce((n, b) => n + b.count, 0), over));
|
|
8999
|
+
const [timeline, folded, children] = await Promise.all([
|
|
9000
|
+
store.queryCheckpoints(runId, {
|
|
9001
|
+
excludeGroups: wide.map(([group]) => group)
|
|
9002
|
+
}),
|
|
9003
|
+
hasFolded ? store.queryCheckpoints(runId, {
|
|
9004
|
+
namePrefixes: [
|
|
9005
|
+
FOLDED_PREFIX
|
|
9006
|
+
]
|
|
9007
|
+
}) : [],
|
|
9008
|
+
edgeRows > 0 || hasFolded ? firstChildIds(store, runId) : []
|
|
9009
|
+
]);
|
|
9010
|
+
const fans = await Promise.all(wide.map(([group, buckets]) => summarizeFanFromStats(store, runId, group, buckets, folded.filter((cp) => cp.parallelGroup === group))));
|
|
9011
|
+
const childCount = folded.reduce((n, cp) => n + childIdsOf([
|
|
9012
|
+
cp
|
|
9013
|
+
]).length, edgeRows);
|
|
9014
|
+
return {
|
|
9015
|
+
timeline,
|
|
9016
|
+
fans: fans.sort((a, b) => a.firstSeq - b.firstSeq),
|
|
9017
|
+
children,
|
|
9018
|
+
childCount
|
|
9019
|
+
};
|
|
9020
|
+
}
|
|
9021
|
+
__name(summarizeRunFromStore, "summarizeRunFromStore");
|
|
9022
|
+
async function pageFanFromStore(store, runId, group, query, stats) {
|
|
9023
|
+
const buckets = stats ?? await store.checkpointStats(runId, {
|
|
9024
|
+
group
|
|
9025
|
+
});
|
|
9026
|
+
const { offset, limit } = fanPageBounds(query);
|
|
9027
|
+
if (!query.failedOnly) {
|
|
9028
|
+
const total2 = buckets.reduce((n, b) => n + b.count, 0);
|
|
9029
|
+
const items = total2 > offset ? await store.queryCheckpoints(runId, {
|
|
9030
|
+
group,
|
|
9031
|
+
offset,
|
|
9032
|
+
limit
|
|
9033
|
+
}) : [];
|
|
9034
|
+
return {
|
|
9035
|
+
items,
|
|
9036
|
+
total: total2
|
|
9037
|
+
};
|
|
9038
|
+
}
|
|
9039
|
+
const failedStatus = buckets.filter((b) => b.status === "failed").reduce((n, b) => n + b.count, 0);
|
|
9040
|
+
const children = await failedChildren(store, runId, group, buckets);
|
|
9041
|
+
const total = failedStatus + children.failed.length;
|
|
9042
|
+
if (total <= offset) return {
|
|
9043
|
+
items: [],
|
|
9044
|
+
total
|
|
9045
|
+
};
|
|
9046
|
+
if (children.failed.length === 0) {
|
|
9047
|
+
return {
|
|
9048
|
+
items: await store.queryCheckpoints(runId, {
|
|
9049
|
+
group,
|
|
9050
|
+
statuses: [
|
|
9051
|
+
"failed"
|
|
9052
|
+
],
|
|
9053
|
+
offset,
|
|
9054
|
+
limit
|
|
9055
|
+
}),
|
|
9056
|
+
total
|
|
9057
|
+
};
|
|
9058
|
+
}
|
|
9059
|
+
const failedRows = await store.queryCheckpoints(runId, {
|
|
9060
|
+
group,
|
|
9061
|
+
statuses: [
|
|
9062
|
+
"failed"
|
|
9063
|
+
],
|
|
9064
|
+
limit: offset + limit
|
|
9065
|
+
});
|
|
9066
|
+
return {
|
|
9067
|
+
items: mergeBySeq(failedRows, children.failed).slice(offset, offset + limit),
|
|
9068
|
+
total
|
|
9069
|
+
};
|
|
9070
|
+
}
|
|
9071
|
+
__name(pageFanFromStore, "pageFanFromStore");
|
|
9072
|
+
function hash(text) {
|
|
9073
|
+
let h1 = 3735928559;
|
|
9074
|
+
let h2 = 1103547991;
|
|
9075
|
+
for (let i = 0; i < text.length; i++) {
|
|
9076
|
+
const ch = text.charCodeAt(i);
|
|
9077
|
+
h1 = Math.imul(h1 ^ ch, 2654435761);
|
|
9078
|
+
h2 = Math.imul(h2 ^ ch, 1597334677);
|
|
9079
|
+
}
|
|
9080
|
+
h1 = Math.imul(h1 ^ h1 >>> 16, 2246822507) ^ Math.imul(h2 ^ h2 >>> 13, 3266489909);
|
|
9081
|
+
h2 = Math.imul(h2 ^ h2 >>> 16, 2246822507) ^ Math.imul(h1 ^ h1 >>> 13, 3266489909);
|
|
9082
|
+
return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16);
|
|
9083
|
+
}
|
|
9084
|
+
__name(hash, "hash");
|
|
9085
|
+
function statsFingerprint(stats) {
|
|
9086
|
+
return stats.map((b) => [
|
|
9087
|
+
b.group ?? "",
|
|
9088
|
+
b.status,
|
|
9089
|
+
b.nameKind,
|
|
9090
|
+
b.count,
|
|
9091
|
+
b.minSeq,
|
|
9092
|
+
b.maxSeq,
|
|
9093
|
+
b.minName,
|
|
9094
|
+
b.maxName,
|
|
9095
|
+
b.minEnqueuedAt.getTime(),
|
|
9096
|
+
b.minStartedAt.getTime(),
|
|
9097
|
+
b.maxStartedAt.getTime(),
|
|
9098
|
+
b.maxFinishedAt.getTime(),
|
|
9099
|
+
b.sumAttempts
|
|
9100
|
+
].join("")).sort().join("");
|
|
9101
|
+
}
|
|
9102
|
+
__name(statsFingerprint, "statsFingerprint");
|
|
9103
|
+
function runDetailEtag(run, stats, over) {
|
|
9104
|
+
return `"rd-${hash(`${over}${JSON.stringify(run)}${statsFingerprint(stats)}`)}"`;
|
|
9105
|
+
}
|
|
9106
|
+
__name(runDetailEtag, "runDetailEtag");
|
|
9107
|
+
function fanPageEtag(group, stats, query) {
|
|
9108
|
+
const { offset, limit } = fanPageBounds(query);
|
|
9109
|
+
const key = [
|
|
9110
|
+
group,
|
|
9111
|
+
offset,
|
|
9112
|
+
limit,
|
|
9113
|
+
query.failedOnly === true
|
|
9114
|
+
].join("");
|
|
9115
|
+
return `"fp-${hash(`${key}${statsFingerprint(stats)}`)}"`;
|
|
9116
|
+
}
|
|
9117
|
+
__name(fanPageEtag, "fanPageEtag");
|
|
9118
|
+
function etagMatches(ifNoneMatch, etag) {
|
|
9119
|
+
if (!ifNoneMatch) return false;
|
|
9120
|
+
const bare = /* @__PURE__ */ __name((tag) => tag.trim().replace(/^W\//, ""), "bare");
|
|
9121
|
+
const target = bare(etag);
|
|
9122
|
+
return ifNoneMatch.split(",").some((tag) => tag.trim() === "*" || bare(tag) === target);
|
|
9123
|
+
}
|
|
9124
|
+
__name(etagMatches, "etagMatches");
|
|
8224
9125
|
|
|
8225
9126
|
// src/codec-state-store.ts
|
|
8226
9127
|
var CodecStateStore = class {
|
|
@@ -8250,11 +9151,19 @@ var CodecStateStore = class {
|
|
|
8250
9151
|
/** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
|
|
8251
9152
|
* plain-save fallback still kicks in through the wrapper. */
|
|
8252
9153
|
saveCheckpointIfPending;
|
|
9154
|
+
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
9155
|
+
saveCheckpointsIfPending;
|
|
8253
9156
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
8254
9157
|
countCheckpoints;
|
|
8255
9158
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
8256
9159
|
* when the inner store prunes, so the retention poller still sees (and warns about) its absence. */
|
|
8257
9160
|
pruneTerminalRuns;
|
|
9161
|
+
/** Forwarded verbatim — the buckets aggregate names, statuses and timestamps, never a payload.
|
|
9162
|
+
* Bound only when the inner store aggregates, so the run detail's fallback still applies. */
|
|
9163
|
+
checkpointStats;
|
|
9164
|
+
/** Filters on unencoded columns, then decodes the rows like {@link listCheckpoints}. Bound only
|
|
9165
|
+
* when the inner store has it. */
|
|
9166
|
+
queryCheckpoints;
|
|
8258
9167
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store counts, so the
|
|
8259
9168
|
* engine's facet/listing fallback still applies through the wrapper. */
|
|
8260
9169
|
countRuns;
|
|
@@ -8281,8 +9190,14 @@ var CodecStateStore = class {
|
|
|
8281
9190
|
if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
|
|
8282
9191
|
const saveIfPending = inner.saveCheckpointIfPending;
|
|
8283
9192
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
9193
|
+
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
9194
|
+
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
8284
9195
|
const countCps = inner.countCheckpoints;
|
|
8285
9196
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
9197
|
+
const stats = inner.checkpointStats;
|
|
9198
|
+
if (stats) this.checkpointStats = (runId, opts) => stats.call(inner, runId, opts);
|
|
9199
|
+
const queryCps = inner.queryCheckpoints;
|
|
9200
|
+
if (queryCps) this.queryCheckpoints = async (runId, query) => (await queryCps.call(inner, runId, query)).map((c) => this.decCp(c));
|
|
8286
9201
|
const prune = inner.pruneTerminalRuns;
|
|
8287
9202
|
if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
|
|
8288
9203
|
const count = inner.countRuns;
|
|
@@ -8490,8 +9405,11 @@ function collectMetrics(engine) {
|
|
|
8490
9405
|
started: 0,
|
|
8491
9406
|
completed: 0,
|
|
8492
9407
|
failed: 0,
|
|
8493
|
-
suspended: 0
|
|
9408
|
+
suspended: 0,
|
|
9409
|
+
routed: 0,
|
|
9410
|
+
unrouted: 0
|
|
8494
9411
|
},
|
|
9412
|
+
routedByCode: {},
|
|
8495
9413
|
steps: {
|
|
8496
9414
|
started: 0,
|
|
8497
9415
|
completed: 0,
|
|
@@ -8533,6 +9451,15 @@ function collectMetrics(engine) {
|
|
|
8533
9451
|
case "run.suspended":
|
|
8534
9452
|
m.runs.suspended += 1;
|
|
8535
9453
|
break;
|
|
9454
|
+
case "run.routed": {
|
|
9455
|
+
m.runs.routed += 1;
|
|
9456
|
+
const code = e.error?.code ?? "*";
|
|
9457
|
+
m.routedByCode[code] = (m.routedByCode[code] ?? 0) + 1;
|
|
9458
|
+
break;
|
|
9459
|
+
}
|
|
9460
|
+
case "run.unrouted":
|
|
9461
|
+
m.runs.unrouted += 1;
|
|
9462
|
+
break;
|
|
8536
9463
|
case "step.started":
|
|
8537
9464
|
m.steps.started += 1;
|
|
8538
9465
|
break;
|
|
@@ -8559,6 +9486,8 @@ function collectMetrics(engine) {
|
|
|
8559
9486
|
for (const [k, v] of Object.entries(m.steps)) lines.push(`durable_steps_total{event="${k}"} ${v}`);
|
|
8560
9487
|
lines.push("# TYPE durable_runs_by_workflow_total counter");
|
|
8561
9488
|
for (const [name, v] of Object.entries(m.byWorkflow)) for (const [event, n] of Object.entries(v)) lines.push(`durable_runs_by_workflow_total{workflow="${name}",event="${event}"} ${n}`);
|
|
9489
|
+
lines.push("# TYPE durable_runs_routed_total counter");
|
|
9490
|
+
for (const [code, n] of Object.entries(m.routedByCode)) lines.push(`durable_runs_routed_total{code="${code}"} ${n}`);
|
|
8562
9491
|
lines.push("# TYPE durable_step_duration_ms_sum counter");
|
|
8563
9492
|
lines.push(`durable_step_duration_ms_sum ${m.stepDuration.sumMs}`);
|
|
8564
9493
|
lines.push("# TYPE durable_step_duration_count counter");
|
|
@@ -8729,6 +9658,11 @@ var InMemoryStateStore = class {
|
|
|
8729
9658
|
* so this lets the benchmark contrast "fetch N, scan in JS" against "fetch the k that match".
|
|
8730
9659
|
*/
|
|
8731
9660
|
lastCheckpointRowsRead = 0;
|
|
9661
|
+
/**
|
|
9662
|
+
* Test/observability hook: checkpoint rows materialized since construction, across every
|
|
9663
|
+
* checkpoint read that returns rows — what a run-detail benchmark compares between paths.
|
|
9664
|
+
*/
|
|
9665
|
+
checkpointRowsRead = 0;
|
|
8732
9666
|
key(runId, seq) {
|
|
8733
9667
|
return `${runId}:${seq}`;
|
|
8734
9668
|
}
|
|
@@ -8854,6 +9788,16 @@ var InMemoryStateStore = class {
|
|
|
8854
9788
|
});
|
|
8855
9789
|
return true;
|
|
8856
9790
|
}
|
|
9791
|
+
async saveCheckpointsIfPending(checkpoints) {
|
|
9792
|
+
return checkpoints.map((checkpoint) => {
|
|
9793
|
+
const key = this.key(checkpoint.runId, checkpoint.seq);
|
|
9794
|
+
if (this.checkpoints.get(key)?.status !== "pending") return false;
|
|
9795
|
+
this.checkpoints.set(key, {
|
|
9796
|
+
...checkpoint
|
|
9797
|
+
});
|
|
9798
|
+
return true;
|
|
9799
|
+
});
|
|
9800
|
+
}
|
|
8857
9801
|
async listIncompleteRuns(namespace) {
|
|
8858
9802
|
return [
|
|
8859
9803
|
...this.runs.values()
|
|
@@ -9133,6 +10077,23 @@ var InMemoryStateStore = class {
|
|
|
9133
10077
|
...cp
|
|
9134
10078
|
}));
|
|
9135
10079
|
this.lastCheckpointRowsRead = rows.length;
|
|
10080
|
+
this.checkpointRowsRead += rows.length;
|
|
10081
|
+
return rows;
|
|
10082
|
+
}
|
|
10083
|
+
runCheckpoints(runId) {
|
|
10084
|
+
return [
|
|
10085
|
+
...this.checkpoints.values()
|
|
10086
|
+
].filter((cp) => cp.runId === runId);
|
|
10087
|
+
}
|
|
10088
|
+
async checkpointStats(runId, opts) {
|
|
10089
|
+
return checkpointStatsOf(this.runCheckpoints(runId), opts);
|
|
10090
|
+
}
|
|
10091
|
+
async queryCheckpoints(runId, query) {
|
|
10092
|
+
const rows = queryCheckpointsOf(this.runCheckpoints(runId), query).map((cp) => ({
|
|
10093
|
+
...cp
|
|
10094
|
+
}));
|
|
10095
|
+
this.lastCheckpointRowsRead = rows.length;
|
|
10096
|
+
this.checkpointRowsRead += rows.length;
|
|
9136
10097
|
return rows;
|
|
9137
10098
|
}
|
|
9138
10099
|
async getLatestCheckpointByName(runId, name) {
|
|
@@ -9160,6 +10121,7 @@ var InMemoryStateStore = class {
|
|
|
9160
10121
|
...cp
|
|
9161
10122
|
}));
|
|
9162
10123
|
this.lastCheckpointRowsRead = rows.length;
|
|
10124
|
+
this.checkpointRowsRead += rows.length;
|
|
9163
10125
|
return rows;
|
|
9164
10126
|
}
|
|
9165
10127
|
};
|
|
@@ -9211,6 +10173,7 @@ var InMemoryTransport = class {
|
|
|
9211
10173
|
};
|
|
9212
10174
|
// Annotate the CommonJS export names for ESM import in node:
|
|
9213
10175
|
0 && (module.exports = {
|
|
10176
|
+
CHILD_EDGE_PREFIXES,
|
|
9214
10177
|
CONTROL_FLOW_SIGNAL,
|
|
9215
10178
|
CURRENT_PROTOCOL_VERSION,
|
|
9216
10179
|
CodecStateStore,
|
|
@@ -9231,6 +10194,7 @@ var InMemoryTransport = class {
|
|
|
9231
10194
|
FAN_SUMMARY_FAILURES,
|
|
9232
10195
|
FatalError,
|
|
9233
10196
|
GatherError,
|
|
10197
|
+
HANDOFF_SEQ_BASE,
|
|
9234
10198
|
InMemoryAdmissionBackend,
|
|
9235
10199
|
InMemoryStateStore,
|
|
9236
10200
|
InMemoryTransport,
|
|
@@ -9265,9 +10229,11 @@ var InMemoryTransport = class {
|
|
|
9265
10229
|
WorkflowEngine,
|
|
9266
10230
|
WorkflowSuspended,
|
|
9267
10231
|
aggregateAnnouncements,
|
|
10232
|
+
aggregatesCheckpoints,
|
|
9268
10233
|
announcementsOf,
|
|
9269
10234
|
applyAttributeQuery,
|
|
9270
10235
|
assertConcurrency,
|
|
10236
|
+
assertRoutes,
|
|
9271
10237
|
attributeColumnFor,
|
|
9272
10238
|
attributeOperand,
|
|
9273
10239
|
attributePredicateOperands,
|
|
@@ -9277,10 +10243,14 @@ var InMemoryTransport = class {
|
|
|
9277
10243
|
buildWorkflowDirectory,
|
|
9278
10244
|
canRoute,
|
|
9279
10245
|
capableWorkers,
|
|
10246
|
+
checkpointNameKind,
|
|
10247
|
+
checkpointStatsOf,
|
|
9280
10248
|
childIdsOf,
|
|
9281
10249
|
classifyWaiterToken,
|
|
9282
10250
|
clearDurableExecutionWrappers,
|
|
9283
10251
|
clearTerminalSingletonAdmission,
|
|
10252
|
+
codeAndDetails,
|
|
10253
|
+
codeOf,
|
|
9284
10254
|
collectMetrics,
|
|
9285
10255
|
concurrencyTag,
|
|
9286
10256
|
controlPlaneDescriptor,
|
|
@@ -9293,13 +10263,20 @@ var InMemoryTransport = class {
|
|
|
9293
10263
|
debug,
|
|
9294
10264
|
describeWorker,
|
|
9295
10265
|
descriptorHash,
|
|
10266
|
+
detailsOf,
|
|
9296
10267
|
error,
|
|
10268
|
+
etagMatches,
|
|
9297
10269
|
eventMatchOf,
|
|
9298
10270
|
eventMatches,
|
|
9299
10271
|
eventNameOf,
|
|
9300
10272
|
eventPrefix,
|
|
9301
10273
|
eventToken,
|
|
9302
10274
|
facetOrigin,
|
|
10275
|
+
fanPageBounds,
|
|
10276
|
+
fanPageEtag,
|
|
10277
|
+
foldedCount,
|
|
10278
|
+
handoffHandlerId,
|
|
10279
|
+
handoffName,
|
|
9303
10280
|
hasDurableExecutionWrappers,
|
|
9304
10281
|
heartbeatStatus,
|
|
9305
10282
|
indexWaitersByRun,
|
|
@@ -9311,6 +10288,7 @@ var InMemoryTransport = class {
|
|
|
9311
10288
|
isNonEmptyRunScope,
|
|
9312
10289
|
isWorkflowControlFlowSignal,
|
|
9313
10290
|
listSignalWaitersOfRuns,
|
|
10291
|
+
liveHandoff,
|
|
9314
10292
|
log,
|
|
9315
10293
|
matchesAttributes,
|
|
9316
10294
|
mergeRunFacetRows,
|
|
@@ -9321,17 +10299,23 @@ var InMemoryTransport = class {
|
|
|
9321
10299
|
normalizeAttributeRows,
|
|
9322
10300
|
normalizeDescriptor,
|
|
9323
10301
|
observedAnnouncements,
|
|
10302
|
+
onErrorRunId,
|
|
9324
10303
|
pageFan,
|
|
10304
|
+
pageFanFromStore,
|
|
9325
10305
|
parentRunIdOf,
|
|
9326
10306
|
parseDuration,
|
|
9327
10307
|
partitionOfToken,
|
|
9328
10308
|
planDispatch,
|
|
9329
10309
|
prevCronFireMs,
|
|
10310
|
+
queryCheckpointsOf,
|
|
9330
10311
|
readSearchAttributes,
|
|
9331
10312
|
requiredCapabilities,
|
|
9332
10313
|
resolveConcurrency,
|
|
9333
10314
|
resolveRouting,
|
|
9334
10315
|
resolveRunWaiting,
|
|
10316
|
+
retiredHandoffName,
|
|
10317
|
+
routeFor,
|
|
10318
|
+
runDetailEtag,
|
|
9335
10319
|
runDurableExecution,
|
|
9336
10320
|
runInStepLogger,
|
|
9337
10321
|
runInStepScope,
|
|
@@ -9352,6 +10336,7 @@ var InMemoryTransport = class {
|
|
|
9352
10336
|
subEvent,
|
|
9353
10337
|
subProcess,
|
|
9354
10338
|
summarizeFan,
|
|
10339
|
+
summarizeRunFromStore,
|
|
9355
10340
|
summarizeTimeline,
|
|
9356
10341
|
tenantGroup,
|
|
9357
10342
|
useDurableExecution,
|