@dudousxd/nestjs-durable-core 0.77.0 → 0.79.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1200 -84
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +590 -120
- package/dist/index.d.ts +590 -120
- package/dist/index.js +1170 -84
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -22,6 +22,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
22
22
|
// src/index.ts
|
|
23
23
|
var index_exports = {};
|
|
24
24
|
__export(index_exports, {
|
|
25
|
+
CHILD_EDGE_PREFIXES: () => CHILD_EDGE_PREFIXES,
|
|
25
26
|
CONTROL_FLOW_SIGNAL: () => CONTROL_FLOW_SIGNAL,
|
|
26
27
|
CURRENT_PROTOCOL_VERSION: () => CURRENT_PROTOCOL_VERSION,
|
|
27
28
|
CodecStateStore: () => CodecStateStore,
|
|
@@ -39,8 +40,10 @@ __export(index_exports, {
|
|
|
39
40
|
EVENT_TOKEN_PREFIX: () => EVENT_TOKEN_PREFIX,
|
|
40
41
|
Entities: () => Entities,
|
|
41
42
|
EventAccumulators: () => EventAccumulators,
|
|
43
|
+
FAN_SUMMARY_FAILURES: () => FAN_SUMMARY_FAILURES,
|
|
42
44
|
FatalError: () => FatalError,
|
|
43
45
|
GatherError: () => GatherError,
|
|
46
|
+
HANDOFF_SEQ_BASE: () => HANDOFF_SEQ_BASE,
|
|
44
47
|
InMemoryAdmissionBackend: () => InMemoryAdmissionBackend,
|
|
45
48
|
InMemoryStateStore: () => InMemoryStateStore,
|
|
46
49
|
InMemoryTransport: () => InMemoryTransport,
|
|
@@ -60,6 +63,7 @@ __export(index_exports, {
|
|
|
60
63
|
SINGLETON_ADMITTED_TAG: () => SINGLETON_ADMITTED_TAG,
|
|
61
64
|
STATE_STORE: () => STATE_STORE,
|
|
62
65
|
STATE_STORE_CANONICAL: () => STATE_STORE_CANONICAL,
|
|
66
|
+
SUMMARY_CHILD_LIMIT: () => SUMMARY_CHILD_LIMIT,
|
|
63
67
|
ScheduleAlreadyExistsError: () => ScheduleAlreadyExistsError,
|
|
64
68
|
ScheduleClient: () => ScheduleClient,
|
|
65
69
|
ScheduleNotFoundError: () => ScheduleNotFoundError,
|
|
@@ -74,9 +78,11 @@ __export(index_exports, {
|
|
|
74
78
|
WorkflowEngine: () => WorkflowEngine,
|
|
75
79
|
WorkflowSuspended: () => WorkflowSuspended,
|
|
76
80
|
aggregateAnnouncements: () => aggregateAnnouncements,
|
|
81
|
+
aggregatesCheckpoints: () => aggregatesCheckpoints,
|
|
77
82
|
announcementsOf: () => announcementsOf,
|
|
78
83
|
applyAttributeQuery: () => applyAttributeQuery,
|
|
79
84
|
assertConcurrency: () => assertConcurrency,
|
|
85
|
+
assertRoutes: () => assertRoutes,
|
|
80
86
|
attributeColumnFor: () => attributeColumnFor,
|
|
81
87
|
attributeOperand: () => attributeOperand,
|
|
82
88
|
attributePredicateOperands: () => attributePredicateOperands,
|
|
@@ -86,9 +92,14 @@ __export(index_exports, {
|
|
|
86
92
|
buildWorkflowDirectory: () => buildWorkflowDirectory,
|
|
87
93
|
canRoute: () => canRoute,
|
|
88
94
|
capableWorkers: () => capableWorkers,
|
|
95
|
+
checkpointNameKind: () => checkpointNameKind,
|
|
96
|
+
checkpointStatsOf: () => checkpointStatsOf,
|
|
97
|
+
childIdsOf: () => childIdsOf,
|
|
89
98
|
classifyWaiterToken: () => classifyWaiterToken,
|
|
90
99
|
clearDurableExecutionWrappers: () => clearDurableExecutionWrappers,
|
|
91
100
|
clearTerminalSingletonAdmission: () => clearTerminalSingletonAdmission,
|
|
101
|
+
codeAndDetails: () => codeAndDetails,
|
|
102
|
+
codeOf: () => codeOf,
|
|
92
103
|
collectMetrics: () => collectMetrics,
|
|
93
104
|
concurrencyTag: () => concurrencyTag,
|
|
94
105
|
controlPlaneDescriptor: () => controlPlaneDescriptor,
|
|
@@ -101,23 +112,32 @@ __export(index_exports, {
|
|
|
101
112
|
debug: () => debug,
|
|
102
113
|
describeWorker: () => describeWorker,
|
|
103
114
|
descriptorHash: () => descriptorHash,
|
|
115
|
+
detailsOf: () => detailsOf,
|
|
104
116
|
error: () => error,
|
|
117
|
+
etagMatches: () => etagMatches,
|
|
105
118
|
eventMatchOf: () => eventMatchOf,
|
|
106
119
|
eventMatches: () => eventMatches,
|
|
107
120
|
eventNameOf: () => eventNameOf,
|
|
108
121
|
eventPrefix: () => eventPrefix,
|
|
109
122
|
eventToken: () => eventToken,
|
|
110
123
|
facetOrigin: () => facetOrigin,
|
|
124
|
+
fanPageBounds: () => fanPageBounds,
|
|
125
|
+
fanPageEtag: () => fanPageEtag,
|
|
126
|
+
foldedCount: () => foldedCount,
|
|
127
|
+
handoffHandlerId: () => handoffHandlerId,
|
|
128
|
+
handoffName: () => handoffName,
|
|
111
129
|
hasDurableExecutionWrappers: () => hasDurableExecutionWrappers,
|
|
112
130
|
heartbeatStatus: () => heartbeatStatus,
|
|
113
131
|
indexWaitersByRun: () => indexWaitersByRun,
|
|
114
132
|
info: () => info,
|
|
115
133
|
isEngineMintedTag: () => isEngineMintedTag,
|
|
116
134
|
isEventToken: () => isEventToken,
|
|
135
|
+
isFailedMember: () => isFailedMember,
|
|
117
136
|
isLegacyDescriptor: () => isLegacyDescriptor,
|
|
118
137
|
isNonEmptyRunScope: () => isNonEmptyRunScope,
|
|
119
138
|
isWorkflowControlFlowSignal: () => isWorkflowControlFlowSignal,
|
|
120
139
|
listSignalWaitersOfRuns: () => listSignalWaitersOfRuns,
|
|
140
|
+
liveHandoff: () => liveHandoff,
|
|
121
141
|
log: () => log,
|
|
122
142
|
matchesAttributes: () => matchesAttributes,
|
|
123
143
|
mergeRunFacetRows: () => mergeRunFacetRows,
|
|
@@ -128,16 +148,23 @@ __export(index_exports, {
|
|
|
128
148
|
normalizeAttributeRows: () => normalizeAttributeRows,
|
|
129
149
|
normalizeDescriptor: () => normalizeDescriptor,
|
|
130
150
|
observedAnnouncements: () => observedAnnouncements,
|
|
151
|
+
onErrorRunId: () => onErrorRunId,
|
|
152
|
+
pageFan: () => pageFan,
|
|
153
|
+
pageFanFromStore: () => pageFanFromStore,
|
|
131
154
|
parentRunIdOf: () => parentRunIdOf,
|
|
132
155
|
parseDuration: () => parseDuration,
|
|
133
156
|
partitionOfToken: () => partitionOfToken,
|
|
134
157
|
planDispatch: () => planDispatch,
|
|
135
158
|
prevCronFireMs: () => prevCronFireMs,
|
|
159
|
+
queryCheckpointsOf: () => queryCheckpointsOf,
|
|
136
160
|
readSearchAttributes: () => readSearchAttributes,
|
|
137
161
|
requiredCapabilities: () => requiredCapabilities,
|
|
138
162
|
resolveConcurrency: () => resolveConcurrency,
|
|
139
163
|
resolveRouting: () => resolveRouting,
|
|
140
164
|
resolveRunWaiting: () => resolveRunWaiting,
|
|
165
|
+
retiredHandoffName: () => retiredHandoffName,
|
|
166
|
+
routeFor: () => routeFor,
|
|
167
|
+
runDetailEtag: () => runDetailEtag,
|
|
141
168
|
runDurableExecution: () => runDurableExecution,
|
|
142
169
|
runInStepLogger: () => runInStepLogger,
|
|
143
170
|
runInStepScope: () => runInStepScope,
|
|
@@ -157,6 +184,9 @@ __export(index_exports, {
|
|
|
157
184
|
sub: () => sub,
|
|
158
185
|
subEvent: () => subEvent,
|
|
159
186
|
subProcess: () => subProcess,
|
|
187
|
+
summarizeFan: () => summarizeFan,
|
|
188
|
+
summarizeRunFromStore: () => summarizeRunFromStore,
|
|
189
|
+
summarizeTimeline: () => summarizeTimeline,
|
|
160
190
|
tenantGroup: () => tenantGroup,
|
|
161
191
|
useDurableExecution: () => useDurableExecution,
|
|
162
192
|
warn: () => warn,
|
|
@@ -428,10 +458,16 @@ var FatalError = class extends Error {
|
|
|
428
458
|
__name(this, "FatalError");
|
|
429
459
|
}
|
|
430
460
|
code;
|
|
431
|
-
|
|
461
|
+
/** Structured, JSON-serializable detail (e.g. validation issues). It travels with the failure to
|
|
462
|
+
* the run's error, a parent's `allSettled` item and an onError handler's input. */
|
|
463
|
+
details;
|
|
464
|
+
/** Read by the step result builders, so a dispatched step's retry policy honours it too. */
|
|
465
|
+
retryable = false;
|
|
466
|
+
constructor(message, code, details) {
|
|
432
467
|
super(message);
|
|
433
468
|
this.name = "FatalError";
|
|
434
469
|
this.code = code;
|
|
470
|
+
this.details = details;
|
|
435
471
|
}
|
|
436
472
|
};
|
|
437
473
|
var ConcurrencyLimitError = class extends Error {
|
|
@@ -509,6 +545,8 @@ var NonDeterminismError = class extends Error {
|
|
|
509
545
|
static {
|
|
510
546
|
__name(this, "NonDeterminismError");
|
|
511
547
|
}
|
|
548
|
+
/** Marks the failure as a broken deploy, which an `onError` route never takes. */
|
|
549
|
+
code = "non_determinism";
|
|
512
550
|
runId;
|
|
513
551
|
seq;
|
|
514
552
|
constructor(runId, seq, expected, recorded) {
|
|
@@ -920,6 +958,9 @@ async function runStepHandler(task, handler) {
|
|
|
920
958
|
...typeof e?.code === "string" ? {
|
|
921
959
|
code: e.code
|
|
922
960
|
} : {},
|
|
961
|
+
...e?.details !== void 0 ? {
|
|
962
|
+
details: e.details
|
|
963
|
+
} : {},
|
|
923
964
|
...typeof e?.retryable === "boolean" ? {
|
|
924
965
|
retryable: e.retryable
|
|
925
966
|
} : {}
|
|
@@ -996,11 +1037,36 @@ __name(isFailedCompletion, "isFailedCompletion");
|
|
|
996
1037
|
function unwrapCompletion(payload, label) {
|
|
997
1038
|
const c = payload;
|
|
998
1039
|
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
999
|
-
throw new FatalError(`${label} failed: ${c.error}
|
|
1040
|
+
throw new FatalError(`${label} failed: ${c.error}`, c.code, c.details);
|
|
1000
1041
|
}
|
|
1001
1042
|
return c?.value;
|
|
1002
1043
|
}
|
|
1003
1044
|
__name(unwrapCompletion, "unwrapCompletion");
|
|
1045
|
+
function failedCompletion(error2) {
|
|
1046
|
+
return {
|
|
1047
|
+
ok: false,
|
|
1048
|
+
error: error2.message,
|
|
1049
|
+
...error2.code !== void 0 ? {
|
|
1050
|
+
code: error2.code
|
|
1051
|
+
} : {},
|
|
1052
|
+
...error2.details !== void 0 ? {
|
|
1053
|
+
details: error2.details
|
|
1054
|
+
} : {}
|
|
1055
|
+
};
|
|
1056
|
+
}
|
|
1057
|
+
__name(failedCompletion, "failedCompletion");
|
|
1058
|
+
function completionError(c) {
|
|
1059
|
+
return {
|
|
1060
|
+
message: c.error,
|
|
1061
|
+
...c.code !== void 0 ? {
|
|
1062
|
+
code: c.code
|
|
1063
|
+
} : {},
|
|
1064
|
+
...c.details !== void 0 ? {
|
|
1065
|
+
details: c.details
|
|
1066
|
+
} : {}
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
__name(completionError, "completionError");
|
|
1004
1070
|
|
|
1005
1071
|
// src/concurrency.ts
|
|
1006
1072
|
var DEFAULT_CONCURRENCY_STATUSES = [
|
|
@@ -1246,8 +1312,9 @@ function allChildId(runId, firstSeq, index, wave) {
|
|
|
1246
1312
|
return `${allBase(runId, firstSeq, wave)}${index}`;
|
|
1247
1313
|
}
|
|
1248
1314
|
__name(allChildId, "allChildId");
|
|
1249
|
-
function allGroup(firstSeq, mode) {
|
|
1250
|
-
|
|
1315
|
+
function allGroup(firstSeq, size, mode) {
|
|
1316
|
+
const group = `all:${firstSeq}:n${size}`;
|
|
1317
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1251
1318
|
}
|
|
1252
1319
|
__name(allGroup, "allGroup");
|
|
1253
1320
|
function stepFanGroup(firstSeq, size, mode) {
|
|
@@ -1256,14 +1323,26 @@ function stepFanGroup(firstSeq, size, mode) {
|
|
|
1256
1323
|
}
|
|
1257
1324
|
__name(stepFanGroup, "stepFanGroup");
|
|
1258
1325
|
function parseStepFanGroup(group) {
|
|
1259
|
-
const
|
|
1260
|
-
if (
|
|
1326
|
+
const fan = parseFanGroup(group);
|
|
1327
|
+
if (fan?.kind !== "steps" || fan.size === void 0) return void 0;
|
|
1261
1328
|
return {
|
|
1262
|
-
firstSeq:
|
|
1263
|
-
size:
|
|
1329
|
+
firstSeq: fan.firstSeq,
|
|
1330
|
+
size: fan.size
|
|
1264
1331
|
};
|
|
1265
1332
|
}
|
|
1266
1333
|
__name(parseStepFanGroup, "parseStepFanGroup");
|
|
1334
|
+
function parseFanGroup(group) {
|
|
1335
|
+
const m = group ? /^(all|steps):(\d+)(?::n?(\d+))?(:failFast)?$/.exec(group) : null;
|
|
1336
|
+
if (!m) return void 0;
|
|
1337
|
+
if (m[1] === "steps" && m[3] === void 0) return void 0;
|
|
1338
|
+
return {
|
|
1339
|
+
kind: m[1],
|
|
1340
|
+
firstSeq: Number(m[2]),
|
|
1341
|
+
size: m[3] === void 0 ? void 0 : Number(m[3]),
|
|
1342
|
+
failFast: m[4] !== void 0
|
|
1343
|
+
};
|
|
1344
|
+
}
|
|
1345
|
+
__name(parseFanGroup, "parseFanGroup");
|
|
1267
1346
|
function allWait(runId, firstSeq, wave) {
|
|
1268
1347
|
const base = allBase(runId, firstSeq, wave);
|
|
1269
1348
|
return {
|
|
@@ -1308,12 +1387,12 @@ function allWaves(size, concurrency) {
|
|
|
1308
1387
|
};
|
|
1309
1388
|
}
|
|
1310
1389
|
__name(allWaves, "allWaves");
|
|
1311
|
-
var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+)$/;
|
|
1390
|
+
var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+|onError\.\d+)$/;
|
|
1312
1391
|
function parentRunIdOf(runId) {
|
|
1313
1392
|
return ENGINE_CHILD_ID.exec(runId)?.[1];
|
|
1314
1393
|
}
|
|
1315
1394
|
__name(parentRunIdOf, "parentRunIdOf");
|
|
1316
|
-
var GROUP = /^all:(\d+)(:failFast)?$/;
|
|
1395
|
+
var GROUP = /^all:(\d+)(?::n\d+)?(:failFast)?$/;
|
|
1317
1396
|
var REST = /^(?:w(\d+)\.)?\d+$/;
|
|
1318
1397
|
function fanOfWaiter(waiter) {
|
|
1319
1398
|
const group = waiter.parallelGroup ? GROUP.exec(waiter.parallelGroup) : null;
|
|
@@ -1393,7 +1472,11 @@ function canonicalizeRegistrations(regs) {
|
|
|
1393
1472
|
requires: r.requires ? [
|
|
1394
1473
|
...new Set(r.requires)
|
|
1395
1474
|
].sort() : null,
|
|
1396
|
-
origin: r.origin ?? null
|
|
1475
|
+
origin: r.origin ?? null,
|
|
1476
|
+
// Conditional, so a registration that states no routes keeps the hash it had before routes existed.
|
|
1477
|
+
...r.onError !== void 0 ? {
|
|
1478
|
+
onError: canonicalRoutes(r.onError)
|
|
1479
|
+
} : {}
|
|
1397
1480
|
}));
|
|
1398
1481
|
const byJson = /* @__PURE__ */ new Map();
|
|
1399
1482
|
for (const entry of canonical) byJson.set(JSON.stringify(entry), entry);
|
|
@@ -1402,6 +1485,20 @@ function canonicalizeRegistrations(regs) {
|
|
|
1402
1485
|
].sort().map((k) => byJson.get(k));
|
|
1403
1486
|
}
|
|
1404
1487
|
__name(canonicalizeRegistrations, "canonicalizeRegistrations");
|
|
1488
|
+
function canonicalRoutes(routes) {
|
|
1489
|
+
const byJson = /* @__PURE__ */ new Map();
|
|
1490
|
+
for (const r of routes) {
|
|
1491
|
+
const entry = {
|
|
1492
|
+
code: r.code ?? null,
|
|
1493
|
+
workflow: r.workflow
|
|
1494
|
+
};
|
|
1495
|
+
byJson.set(JSON.stringify(entry), entry);
|
|
1496
|
+
}
|
|
1497
|
+
return [
|
|
1498
|
+
...byJson.keys()
|
|
1499
|
+
].sort().map((k) => byJson.get(k));
|
|
1500
|
+
}
|
|
1501
|
+
__name(canonicalRoutes, "canonicalRoutes");
|
|
1405
1502
|
function canonicalizeForHash(d) {
|
|
1406
1503
|
const set = /* @__PURE__ */ __name((xs) => [
|
|
1407
1504
|
...new Set(xs)
|
|
@@ -1436,13 +1533,13 @@ function fnv1a64Hex(input) {
|
|
|
1436
1533
|
const OFFSET = 0xcbf29ce484222325n;
|
|
1437
1534
|
const PRIME = 0x100000001b3n;
|
|
1438
1535
|
const MASK = 0xffffffffffffffffn;
|
|
1439
|
-
let
|
|
1536
|
+
let hash2 = OFFSET;
|
|
1440
1537
|
for (let i = 0; i < input.length; i += 1) {
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1538
|
+
hash2 ^= BigInt(input.charCodeAt(i) & 255);
|
|
1539
|
+
hash2 ^= BigInt(input.charCodeAt(i) >> 8 & 255);
|
|
1540
|
+
hash2 = hash2 * PRIME & MASK;
|
|
1444
1541
|
}
|
|
1445
|
-
return
|
|
1542
|
+
return hash2.toString(16).padStart(16, "0");
|
|
1446
1543
|
}
|
|
1447
1544
|
__name(fnv1a64Hex, "fnv1a64Hex");
|
|
1448
1545
|
function descriptorHash(descriptor) {
|
|
@@ -1558,6 +1655,20 @@ function aggregateAnnouncements(descriptors) {
|
|
|
1558
1655
|
const requires = [
|
|
1559
1656
|
...requiresByJson.values()
|
|
1560
1657
|
].sort((x, y) => x.join(",").localeCompare(y.join(",")));
|
|
1658
|
+
const routesByJson = /* @__PURE__ */ new Map();
|
|
1659
|
+
for (const a of announcements) {
|
|
1660
|
+
if (a.onError === void 0) continue;
|
|
1661
|
+
const canonical = canonicalRoutes(a.onError).map((r) => r.code === null ? {
|
|
1662
|
+
workflow: r.workflow
|
|
1663
|
+
} : {
|
|
1664
|
+
code: r.code,
|
|
1665
|
+
workflow: r.workflow
|
|
1666
|
+
});
|
|
1667
|
+
routesByJson.set(JSON.stringify(canonical), canonical);
|
|
1668
|
+
}
|
|
1669
|
+
const onError = [
|
|
1670
|
+
...routesByJson.keys()
|
|
1671
|
+
].sort().map((k) => routesByJson.get(k) ?? []);
|
|
1561
1672
|
const disagreements = [];
|
|
1562
1673
|
if (groups.length > 1) disagreements.push({
|
|
1563
1674
|
axis: "group",
|
|
@@ -1573,6 +1684,14 @@ function aggregateAnnouncements(descriptors) {
|
|
|
1573
1684
|
values: requires.map((r) => r.join(","))
|
|
1574
1685
|
});
|
|
1575
1686
|
}
|
|
1687
|
+
if (onError.length > 1) {
|
|
1688
|
+
disagreements.push({
|
|
1689
|
+
axis: "onError",
|
|
1690
|
+
values: [
|
|
1691
|
+
...routesByJson.keys()
|
|
1692
|
+
].sort()
|
|
1693
|
+
});
|
|
1694
|
+
}
|
|
1576
1695
|
const first = announcements[0];
|
|
1577
1696
|
if (!first) continue;
|
|
1578
1697
|
out.push({
|
|
@@ -1585,6 +1704,7 @@ function aggregateAnnouncements(descriptors) {
|
|
|
1585
1704
|
groups,
|
|
1586
1705
|
origins,
|
|
1587
1706
|
requires,
|
|
1707
|
+
onError,
|
|
1588
1708
|
runtimes: [
|
|
1589
1709
|
...new Set(announcements.map((a) => a.runtime))
|
|
1590
1710
|
].sort(),
|
|
@@ -1618,6 +1738,7 @@ function observedAnnouncements(heartbeats, covered) {
|
|
|
1618
1738
|
],
|
|
1619
1739
|
origins: [],
|
|
1620
1740
|
requires: [],
|
|
1741
|
+
onError: [],
|
|
1621
1742
|
runtimes: [],
|
|
1622
1743
|
instances: [
|
|
1623
1744
|
...instances
|
|
@@ -1976,6 +2097,13 @@ function planDispatch(requires, liveDescriptors, cp, token) {
|
|
|
1976
2097
|
__name(planDispatch, "planDispatch");
|
|
1977
2098
|
|
|
1978
2099
|
// src/interfaces.ts
|
|
2100
|
+
function checkpointNameKind(name) {
|
|
2101
|
+
if (name.startsWith("signal:child:")) return "child";
|
|
2102
|
+
if (name.startsWith("fan:")) return "folded";
|
|
2103
|
+
if (name.startsWith("spawn:")) return "spawn";
|
|
2104
|
+
return "step";
|
|
2105
|
+
}
|
|
2106
|
+
__name(checkpointNameKind, "checkpointNameKind");
|
|
1979
2107
|
var TERMINAL_RUN_STATUSES = [
|
|
1980
2108
|
"completed",
|
|
1981
2109
|
"failed",
|
|
@@ -1988,6 +2116,71 @@ function isNonEmptyRunScope(scope) {
|
|
|
1988
2116
|
}
|
|
1989
2117
|
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
1990
2118
|
|
|
2119
|
+
// src/on-error.ts
|
|
2120
|
+
var UNROUTABLE_CODES = /* @__PURE__ */ new Set([
|
|
2121
|
+
"non_determinism",
|
|
2122
|
+
"nested_workflow_call"
|
|
2123
|
+
]);
|
|
2124
|
+
var HANDOFF_SEQ_BASE = 1e9;
|
|
2125
|
+
var onErrorRunId = /* @__PURE__ */ __name((runId, attempt) => `${runId}.onError.${attempt}`, "onErrorRunId");
|
|
2126
|
+
var handoffName = /* @__PURE__ */ __name((handlerId) => `signal:child:${handlerId}`, "handoffName");
|
|
2127
|
+
var retiredHandoffName = /* @__PURE__ */ __name((handlerId) => `onError:retried:${handlerId}`, "retiredHandoffName");
|
|
2128
|
+
function handoffHandlerId(runId, cp) {
|
|
2129
|
+
const prefix = `signal:child:${runId}.onError.`;
|
|
2130
|
+
return cp.seq >= HANDOFF_SEQ_BASE && cp.name.startsWith(prefix) ? cp.name.slice("signal:child:".length) : void 0;
|
|
2131
|
+
}
|
|
2132
|
+
__name(handoffHandlerId, "handoffHandlerId");
|
|
2133
|
+
function liveHandoff(runId, checkpoints) {
|
|
2134
|
+
let live;
|
|
2135
|
+
for (const cp of checkpoints) {
|
|
2136
|
+
if (handoffHandlerId(runId, cp) !== void 0 && (!live || cp.seq > live.seq)) live = cp;
|
|
2137
|
+
}
|
|
2138
|
+
return live;
|
|
2139
|
+
}
|
|
2140
|
+
__name(liveHandoff, "liveHandoff");
|
|
2141
|
+
function routeFor(routes, error2) {
|
|
2142
|
+
if (!routes?.length || error2.code && UNROUTABLE_CODES.has(error2.code)) return void 0;
|
|
2143
|
+
return (error2.code ? routes.find((r) => r.code === error2.code) : void 0) ?? routes.find((r) => r.code === void 0);
|
|
2144
|
+
}
|
|
2145
|
+
__name(routeFor, "routeFor");
|
|
2146
|
+
function assertRoutes(workflow, routes) {
|
|
2147
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2148
|
+
for (const r of routes) {
|
|
2149
|
+
if (seen.has(r.code)) {
|
|
2150
|
+
throw new Error(r.code === void 0 ? `workflow ${workflow}: more than one onError route without a code` : `workflow ${workflow}: more than one onError route for code "${r.code}"`);
|
|
2151
|
+
}
|
|
2152
|
+
seen.add(r.code);
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2155
|
+
__name(assertRoutes, "assertRoutes");
|
|
2156
|
+
function codeOf(err) {
|
|
2157
|
+
const e = err;
|
|
2158
|
+
return [
|
|
2159
|
+
e?.code,
|
|
2160
|
+
e?.stepError?.code,
|
|
2161
|
+
e?.error?.code
|
|
2162
|
+
].find((c) => typeof c === "string");
|
|
2163
|
+
}
|
|
2164
|
+
__name(codeOf, "codeOf");
|
|
2165
|
+
function detailsOf(err) {
|
|
2166
|
+
const e = err;
|
|
2167
|
+
return e?.details ?? e?.stepError?.details ?? e?.error?.details;
|
|
2168
|
+
}
|
|
2169
|
+
__name(detailsOf, "detailsOf");
|
|
2170
|
+
function codeAndDetails(err) {
|
|
2171
|
+
const code = codeOf(err);
|
|
2172
|
+
const details = detailsOf(err);
|
|
2173
|
+
return {
|
|
2174
|
+
...code !== void 0 ? {
|
|
2175
|
+
code
|
|
2176
|
+
} : {},
|
|
2177
|
+
...details !== void 0 ? {
|
|
2178
|
+
details
|
|
2179
|
+
} : {}
|
|
2180
|
+
};
|
|
2181
|
+
}
|
|
2182
|
+
__name(codeAndDetails, "codeAndDetails");
|
|
2183
|
+
|
|
1991
2184
|
// src/tenant-group.ts
|
|
1992
2185
|
function tenantGroup(baseGroup, tenant) {
|
|
1993
2186
|
return tenant !== void 0 && tenant !== "" && tenant !== "default" ? `${baseGroup}@${tenant}` : baseGroup;
|
|
@@ -3047,7 +3240,8 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3047
3240
|
kind: "local",
|
|
3048
3241
|
error: {
|
|
3049
3242
|
message: err instanceof Error ? err.message : String(err),
|
|
3050
|
-
stack: err instanceof Error ? err.stack : void 0
|
|
3243
|
+
stack: err instanceof Error ? err.stack : void 0,
|
|
3244
|
+
...codeAndDetails(err)
|
|
3051
3245
|
},
|
|
3052
3246
|
events,
|
|
3053
3247
|
attempts: attempt,
|
|
@@ -3303,17 +3497,17 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3303
3497
|
}
|
|
3304
3498
|
]));
|
|
3305
3499
|
}, "child");
|
|
3306
|
-
const
|
|
3500
|
+
const fan = /* @__PURE__ */ __name(async (workflow2, inputs, opts, settle) => {
|
|
3307
3501
|
if (inputs.length === 0) return [];
|
|
3308
3502
|
if (stepNameOf(workflow2) !== void 0) {
|
|
3309
|
-
return allSteps(workflow2, inputs, opts);
|
|
3503
|
+
return allSteps(workflow2, inputs, opts, settle);
|
|
3310
3504
|
}
|
|
3311
|
-
const mode = opts?.mode ?? "waitAll";
|
|
3505
|
+
const mode = settle ? "waitAll" : opts?.mode ?? "waitAll";
|
|
3312
3506
|
const name = workflowName(workflow2);
|
|
3313
3507
|
const waves = allWaves(inputs.length, opts?.concurrency);
|
|
3314
3508
|
const positions = inputs.map(() => pos.next());
|
|
3315
3509
|
const firstSeq = positions[0];
|
|
3316
|
-
const group = allGroup(firstSeq, mode);
|
|
3510
|
+
const group = allGroup(firstSeq, inputs.length, mode);
|
|
3317
3511
|
const waveOf = /* @__PURE__ */ __name((i) => waves ? Math.floor(i / waves.waveSize) : void 0, "waveOf");
|
|
3318
3512
|
const id = /* @__PURE__ */ __name((i) => allChildId(runId, firstSeq, i, waveOf(i)), "id");
|
|
3319
3513
|
const { checkpoints: existing, folded } = await readFan(positions, [
|
|
@@ -3430,24 +3624,45 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3430
3624
|
throw new WorkflowSuspended(void 0, fanJoin(active.map((k) => allWait(runId, firstSeq, waves ? k : void 0))));
|
|
3431
3625
|
}
|
|
3432
3626
|
const outputs = [];
|
|
3627
|
+
const results = [];
|
|
3433
3628
|
const failures = [];
|
|
3434
3629
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
3435
3630
|
const c = existing[i]?.output;
|
|
3436
3631
|
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
3632
|
+
const { message, ...rest } = completionError({
|
|
3633
|
+
...c,
|
|
3634
|
+
error: c.error ?? "unknown"
|
|
3635
|
+
});
|
|
3636
|
+
const failure = {
|
|
3637
|
+
id: id(i),
|
|
3638
|
+
error: message,
|
|
3639
|
+
...rest
|
|
3640
|
+
};
|
|
3437
3641
|
failures.push({
|
|
3438
3642
|
index: i,
|
|
3439
|
-
|
|
3440
|
-
|
|
3643
|
+
...failure
|
|
3644
|
+
});
|
|
3645
|
+
results.push({
|
|
3646
|
+
status: "rejected",
|
|
3647
|
+
...failure
|
|
3441
3648
|
});
|
|
3442
3649
|
outputs.push(void 0);
|
|
3443
3650
|
} else {
|
|
3444
|
-
|
|
3651
|
+
const value = c?.value;
|
|
3652
|
+
results.push({
|
|
3653
|
+
status: "fulfilled",
|
|
3654
|
+
value
|
|
3655
|
+
});
|
|
3656
|
+
outputs.push(value);
|
|
3445
3657
|
}
|
|
3446
3658
|
}
|
|
3659
|
+
if (settle) return results;
|
|
3447
3660
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
3448
3661
|
return outputs;
|
|
3449
|
-
}, "
|
|
3450
|
-
const
|
|
3662
|
+
}, "fan");
|
|
3663
|
+
const all = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, false), "all");
|
|
3664
|
+
const allSettled = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, true), "allSettled");
|
|
3665
|
+
const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts, settle) => {
|
|
3451
3666
|
if (opts?.queue !== void 0) {
|
|
3452
3667
|
throw new Error("ctx.all: a flow-control queue is not supported on a step fan");
|
|
3453
3668
|
}
|
|
@@ -3458,7 +3673,8 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3458
3673
|
};
|
|
3459
3674
|
const positions = inputs.map(() => pos.next());
|
|
3460
3675
|
return host.callRemoteAll(runId, positions, def, inputs, {
|
|
3461
|
-
mode: opts?.mode ?? "waitAll",
|
|
3676
|
+
mode: settle ? "waitAll" : opts?.mode ?? "waitAll",
|
|
3677
|
+
settle,
|
|
3462
3678
|
concurrency: opts?.concurrency,
|
|
3463
3679
|
transport: opts?.transport,
|
|
3464
3680
|
admission: {
|
|
@@ -3776,6 +3992,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3776
3992
|
task: guard("task", task),
|
|
3777
3993
|
child: guard("child", child),
|
|
3778
3994
|
all: guard("all", all),
|
|
3995
|
+
allSettled: guard("allSettled", allSettled),
|
|
3779
3996
|
startChild: guard("startChild", startChild),
|
|
3780
3997
|
breakpoint: guard("breakpoint", breakpoint),
|
|
3781
3998
|
webhook: guardedWebhook,
|
|
@@ -3801,9 +4018,11 @@ function mergeTags(staticTags, runTags) {
|
|
|
3801
4018
|
__name(mergeTags, "mergeTags");
|
|
3802
4019
|
var BREAKPOINT = "breakpoint";
|
|
3803
4020
|
var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
|
|
4021
|
+
var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
3804
4022
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
3805
4023
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
3806
4024
|
var STEP_FAN_CHUNK = 200;
|
|
4025
|
+
var RESULT_BATCH_MAX = 256;
|
|
3807
4026
|
function shuffle(items) {
|
|
3808
4027
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
3809
4028
|
const j = Math.floor(Math.random() * (i + 1));
|
|
@@ -3911,6 +4130,8 @@ var WorkflowEngine = class {
|
|
|
3911
4130
|
updateValidators = /* @__PURE__ */ new Map();
|
|
3912
4131
|
/** Runs being cancelled WITH saga compensation — see `cancel({ compensate: true })`. */
|
|
3913
4132
|
cancelRequested = /* @__PURE__ */ new Set();
|
|
4133
|
+
/** {@link announcedRoutes} reads, by group + workflow + version. */
|
|
4134
|
+
announcedRoutesCache = /* @__PURE__ */ new Map();
|
|
3914
4135
|
/**
|
|
3915
4136
|
* Seqs of the dispatched remote steps a turn suspended on, by run, while that turn executes on
|
|
3916
4137
|
* THIS instance — so the turn can re-check them once its lease is released (see
|
|
@@ -3932,6 +4153,8 @@ var WorkflowEngine = class {
|
|
|
3932
4153
|
stepFansFiredHere = /* @__PURE__ */ new Map();
|
|
3933
4154
|
/** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
|
|
3934
4155
|
resultChains = /* @__PURE__ */ new Map();
|
|
4156
|
+
/** Each run's results waiting to be settled on this instance — see {@link completeRemoteResult}. */
|
|
4157
|
+
resultQueues = /* @__PURE__ */ new Map();
|
|
3935
4158
|
/** Flow-control admission backend for remote steps (see {@link registerQueue}). */
|
|
3936
4159
|
admission;
|
|
3937
4160
|
/** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
|
|
@@ -4138,6 +4361,7 @@ var WorkflowEngine = class {
|
|
|
4138
4361
|
name,
|
|
4139
4362
|
version,
|
|
4140
4363
|
fn,
|
|
4364
|
+
onError: this.checkedRoutes(name, opts?.onError),
|
|
4141
4365
|
tags: opts?.tags,
|
|
4142
4366
|
singleton: opts?.singleton,
|
|
4143
4367
|
concurrency: opts?.concurrency,
|
|
@@ -4207,6 +4431,7 @@ var WorkflowEngine = class {
|
|
|
4207
4431
|
executionTimeoutMs: opts.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
|
|
4208
4432
|
validateInput: opts.validateInput,
|
|
4209
4433
|
requires: opts.requires,
|
|
4434
|
+
onError: this.checkedRoutes(name, opts.onError),
|
|
4210
4435
|
remote: {
|
|
4211
4436
|
group: opts.group,
|
|
4212
4437
|
executor: opts.executor
|
|
@@ -4984,6 +5209,24 @@ var WorkflowEngine = class {
|
|
|
4984
5209
|
if (resurrecting) {
|
|
4985
5210
|
const isFailureCompletion = /* @__PURE__ */ __name((v) => typeof v === "object" && v !== null && v.ok === false, "isFailureCompletion");
|
|
4986
5211
|
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
5212
|
+
const handlerId = handoffHandlerId(runId, cp);
|
|
5213
|
+
if (handlerId !== void 0) {
|
|
5214
|
+
await this.store.takeSignalWaiter(`child:${handlerId}`);
|
|
5215
|
+
await this.store.saveCheckpoint({
|
|
5216
|
+
...cp,
|
|
5217
|
+
name: retiredHandoffName(handlerId),
|
|
5218
|
+
status: "completed"
|
|
5219
|
+
});
|
|
5220
|
+
const tags = run.tags?.filter((t) => !t.startsWith("onError:"));
|
|
5221
|
+
if (tags && tags.length !== run.tags?.length) {
|
|
5222
|
+
await this.store.updateRun(runId, {
|
|
5223
|
+
tags,
|
|
5224
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
5225
|
+
});
|
|
5226
|
+
run.tags = tags;
|
|
5227
|
+
}
|
|
5228
|
+
continue;
|
|
5229
|
+
}
|
|
4987
5230
|
if (cp.status === "failed") {
|
|
4988
5231
|
await this.store.saveCheckpoint({
|
|
4989
5232
|
...cp,
|
|
@@ -5128,13 +5371,13 @@ var WorkflowEngine = class {
|
|
|
5128
5371
|
* `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
|
|
5129
5372
|
*/
|
|
5130
5373
|
waitForRun(runId, opts) {
|
|
5131
|
-
const
|
|
5374
|
+
const TERMINAL2 = /* @__PURE__ */ new Set([
|
|
5132
5375
|
"completed",
|
|
5133
5376
|
"failed",
|
|
5134
5377
|
"cancelled",
|
|
5135
5378
|
"dead"
|
|
5136
5379
|
]);
|
|
5137
|
-
const isSettled = /* @__PURE__ */ __name((run) => opts?.until === "terminal" ?
|
|
5380
|
+
const isSettled = /* @__PURE__ */ __name((run) => opts?.until === "terminal" ? TERMINAL2.has(run.status) : run.status !== "pending" && run.status !== "running" && run.status !== "cancelling" && !(run.status === "suspended" && run.awaitingDecisionTaskId !== void 0), "isSettled");
|
|
5138
5381
|
const toResult = /* @__PURE__ */ __name((run) => ({
|
|
5139
5382
|
runId,
|
|
5140
5383
|
status: run.status,
|
|
@@ -5389,10 +5632,7 @@ var WorkflowEngine = class {
|
|
|
5389
5632
|
async redriveSettledStepFans(runId, fans) {
|
|
5390
5633
|
for (const fan of fans) {
|
|
5391
5634
|
if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
|
|
5392
|
-
|
|
5393
|
-
"pending"
|
|
5394
|
-
]) : (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= fan.from && c.seq <= fan.to && c.status === "pending").length;
|
|
5395
|
-
if (pending > 0) continue;
|
|
5635
|
+
if (await this.stepFanPending(runId, fan.from, fan.to) > 0) continue;
|
|
5396
5636
|
const nowMs = this.clock();
|
|
5397
5637
|
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5398
5638
|
return;
|
|
@@ -6102,6 +6342,209 @@ var WorkflowEngine = class {
|
|
|
6102
6342
|
* us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
|
|
6103
6343
|
*/
|
|
6104
6344
|
/**
|
|
6345
|
+
* The registered onError routes whose handler workflow nothing can run: not registered on this
|
|
6346
|
+
* engine, and no live worker group serves it. Meant for a boot-time warning: a typo in a handler's
|
|
6347
|
+
* name would otherwise surface only when the first run fails.
|
|
6348
|
+
*/
|
|
6349
|
+
async unservedOnErrorRoutes() {
|
|
6350
|
+
const unserved = [];
|
|
6351
|
+
let groups;
|
|
6352
|
+
for (const registered of this.workflows.values()) {
|
|
6353
|
+
for (const route of registered.onError ?? []) {
|
|
6354
|
+
if (this.latest.has(route.workflow)) continue;
|
|
6355
|
+
groups ??= await this.pool.listWorkerGroups().catch(() => []);
|
|
6356
|
+
const token = sanitizeQueueToken(route.workflow);
|
|
6357
|
+
if (groups.some((g) => g === token || g.startsWith(`${token}@`))) continue;
|
|
6358
|
+
unserved.push({
|
|
6359
|
+
workflow: registered.name,
|
|
6360
|
+
route
|
|
6361
|
+
});
|
|
6362
|
+
}
|
|
6363
|
+
}
|
|
6364
|
+
return unserved;
|
|
6365
|
+
}
|
|
6366
|
+
/** Validate a workflow's {@link OnErrorRoute}s at registration. */
|
|
6367
|
+
checkedRoutes(name, routes) {
|
|
6368
|
+
if (!routes?.length) return void 0;
|
|
6369
|
+
assertRoutes(name, routes);
|
|
6370
|
+
return routes;
|
|
6371
|
+
}
|
|
6372
|
+
/**
|
|
6373
|
+
* Route a failed run to its {@link OnErrorRoute} handler, read from the workflow's local registration
|
|
6374
|
+
* or, for one served by convention, from what its live workers announce. Starts the handler run and
|
|
6375
|
+
* waits for it the way a parent waits for a `ctx.child`: a `child:<handlerId>` waiter plus a
|
|
6376
|
+
* `running` placeholder checkpoint carrying the handler's name and input, so the handler's terminal
|
|
6377
|
+
* state wakes the run (buffered and reconciled like any child). From then on every execution of the
|
|
6378
|
+
* run only follows the handler ({@link followHandoff}); the body never runs again. Undefined when no
|
|
6379
|
+
* route matches: the run fails as usual.
|
|
6380
|
+
*
|
|
6381
|
+
* The placeholder sits past every seq the body used and is left out of {@link remoteHistory}.
|
|
6382
|
+
*/
|
|
6383
|
+
async routeFailure(run, error2) {
|
|
6384
|
+
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
|
|
6385
|
+
const routes = registered ? registered.onError : await this.announcedRoutes(run, error2);
|
|
6386
|
+
const route = routeFor(routes, error2);
|
|
6387
|
+
if (!route) return void 0;
|
|
6388
|
+
const checkpoints = await this.handoffCheckpoints(run.id);
|
|
6389
|
+
const live = liveHandoff(run.id, checkpoints);
|
|
6390
|
+
if (live) return this.followHandoff(run, live);
|
|
6391
|
+
const attempt = checkpoints.length;
|
|
6392
|
+
const seq = HANDOFF_SEQ_BASE + attempt;
|
|
6393
|
+
const handlerId = onErrorRunId(run.id, attempt);
|
|
6394
|
+
const handler = {
|
|
6395
|
+
workflow: route.workflow,
|
|
6396
|
+
input: {
|
|
6397
|
+
runId: run.id,
|
|
6398
|
+
workflow: run.workflow,
|
|
6399
|
+
input: run.input,
|
|
6400
|
+
error: error2
|
|
6401
|
+
}
|
|
6402
|
+
};
|
|
6403
|
+
await this.store.putSignalWaiter({
|
|
6404
|
+
token: `child:${handlerId}`,
|
|
6405
|
+
runId: run.id,
|
|
6406
|
+
seq
|
|
6407
|
+
});
|
|
6408
|
+
await this.store.saveCheckpoint({
|
|
6409
|
+
...instantCheckpoint({
|
|
6410
|
+
runId: run.id,
|
|
6411
|
+
seq,
|
|
6412
|
+
name: handoffName(handlerId),
|
|
6413
|
+
kind: "signal",
|
|
6414
|
+
status: "running"
|
|
6415
|
+
}),
|
|
6416
|
+
input: handler
|
|
6417
|
+
});
|
|
6418
|
+
const tag = `onError:${route.code ?? "*"}`;
|
|
6419
|
+
if (!run.tags?.includes(tag)) {
|
|
6420
|
+
const tags = [
|
|
6421
|
+
...run.tags ?? [],
|
|
6422
|
+
tag
|
|
6423
|
+
];
|
|
6424
|
+
await this.store.updateRun(run.id, {
|
|
6425
|
+
tags,
|
|
6426
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
6427
|
+
});
|
|
6428
|
+
run.tags = tags;
|
|
6429
|
+
}
|
|
6430
|
+
this.emit({
|
|
6431
|
+
type: "run.routed",
|
|
6432
|
+
runId: run.id,
|
|
6433
|
+
workflow: run.workflow,
|
|
6434
|
+
namespace: run.namespace,
|
|
6435
|
+
name: route.workflow,
|
|
6436
|
+
error: error2
|
|
6437
|
+
});
|
|
6438
|
+
this.startChildDeferred(handler.workflow, handler.input, handlerId, {
|
|
6439
|
+
namespace: run.namespace
|
|
6440
|
+
});
|
|
6441
|
+
return {
|
|
6442
|
+
kind: "suspended"
|
|
6443
|
+
};
|
|
6444
|
+
}
|
|
6445
|
+
/** A run's handoff checkpoints, live and retired: the only ones at {@link HANDOFF_SEQ_BASE} and up. */
|
|
6446
|
+
async handoffCheckpoints(runId) {
|
|
6447
|
+
const handoffs = await this.store.listCheckpointsByNamePrefix?.(runId, [
|
|
6448
|
+
`signal:child:${runId}.onError.`,
|
|
6449
|
+
`onError:retried:${runId}.onError.`
|
|
6450
|
+
]);
|
|
6451
|
+
return (handoffs ?? await this.store.listCheckpoints(runId)).filter((cp) => cp.seq >= HANDOFF_SEQ_BASE);
|
|
6452
|
+
}
|
|
6453
|
+
/**
|
|
6454
|
+
* Where a run that handed its failure off stands: the handler's outcome once it settled (from the
|
|
6455
|
+
* wake's checkpoint, or off the handler run when the wake never arrived), else still waiting. Starts
|
|
6456
|
+
* the handler if the turn that handed off crashed before its deferred start ran.
|
|
6457
|
+
*/
|
|
6458
|
+
async followHandoff(run, handoff) {
|
|
6459
|
+
if (handoff.status === "completed") {
|
|
6460
|
+
const outcome = handoff.output;
|
|
6461
|
+
return outcome.ok ? {
|
|
6462
|
+
kind: "completed",
|
|
6463
|
+
output: outcome.value
|
|
6464
|
+
} : {
|
|
6465
|
+
kind: "failed",
|
|
6466
|
+
error: completionError(outcome)
|
|
6467
|
+
};
|
|
6468
|
+
}
|
|
6469
|
+
const handlerId = handoffHandlerId(run.id, handoff);
|
|
6470
|
+
const handlerRun = await this.store.getRun(handlerId);
|
|
6471
|
+
if (handlerRun?.status === "completed") return {
|
|
6472
|
+
kind: "completed",
|
|
6473
|
+
output: handlerRun.output
|
|
6474
|
+
};
|
|
6475
|
+
if (handlerRun && TERMINAL_RUN_STATUSES.includes(handlerRun.status)) {
|
|
6476
|
+
return {
|
|
6477
|
+
kind: "failed",
|
|
6478
|
+
error: handlerRun.error ?? {
|
|
6479
|
+
message: `onError handler ${handlerId} ${handlerRun.status}`
|
|
6480
|
+
}
|
|
6481
|
+
};
|
|
6482
|
+
}
|
|
6483
|
+
if (!handlerRun) {
|
|
6484
|
+
const handler = handoff.input;
|
|
6485
|
+
this.startChildDeferred(handler.workflow, handler.input, handlerId, {
|
|
6486
|
+
namespace: run.namespace
|
|
6487
|
+
});
|
|
6488
|
+
}
|
|
6489
|
+
return {
|
|
6490
|
+
kind: "suspended"
|
|
6491
|
+
};
|
|
6492
|
+
}
|
|
6493
|
+
/**
|
|
6494
|
+
* The onError routes the live workers announce for a workflow the engine serves by convention: the
|
|
6495
|
+
* one set they agree on, or none when they state none or cannot be read. Workers that announce
|
|
6496
|
+
* different sets (a rollout in progress) route nothing, and the run's failure is reported as a
|
|
6497
|
+
* `run.unrouted` event. Announcers of another version of the workflow are ignored. The read is
|
|
6498
|
+
* cached for {@link ANNOUNCED_ROUTES_TTL_MS}, so a burst of failures costs one descriptor read.
|
|
6499
|
+
*/
|
|
6500
|
+
async announcedRoutes(run, error2) {
|
|
6501
|
+
const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
|
|
6502
|
+
const key = `${group}
|
|
6503
|
+
${run.workflow}
|
|
6504
|
+
${run.workflowVersion}`;
|
|
6505
|
+
const now = this.clock();
|
|
6506
|
+
let cached = this.announcedRoutesCache.get(key);
|
|
6507
|
+
if (!cached || now - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
|
|
6508
|
+
cached = {
|
|
6509
|
+
at: now,
|
|
6510
|
+
sets: await this.readAnnouncedRoutes(run, group)
|
|
6511
|
+
};
|
|
6512
|
+
this.announcedRoutesCache.set(key, cached);
|
|
6513
|
+
}
|
|
6514
|
+
if (cached.sets.length > 1) {
|
|
6515
|
+
this.emit({
|
|
6516
|
+
type: "run.unrouted",
|
|
6517
|
+
runId: run.id,
|
|
6518
|
+
workflow: run.workflow,
|
|
6519
|
+
namespace: run.namespace,
|
|
6520
|
+
error: error2,
|
|
6521
|
+
reason: `live workers announce ${cached.sets.length} different onError route sets`
|
|
6522
|
+
});
|
|
6523
|
+
return void 0;
|
|
6524
|
+
}
|
|
6525
|
+
return cached.sets[0];
|
|
6526
|
+
}
|
|
6527
|
+
/** Every distinct onError route set the live workers of `group` announce for the run's workflow. */
|
|
6528
|
+
async readAnnouncedRoutes(run, group) {
|
|
6529
|
+
const transport = await this.pool.transportWithLiveGroup(group);
|
|
6530
|
+
if (!transport?.readWorkerDescriptors) return [];
|
|
6531
|
+
let descriptors;
|
|
6532
|
+
try {
|
|
6533
|
+
descriptors = await transport.readWorkerDescriptors(group);
|
|
6534
|
+
} catch {
|
|
6535
|
+
return [];
|
|
6536
|
+
}
|
|
6537
|
+
const sets = /* @__PURE__ */ new Map();
|
|
6538
|
+
for (const announced of aggregateAnnouncements(descriptors)) {
|
|
6539
|
+
if (announced.name !== run.workflow) continue;
|
|
6540
|
+
if (announced.version !== void 0 && announced.version !== run.workflowVersion) continue;
|
|
6541
|
+
for (const routes of announced.onError) sets.set(JSON.stringify(routes), routes);
|
|
6542
|
+
}
|
|
6543
|
+
return [
|
|
6544
|
+
...sets.values()
|
|
6545
|
+
];
|
|
6546
|
+
}
|
|
6547
|
+
/**
|
|
6105
6548
|
* Apply a terminal/suspended transition once: persist the new run status, emit the matching
|
|
6106
6549
|
* lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
|
|
6107
6550
|
* executors funnel their completed/failed/suspended outcomes through here so a status update can
|
|
@@ -6135,9 +6578,12 @@ var WorkflowEngine = class {
|
|
|
6135
6578
|
};
|
|
6136
6579
|
}
|
|
6137
6580
|
if (outcome.kind === "failed") {
|
|
6581
|
+
const handed = await this.routeFailure(run, outcome.error);
|
|
6582
|
+
if (handed && handed.kind !== "failed") return this.settleRun(run, handed);
|
|
6583
|
+
const error2 = handed?.error ?? outcome.error;
|
|
6138
6584
|
await this.store.updateRun(run.id, {
|
|
6139
6585
|
status: "failed",
|
|
6140
|
-
error:
|
|
6586
|
+
error: error2,
|
|
6141
6587
|
updatedAt
|
|
6142
6588
|
});
|
|
6143
6589
|
this.emit({
|
|
@@ -6145,16 +6591,13 @@ var WorkflowEngine = class {
|
|
|
6145
6591
|
runId: run.id,
|
|
6146
6592
|
workflow: run.workflow,
|
|
6147
6593
|
namespace: run.namespace,
|
|
6148
|
-
error:
|
|
6149
|
-
});
|
|
6150
|
-
void this.notifyParent(run.id, {
|
|
6151
|
-
ok: false,
|
|
6152
|
-
error: outcome.error.message
|
|
6594
|
+
error: error2
|
|
6153
6595
|
});
|
|
6596
|
+
void this.notifyParent(run.id, failedCompletion(error2));
|
|
6154
6597
|
return {
|
|
6155
6598
|
runId: run.id,
|
|
6156
6599
|
status: "failed",
|
|
6157
|
-
error:
|
|
6600
|
+
error: error2
|
|
6158
6601
|
};
|
|
6159
6602
|
}
|
|
6160
6603
|
if (outcome.kind === "blocked") return this.parkBlocked(run, outcome.blocked);
|
|
@@ -6243,7 +6686,14 @@ var WorkflowEngine = class {
|
|
|
6243
6686
|
}
|
|
6244
6687
|
const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
|
|
6245
6688
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
6246
|
-
const
|
|
6689
|
+
const checkpoints = await this.store.listCheckpoints(run.id);
|
|
6690
|
+
const handoff = liveHandoff(run.id, checkpoints);
|
|
6691
|
+
if (handoff) {
|
|
6692
|
+
const result = await this.settleRun(run, await this.followHandoff(run, handoff));
|
|
6693
|
+
await this.store.releaseRunLock(run.id);
|
|
6694
|
+
return result;
|
|
6695
|
+
}
|
|
6696
|
+
const history = await this.remoteHistory(run.id, checkpoints);
|
|
6247
6697
|
if (remote.executor.dispatch) {
|
|
6248
6698
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
6249
6699
|
const silenceMs = this.remoteAdvanceSilenceMs;
|
|
@@ -6567,8 +7017,8 @@ var WorkflowEngine = class {
|
|
|
6567
7017
|
}
|
|
6568
7018
|
}
|
|
6569
7019
|
/** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
|
|
6570
|
-
async remoteHistory(runId) {
|
|
6571
|
-
const checkpoints = await this.store.listCheckpoints(runId);
|
|
7020
|
+
async remoteHistory(runId, listed) {
|
|
7021
|
+
const checkpoints = listed ?? await this.store.listCheckpoints(runId);
|
|
6572
7022
|
const kindOf = {
|
|
6573
7023
|
remote: "call",
|
|
6574
7024
|
local: "step",
|
|
@@ -6578,15 +7028,14 @@ var WorkflowEngine = class {
|
|
|
6578
7028
|
const events = [];
|
|
6579
7029
|
for (const cp of checkpoints) {
|
|
6580
7030
|
if (cp.status === "completed" || cp.status === "failed") {
|
|
7031
|
+
if (cp.seq >= HANDOFF_SEQ_BASE) continue;
|
|
6581
7032
|
if (cp.kind === "signal" && cp.name.startsWith("signal:child:")) {
|
|
6582
7033
|
const completion = cp.output;
|
|
6583
7034
|
events.push({
|
|
6584
7035
|
seq: cp.seq,
|
|
6585
7036
|
kind: "child",
|
|
6586
7037
|
output: completion?.ok ? completion.value : void 0,
|
|
6587
|
-
error: completion && completion.ok === false ?
|
|
6588
|
-
message: completion.error
|
|
6589
|
-
} : void 0
|
|
7038
|
+
error: completion && completion.ok === false ? completionError(completion) : void 0
|
|
6590
7039
|
});
|
|
6591
7040
|
continue;
|
|
6592
7041
|
}
|
|
@@ -6867,6 +7316,10 @@ var WorkflowEngine = class {
|
|
|
6867
7316
|
this.awaitedStepFans.set(run.id, awaitedFans);
|
|
6868
7317
|
const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
|
|
6869
7318
|
try {
|
|
7319
|
+
const handoff = liveHandoff(run.id, snapshot);
|
|
7320
|
+
if (handoff && !this.cancelRequested.has(run.id)) {
|
|
7321
|
+
return await this.settleRun(run, await this.followHandoff(run, handoff));
|
|
7322
|
+
}
|
|
6870
7323
|
const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
|
|
6871
7324
|
if (this.cancelRequested.has(run.id)) {
|
|
6872
7325
|
return await this.compensateAndCancel(run, compensations, replay);
|
|
@@ -6929,7 +7382,8 @@ var WorkflowEngine = class {
|
|
|
6929
7382
|
}
|
|
6930
7383
|
const error2 = {
|
|
6931
7384
|
message: err instanceof Error ? err.message : String(err),
|
|
6932
|
-
stack: err instanceof Error ? err.stack : void 0
|
|
7385
|
+
stack: err instanceof Error ? err.stack : void 0,
|
|
7386
|
+
...codeAndDetails(err)
|
|
6933
7387
|
};
|
|
6934
7388
|
const pending = await this.unwindCompensations(run, compensations, replay);
|
|
6935
7389
|
if (pending) return await this.settleRun(run, {
|
|
@@ -7565,7 +8019,8 @@ var WorkflowEngine = class {
|
|
|
7565
8019
|
const failure = {
|
|
7566
8020
|
index: i,
|
|
7567
8021
|
id: stepId(runId, seq),
|
|
7568
|
-
error: err.message
|
|
8022
|
+
error: err.message,
|
|
8023
|
+
...codeAndDetails(err)
|
|
7569
8024
|
};
|
|
7570
8025
|
if (opts.mode === "failFast") throw new GatherError([
|
|
7571
8026
|
failure
|
|
@@ -7587,6 +8042,24 @@ var WorkflowEngine = class {
|
|
|
7587
8042
|
});
|
|
7588
8043
|
throw new WorkflowSuspended(wakeAt);
|
|
7589
8044
|
}
|
|
8045
|
+
if (opts.settle) {
|
|
8046
|
+
const failed = new Map(failures.map((f) => [
|
|
8047
|
+
f.index,
|
|
8048
|
+
f
|
|
8049
|
+
]));
|
|
8050
|
+
return Array.from(outputs, (value, i) => {
|
|
8051
|
+
const f = failed.get(i);
|
|
8052
|
+
if (!f) return {
|
|
8053
|
+
status: "fulfilled",
|
|
8054
|
+
value
|
|
8055
|
+
};
|
|
8056
|
+
const { index: _index, ...rejected } = f;
|
|
8057
|
+
return {
|
|
8058
|
+
status: "rejected",
|
|
8059
|
+
...rejected
|
|
8060
|
+
};
|
|
8061
|
+
});
|
|
8062
|
+
}
|
|
7590
8063
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
7591
8064
|
return outputs;
|
|
7592
8065
|
}
|
|
@@ -7658,6 +8131,25 @@ var WorkflowEngine = class {
|
|
|
7658
8131
|
await this.store.saveCheckpoint(checkpoint);
|
|
7659
8132
|
return true;
|
|
7660
8133
|
}
|
|
8134
|
+
/**
|
|
8135
|
+
* {@link savePendingCheckpoint} for checkpoints at distinct seqs of one run: one
|
|
8136
|
+
* {@link StateStore.saveCheckpointsIfPending} call when the store has it, else one write each.
|
|
8137
|
+
* Never rejects as a whole: a write that failed is reported as that entry's error, and the batch
|
|
8138
|
+
* call rejecting means it wrote nothing, so the entries are then written one by one.
|
|
8139
|
+
*/
|
|
8140
|
+
async savePendingCheckpoints(checkpoints) {
|
|
8141
|
+
if (checkpoints.length > 1 && this.store.saveCheckpointsIfPending) {
|
|
8142
|
+
try {
|
|
8143
|
+
const wrote = await this.store.saveCheckpointsIfPending(checkpoints);
|
|
8144
|
+
return checkpoints.map((_, i) => wrote[i] === true);
|
|
8145
|
+
} catch {
|
|
8146
|
+
}
|
|
8147
|
+
}
|
|
8148
|
+
const settled = await Promise.allSettled(checkpoints.map((cp) => this.savePendingCheckpoint(cp)));
|
|
8149
|
+
return settled.map((s) => s.status === "fulfilled" ? s.value : {
|
|
8150
|
+
error: s.reason
|
|
8151
|
+
});
|
|
8152
|
+
}
|
|
7661
8153
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
7662
8154
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
7663
8155
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -7696,57 +8188,161 @@ var WorkflowEngine = class {
|
|
|
7696
8188
|
* result (the dispatching one may be gone), so the run is crash/scale-safe. Resolves whether the
|
|
7697
8189
|
* run should now be resumed: not for a duplicate or late delivery (the checkpoint isn't `pending`),
|
|
7698
8190
|
* a run already finished, or a step fan whose barrier is still closed.
|
|
8191
|
+
*
|
|
8192
|
+
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
8193
|
+
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
8194
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked once
|
|
8195
|
+
* per batch, not once per result. A result resolves only once its own write has committed, so the
|
|
8196
|
+
* transport acknowledges nothing that is not durable yet.
|
|
7699
8197
|
*/
|
|
7700
|
-
|
|
7701
|
-
|
|
7702
|
-
|
|
7703
|
-
|
|
7704
|
-
|
|
7705
|
-
|
|
8198
|
+
completeRemoteResult(result) {
|
|
8199
|
+
return new Promise((resolve, reject) => {
|
|
8200
|
+
const entry = {
|
|
8201
|
+
result,
|
|
8202
|
+
resolve,
|
|
8203
|
+
reject
|
|
8204
|
+
};
|
|
8205
|
+
const queued = this.resultQueues.get(result.runId);
|
|
8206
|
+
if (queued) {
|
|
8207
|
+
queued.push(entry);
|
|
8208
|
+
return;
|
|
8209
|
+
}
|
|
8210
|
+
const queue = [
|
|
8211
|
+
entry
|
|
8212
|
+
];
|
|
8213
|
+
this.resultQueues.set(result.runId, queue);
|
|
8214
|
+
void this.drainResultQueue(result.runId, queue);
|
|
8215
|
+
});
|
|
8216
|
+
}
|
|
8217
|
+
/** Settle `queue` (a run's queued results) batch by batch until it is empty, then retire it. */
|
|
8218
|
+
async drainResultQueue(runId, queue) {
|
|
8219
|
+
await Promise.resolve();
|
|
8220
|
+
while (queue.length > 0) {
|
|
8221
|
+
const batch = queue.splice(0, RESULT_BATCH_MAX);
|
|
8222
|
+
let outcomes;
|
|
8223
|
+
try {
|
|
8224
|
+
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
|
|
8225
|
+
} catch (err) {
|
|
8226
|
+
outcomes = batch.map(() => ({
|
|
8227
|
+
error: err
|
|
8228
|
+
}));
|
|
8229
|
+
}
|
|
8230
|
+
batch.forEach((q, i) => {
|
|
8231
|
+
const outcome = outcomes[i] ?? false;
|
|
8232
|
+
if (typeof outcome === "boolean") q.resolve(outcome);
|
|
8233
|
+
else q.reject(outcome.error);
|
|
8234
|
+
});
|
|
8235
|
+
}
|
|
8236
|
+
this.resultQueues.delete(runId);
|
|
8237
|
+
}
|
|
8238
|
+
/**
|
|
8239
|
+
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
8240
|
+
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
8241
|
+
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
8242
|
+
*/
|
|
8243
|
+
async completeRemoteResults(runId, results) {
|
|
8244
|
+
const outcomes = results.map(() => false);
|
|
8245
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8246
|
+
const live = results.flatMap((r, i) => {
|
|
8247
|
+
if (seen.has(r.seq)) return [];
|
|
8248
|
+
seen.add(r.seq);
|
|
8249
|
+
return [
|
|
8250
|
+
i
|
|
8251
|
+
];
|
|
8252
|
+
});
|
|
8253
|
+
const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
|
|
8254
|
+
const pending = [];
|
|
8255
|
+
reads.forEach((read, k) => {
|
|
8256
|
+
const index = live[k];
|
|
8257
|
+
if (read.status === "rejected") outcomes[index] = {
|
|
8258
|
+
error: read.reason
|
|
8259
|
+
};
|
|
8260
|
+
else if (read.value?.status === "pending") pending.push({
|
|
8261
|
+
index,
|
|
8262
|
+
cp: read.value
|
|
8263
|
+
});
|
|
8264
|
+
});
|
|
8265
|
+
if (pending.length === 0) return outcomes;
|
|
8266
|
+
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
8267
|
+
const run = await this.store.getRun(runId);
|
|
8268
|
+
if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
|
|
7706
8269
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
7707
|
-
const
|
|
7708
|
-
|
|
7709
|
-
|
|
7710
|
-
|
|
7711
|
-
|
|
7712
|
-
|
|
7713
|
-
|
|
7714
|
-
|
|
7715
|
-
|
|
8270
|
+
const writes = pending.map(({ index, cp }) => {
|
|
8271
|
+
const result = results[index];
|
|
8272
|
+
return {
|
|
8273
|
+
...cp,
|
|
8274
|
+
status: result.status,
|
|
8275
|
+
output: result.status === "completed" ? result.output : cp.output,
|
|
8276
|
+
error: result.error,
|
|
8277
|
+
events: result.events ?? cp.events,
|
|
8278
|
+
startedAt: result.startedAt ? new Date(result.startedAt) : cp.startedAt,
|
|
8279
|
+
finishedAt
|
|
8280
|
+
};
|
|
7716
8281
|
});
|
|
7717
|
-
|
|
7718
|
-
|
|
7719
|
-
|
|
7720
|
-
|
|
7721
|
-
|
|
7722
|
-
|
|
7723
|
-
|
|
7724
|
-
|
|
7725
|
-
|
|
7726
|
-
|
|
7727
|
-
queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
7728
|
-
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
8282
|
+
const claims = await this.savePendingCheckpoints(writes);
|
|
8283
|
+
const settled = [];
|
|
8284
|
+
claims.forEach((claim, k) => {
|
|
8285
|
+
const { index, cp } = pending[k];
|
|
8286
|
+
if (claim === true) settled.push({
|
|
8287
|
+
index,
|
|
8288
|
+
cp,
|
|
8289
|
+
result: results[index]
|
|
8290
|
+
});
|
|
8291
|
+
else if (claim !== false) outcomes[index] = claim;
|
|
7729
8292
|
});
|
|
7730
|
-
|
|
8293
|
+
for (const { cp, result } of settled) {
|
|
8294
|
+
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
8295
|
+
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
8296
|
+
this.emit({
|
|
8297
|
+
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
8298
|
+
runId,
|
|
8299
|
+
seq: result.seq,
|
|
8300
|
+
name: cp.name,
|
|
8301
|
+
kind: cp.kind,
|
|
8302
|
+
output: result.output,
|
|
8303
|
+
error: result.error,
|
|
8304
|
+
queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
8305
|
+
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
8306
|
+
});
|
|
8307
|
+
}
|
|
8308
|
+
const barriers = /* @__PURE__ */ new Map();
|
|
8309
|
+
let carrier;
|
|
8310
|
+
for (const { index, cp, result } of settled) {
|
|
8311
|
+
if (!await this.stepFanStillRunning(cp, result, barriers)) carrier = index;
|
|
8312
|
+
}
|
|
8313
|
+
if (carrier !== void 0) outcomes[carrier] = true;
|
|
8314
|
+
return outcomes;
|
|
7731
8315
|
}
|
|
7732
8316
|
/**
|
|
7733
8317
|
* The join barrier of a step fan: a result for one of its steps resumes the parent only once no
|
|
7734
8318
|
* step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
|
|
7735
8319
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
7736
|
-
* which makes a fan of N steps cost O(N²) checkpoint reads.
|
|
8320
|
+
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
8321
|
+
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
8322
|
+
* re-suspends, a lost one strands the run.
|
|
7737
8323
|
*/
|
|
7738
|
-
async stepFanStillRunning(cp, result) {
|
|
8324
|
+
async stepFanStillRunning(cp, result, barriers) {
|
|
7739
8325
|
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
7740
8326
|
if (!fan) return false;
|
|
7741
8327
|
if (result.status === "failed") return false;
|
|
7742
|
-
|
|
7743
|
-
|
|
7744
|
-
|
|
7745
|
-
|
|
7746
|
-
|
|
8328
|
+
let closed = barriers.get(fan.firstSeq);
|
|
8329
|
+
if (!closed) {
|
|
8330
|
+
closed = this.stepFanPending(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1).then((pending) => pending > 0, () => false);
|
|
8331
|
+
barriers.set(fan.firstSeq, closed);
|
|
8332
|
+
}
|
|
8333
|
+
if (await closed) return true;
|
|
7747
8334
|
this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
|
|
7748
8335
|
return false;
|
|
7749
8336
|
}
|
|
8337
|
+
/** How many steps of a step fan's seq range are still `pending`. */
|
|
8338
|
+
async stepFanPending(runId, from, to) {
|
|
8339
|
+
if (this.store.countCheckpoints) {
|
|
8340
|
+
return this.store.countCheckpoints(runId, from, to, [
|
|
8341
|
+
"pending"
|
|
8342
|
+
]);
|
|
8343
|
+
}
|
|
8344
|
+
return (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= from && c.seq <= to && c.status === "pending").length;
|
|
8345
|
+
}
|
|
7750
8346
|
/**
|
|
7751
8347
|
* Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
|
|
7752
8348
|
* it over the broker (the dispatching one may be gone, or a point-to-point broker handed it to a
|
|
@@ -8098,6 +8694,435 @@ var RunGateway = class {
|
|
|
8098
8694
|
}
|
|
8099
8695
|
};
|
|
8100
8696
|
|
|
8697
|
+
// src/run-detail-summary.ts
|
|
8698
|
+
var FAN_SUMMARY_FAILURES = 20;
|
|
8699
|
+
var SUMMARY_CHILD_LIMIT = 200;
|
|
8700
|
+
var FAN_PAGE_MAX = 500;
|
|
8701
|
+
function foldedCount(cp) {
|
|
8702
|
+
if (!cp.name.startsWith(FOLDED_PREFIX)) return void 0;
|
|
8703
|
+
const values = cp.output?.values;
|
|
8704
|
+
return Array.isArray(values) ? values.length : 0;
|
|
8705
|
+
}
|
|
8706
|
+
__name(foldedCount, "foldedCount");
|
|
8707
|
+
function isFailedMember(cp) {
|
|
8708
|
+
if (cp.status === "failed") return true;
|
|
8709
|
+
if (!cp.name.startsWith("signal:child:") || cp.status !== "completed") return false;
|
|
8710
|
+
return cp.output?.ok === false;
|
|
8711
|
+
}
|
|
8712
|
+
__name(isFailedMember, "isFailedMember");
|
|
8713
|
+
function fanLabel(members) {
|
|
8714
|
+
if (members.some((cp) => cp.name.startsWith("signal:child:") || foldedCount(cp) !== void 0)) {
|
|
8715
|
+
return "child runs";
|
|
8716
|
+
}
|
|
8717
|
+
const first = members[0]?.name ?? "";
|
|
8718
|
+
return members.every((cp) => cp.name === first) ? first : "steps";
|
|
8719
|
+
}
|
|
8720
|
+
__name(fanLabel, "fanLabel");
|
|
8721
|
+
function summarizeFan(group, members) {
|
|
8722
|
+
const counts = {};
|
|
8723
|
+
const failures = [];
|
|
8724
|
+
let size = 0;
|
|
8725
|
+
let startedAt;
|
|
8726
|
+
let finishedAt;
|
|
8727
|
+
let firstSeq = Number.POSITIVE_INFINITY;
|
|
8728
|
+
let lastSeq = Number.NEGATIVE_INFINITY;
|
|
8729
|
+
for (const cp of members) {
|
|
8730
|
+
firstSeq = Math.min(firstSeq, cp.seq);
|
|
8731
|
+
lastSeq = Math.max(lastSeq, cp.seq);
|
|
8732
|
+
const folded = foldedCount(cp);
|
|
8733
|
+
const n = folded ?? 1;
|
|
8734
|
+
size += n;
|
|
8735
|
+
const status = isFailedMember(cp) ? "failed" : cp.status;
|
|
8736
|
+
counts[status] = (counts[status] ?? 0) + n;
|
|
8737
|
+
if (status === "failed" && failures.length < FAN_SUMMARY_FAILURES) failures.push(cp);
|
|
8738
|
+
if (cp.status !== "pending" && cp.status !== "running") {
|
|
8739
|
+
if (!startedAt || cp.startedAt < startedAt) startedAt = cp.startedAt;
|
|
8740
|
+
if (!finishedAt || cp.finishedAt > finishedAt) finishedAt = cp.finishedAt;
|
|
8741
|
+
} else if (!startedAt || cp.enqueuedAt < startedAt) {
|
|
8742
|
+
startedAt = cp.enqueuedAt;
|
|
8743
|
+
}
|
|
8744
|
+
}
|
|
8745
|
+
const parsed = parseFanGroup(group);
|
|
8746
|
+
const total = parsed?.size;
|
|
8747
|
+
return {
|
|
8748
|
+
group,
|
|
8749
|
+
label: fanLabel(members),
|
|
8750
|
+
firstSeq,
|
|
8751
|
+
lastSeq,
|
|
8752
|
+
size,
|
|
8753
|
+
total,
|
|
8754
|
+
notStarted: total === void 0 ? void 0 : Math.max(0, total - size),
|
|
8755
|
+
failFast: parsed?.failFast ?? false,
|
|
8756
|
+
counts,
|
|
8757
|
+
startedAt,
|
|
8758
|
+
finishedAt,
|
|
8759
|
+
failures
|
|
8760
|
+
};
|
|
8761
|
+
}
|
|
8762
|
+
__name(summarizeFan, "summarizeFan");
|
|
8763
|
+
function summarizeTimeline(timeline, over) {
|
|
8764
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
8765
|
+
for (const cp of timeline) {
|
|
8766
|
+
if (cp.parallelGroup === void 0) continue;
|
|
8767
|
+
const members = byGroup.get(cp.parallelGroup);
|
|
8768
|
+
if (members) members.push(cp);
|
|
8769
|
+
else byGroup.set(cp.parallelGroup, [
|
|
8770
|
+
cp
|
|
8771
|
+
]);
|
|
8772
|
+
}
|
|
8773
|
+
const summarized = /* @__PURE__ */ new Set();
|
|
8774
|
+
const fans = [];
|
|
8775
|
+
for (const [group, members] of byGroup) {
|
|
8776
|
+
if (members.length <= over && (parseFanGroup(group)?.size ?? 0) <= over) continue;
|
|
8777
|
+
summarized.add(group);
|
|
8778
|
+
fans.push(summarizeFan(group, members));
|
|
8779
|
+
}
|
|
8780
|
+
if (fans.length === 0) return {
|
|
8781
|
+
timeline,
|
|
8782
|
+
fans
|
|
8783
|
+
};
|
|
8784
|
+
return {
|
|
8785
|
+
timeline: timeline.filter((cp) => cp.parallelGroup === void 0 || !summarized.has(cp.parallelGroup)),
|
|
8786
|
+
fans: fans.sort((a, b) => a.firstSeq - b.firstSeq)
|
|
8787
|
+
};
|
|
8788
|
+
}
|
|
8789
|
+
__name(summarizeTimeline, "summarizeTimeline");
|
|
8790
|
+
function pageFan(timeline, group, query) {
|
|
8791
|
+
const matching = timeline.filter((cp) => cp.parallelGroup === group && (!query.failedOnly || isFailedMember(cp)));
|
|
8792
|
+
const { offset, limit } = fanPageBounds(query);
|
|
8793
|
+
return {
|
|
8794
|
+
items: matching.slice(offset, offset + limit),
|
|
8795
|
+
total: matching.length
|
|
8796
|
+
};
|
|
8797
|
+
}
|
|
8798
|
+
__name(pageFan, "pageFan");
|
|
8799
|
+
function fanPageBounds(query) {
|
|
8800
|
+
const offset = query.offset !== void 0 && Number.isFinite(query.offset) ? Math.max(0, Math.floor(query.offset)) : 0;
|
|
8801
|
+
const limit = query.limit !== void 0 && Number.isFinite(query.limit) ? Math.min(FAN_PAGE_MAX, Math.max(1, Math.floor(query.limit))) : 100;
|
|
8802
|
+
return {
|
|
8803
|
+
offset,
|
|
8804
|
+
limit
|
|
8805
|
+
};
|
|
8806
|
+
}
|
|
8807
|
+
__name(fanPageBounds, "fanPageBounds");
|
|
8808
|
+
|
|
8809
|
+
// src/run-detail-store.ts
|
|
8810
|
+
function aggregatesCheckpoints(store) {
|
|
8811
|
+
return store.checkpointStats !== void 0 && store.queryCheckpoints !== void 0;
|
|
8812
|
+
}
|
|
8813
|
+
__name(aggregatesCheckpoints, "aggregatesCheckpoints");
|
|
8814
|
+
function checkpointStatsOf(checkpoints, opts) {
|
|
8815
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
8816
|
+
for (const cp of checkpoints) {
|
|
8817
|
+
const group = cp.parallelGroup ?? null;
|
|
8818
|
+
if (opts?.group !== void 0 && group !== opts.group) continue;
|
|
8819
|
+
const nameKind = checkpointNameKind(cp.name);
|
|
8820
|
+
const key = JSON.stringify([
|
|
8821
|
+
group,
|
|
8822
|
+
cp.status,
|
|
8823
|
+
nameKind
|
|
8824
|
+
]);
|
|
8825
|
+
const enqueuedAt = cp.enqueuedAt ?? cp.startedAt;
|
|
8826
|
+
const bucket = buckets.get(key);
|
|
8827
|
+
if (!bucket) {
|
|
8828
|
+
buckets.set(key, {
|
|
8829
|
+
group,
|
|
8830
|
+
status: cp.status,
|
|
8831
|
+
nameKind,
|
|
8832
|
+
count: 1,
|
|
8833
|
+
minSeq: cp.seq,
|
|
8834
|
+
maxSeq: cp.seq,
|
|
8835
|
+
minName: cp.name,
|
|
8836
|
+
maxName: cp.name,
|
|
8837
|
+
minEnqueuedAt: enqueuedAt,
|
|
8838
|
+
minStartedAt: cp.startedAt,
|
|
8839
|
+
maxStartedAt: cp.startedAt,
|
|
8840
|
+
maxFinishedAt: cp.finishedAt,
|
|
8841
|
+
sumAttempts: cp.attempts
|
|
8842
|
+
});
|
|
8843
|
+
continue;
|
|
8844
|
+
}
|
|
8845
|
+
bucket.count += 1;
|
|
8846
|
+
bucket.minSeq = Math.min(bucket.minSeq, cp.seq);
|
|
8847
|
+
bucket.maxSeq = Math.max(bucket.maxSeq, cp.seq);
|
|
8848
|
+
if (cp.name < bucket.minName) bucket.minName = cp.name;
|
|
8849
|
+
if (cp.name > bucket.maxName) bucket.maxName = cp.name;
|
|
8850
|
+
if (enqueuedAt < bucket.minEnqueuedAt) bucket.minEnqueuedAt = enqueuedAt;
|
|
8851
|
+
if (cp.startedAt < bucket.minStartedAt) bucket.minStartedAt = cp.startedAt;
|
|
8852
|
+
if (cp.startedAt > bucket.maxStartedAt) bucket.maxStartedAt = cp.startedAt;
|
|
8853
|
+
if (cp.finishedAt > bucket.maxFinishedAt) bucket.maxFinishedAt = cp.finishedAt;
|
|
8854
|
+
bucket.sumAttempts += cp.attempts;
|
|
8855
|
+
}
|
|
8856
|
+
return [
|
|
8857
|
+
...buckets.values()
|
|
8858
|
+
];
|
|
8859
|
+
}
|
|
8860
|
+
__name(checkpointStatsOf, "checkpointStatsOf");
|
|
8861
|
+
function queryCheckpointsOf(checkpoints, query) {
|
|
8862
|
+
const excluded = new Set(query.excludeGroups ?? []);
|
|
8863
|
+
const matching = checkpoints.filter((cp) => (query.group === void 0 || cp.parallelGroup === query.group) && (cp.parallelGroup === void 0 || !excluded.has(cp.parallelGroup)) && (query.statuses === void 0 || query.statuses.includes(cp.status)) && (query.namePrefixes === void 0 || query.namePrefixes.some((prefix) => cp.name.startsWith(prefix)))).sort((a, b) => a.seq - b.seq);
|
|
8864
|
+
const offset = query.offset ?? 0;
|
|
8865
|
+
return matching.slice(offset, query.limit === void 0 ? void 0 : offset + query.limit);
|
|
8866
|
+
}
|
|
8867
|
+
__name(queryCheckpointsOf, "queryCheckpointsOf");
|
|
8868
|
+
var TERMINAL = /* @__PURE__ */ __name((status) => status !== "pending" && status !== "running", "TERMINAL");
|
|
8869
|
+
function isWide(group, rows, over) {
|
|
8870
|
+
return rows > over || (parseFanGroup(group)?.size ?? 0) > over;
|
|
8871
|
+
}
|
|
8872
|
+
__name(isWide, "isWide");
|
|
8873
|
+
function mergeBySeq(a, b) {
|
|
8874
|
+
return [
|
|
8875
|
+
...a,
|
|
8876
|
+
...b
|
|
8877
|
+
].sort((x, y) => x.seq - y.seq);
|
|
8878
|
+
}
|
|
8879
|
+
__name(mergeBySeq, "mergeBySeq");
|
|
8880
|
+
async function failedChildren(store, runId, group, buckets) {
|
|
8881
|
+
if (!buckets.some((b) => b.nameKind === "child" && b.status === "completed")) {
|
|
8882
|
+
return {
|
|
8883
|
+
completed: [],
|
|
8884
|
+
failed: []
|
|
8885
|
+
};
|
|
8886
|
+
}
|
|
8887
|
+
const completed = await store.queryCheckpoints(runId, {
|
|
8888
|
+
group,
|
|
8889
|
+
statuses: [
|
|
8890
|
+
"completed"
|
|
8891
|
+
],
|
|
8892
|
+
namePrefixes: [
|
|
8893
|
+
"signal:child:"
|
|
8894
|
+
]
|
|
8895
|
+
});
|
|
8896
|
+
return {
|
|
8897
|
+
completed,
|
|
8898
|
+
failed: completed.filter(isFailedMember)
|
|
8899
|
+
};
|
|
8900
|
+
}
|
|
8901
|
+
__name(failedChildren, "failedChildren");
|
|
8902
|
+
async function summarizeFanFromStats(store, runId, group, buckets, folded) {
|
|
8903
|
+
const counts = {};
|
|
8904
|
+
const add = /* @__PURE__ */ __name((status, n) => {
|
|
8905
|
+
counts[status] = (counts[status] ?? 0) + n;
|
|
8906
|
+
}, "add");
|
|
8907
|
+
let size = 0;
|
|
8908
|
+
let startedAt;
|
|
8909
|
+
let finishedAt;
|
|
8910
|
+
let firstSeq = Number.POSITIVE_INFINITY;
|
|
8911
|
+
let lastSeq = Number.NEGATIVE_INFINITY;
|
|
8912
|
+
let minName;
|
|
8913
|
+
let maxName;
|
|
8914
|
+
let childRuns = false;
|
|
8915
|
+
for (const b of buckets) {
|
|
8916
|
+
firstSeq = Math.min(firstSeq, b.minSeq);
|
|
8917
|
+
lastSeq = Math.max(lastSeq, b.maxSeq);
|
|
8918
|
+
if (minName === void 0 || b.minName < minName) minName = b.minName;
|
|
8919
|
+
if (maxName === void 0 || b.maxName > maxName) maxName = b.maxName;
|
|
8920
|
+
if (b.nameKind === "child" || b.nameKind === "folded") childRuns = true;
|
|
8921
|
+
if (TERMINAL(b.status)) {
|
|
8922
|
+
if (!startedAt || b.minStartedAt < startedAt) startedAt = b.minStartedAt;
|
|
8923
|
+
if (!finishedAt || b.maxFinishedAt > finishedAt) finishedAt = b.maxFinishedAt;
|
|
8924
|
+
} else if (!startedAt || b.minEnqueuedAt < startedAt) {
|
|
8925
|
+
startedAt = b.minEnqueuedAt;
|
|
8926
|
+
}
|
|
8927
|
+
if (b.nameKind === "folded") continue;
|
|
8928
|
+
if (b.nameKind === "child" && b.status === "completed") continue;
|
|
8929
|
+
add(b.status, b.count);
|
|
8930
|
+
size += b.count;
|
|
8931
|
+
}
|
|
8932
|
+
for (const cp of folded) {
|
|
8933
|
+
const n = foldedCount(cp) ?? 1;
|
|
8934
|
+
add(isFailedMember(cp) ? "failed" : cp.status, n);
|
|
8935
|
+
size += n;
|
|
8936
|
+
}
|
|
8937
|
+
const children = await failedChildren(store, runId, group, buckets);
|
|
8938
|
+
const failedChildSet = new Set(children.failed);
|
|
8939
|
+
for (const cp of children.completed) add(failedChildSet.has(cp) ? "failed" : "completed", 1);
|
|
8940
|
+
size += children.completed.length;
|
|
8941
|
+
const failedStatusRows = buckets.some((b) => b.status === "failed") ? await store.queryCheckpoints(runId, {
|
|
8942
|
+
group,
|
|
8943
|
+
statuses: [
|
|
8944
|
+
"failed"
|
|
8945
|
+
],
|
|
8946
|
+
limit: FAN_SUMMARY_FAILURES
|
|
8947
|
+
}) : [];
|
|
8948
|
+
const failures = mergeBySeq(failedStatusRows, children.failed).slice(0, FAN_SUMMARY_FAILURES);
|
|
8949
|
+
const parsed = parseFanGroup(group);
|
|
8950
|
+
const total = parsed?.size;
|
|
8951
|
+
return {
|
|
8952
|
+
group,
|
|
8953
|
+
label: childRuns ? "child runs" : minName === maxName ? minName ?? "" : "steps",
|
|
8954
|
+
firstSeq,
|
|
8955
|
+
lastSeq,
|
|
8956
|
+
size,
|
|
8957
|
+
total,
|
|
8958
|
+
notStarted: total === void 0 ? void 0 : Math.max(0, total - size),
|
|
8959
|
+
failFast: parsed?.failFast ?? false,
|
|
8960
|
+
counts,
|
|
8961
|
+
startedAt,
|
|
8962
|
+
finishedAt,
|
|
8963
|
+
failures
|
|
8964
|
+
};
|
|
8965
|
+
}
|
|
8966
|
+
__name(summarizeFanFromStats, "summarizeFanFromStats");
|
|
8967
|
+
async function firstChildIds(store, runId) {
|
|
8968
|
+
const ids = [];
|
|
8969
|
+
for (let offset = 0; ids.length < SUMMARY_CHILD_LIMIT; offset += SUMMARY_CHILD_LIMIT) {
|
|
8970
|
+
const page = await store.queryCheckpoints(runId, {
|
|
8971
|
+
namePrefixes: CHILD_EDGE_PREFIXES,
|
|
8972
|
+
offset,
|
|
8973
|
+
limit: SUMMARY_CHILD_LIMIT
|
|
8974
|
+
});
|
|
8975
|
+
ids.push(...childIdsOf(page));
|
|
8976
|
+
if (page.length < SUMMARY_CHILD_LIMIT) break;
|
|
8977
|
+
}
|
|
8978
|
+
return ids.slice(0, SUMMARY_CHILD_LIMIT);
|
|
8979
|
+
}
|
|
8980
|
+
__name(firstChildIds, "firstChildIds");
|
|
8981
|
+
async function summarizeRunFromStore(store, runId, over, stats) {
|
|
8982
|
+
const all = stats ?? await store.checkpointStats(runId);
|
|
8983
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
8984
|
+
let edgeRows = 0;
|
|
8985
|
+
let hasFolded = false;
|
|
8986
|
+
for (const b of all) {
|
|
8987
|
+
if (b.nameKind === "child" || b.nameKind === "spawn") edgeRows += b.count;
|
|
8988
|
+
if (b.nameKind === "folded") hasFolded = true;
|
|
8989
|
+
if (b.group === null) continue;
|
|
8990
|
+
const buckets = byGroup.get(b.group);
|
|
8991
|
+
if (buckets) buckets.push(b);
|
|
8992
|
+
else byGroup.set(b.group, [
|
|
8993
|
+
b
|
|
8994
|
+
]);
|
|
8995
|
+
}
|
|
8996
|
+
const wide = [
|
|
8997
|
+
...byGroup
|
|
8998
|
+
].filter(([group, buckets]) => isWide(group, buckets.reduce((n, b) => n + b.count, 0), over));
|
|
8999
|
+
const [timeline, folded, children] = await Promise.all([
|
|
9000
|
+
store.queryCheckpoints(runId, {
|
|
9001
|
+
excludeGroups: wide.map(([group]) => group)
|
|
9002
|
+
}),
|
|
9003
|
+
hasFolded ? store.queryCheckpoints(runId, {
|
|
9004
|
+
namePrefixes: [
|
|
9005
|
+
FOLDED_PREFIX
|
|
9006
|
+
]
|
|
9007
|
+
}) : [],
|
|
9008
|
+
edgeRows > 0 || hasFolded ? firstChildIds(store, runId) : []
|
|
9009
|
+
]);
|
|
9010
|
+
const fans = await Promise.all(wide.map(([group, buckets]) => summarizeFanFromStats(store, runId, group, buckets, folded.filter((cp) => cp.parallelGroup === group))));
|
|
9011
|
+
const childCount = folded.reduce((n, cp) => n + childIdsOf([
|
|
9012
|
+
cp
|
|
9013
|
+
]).length, edgeRows);
|
|
9014
|
+
return {
|
|
9015
|
+
timeline,
|
|
9016
|
+
fans: fans.sort((a, b) => a.firstSeq - b.firstSeq),
|
|
9017
|
+
children,
|
|
9018
|
+
childCount
|
|
9019
|
+
};
|
|
9020
|
+
}
|
|
9021
|
+
__name(summarizeRunFromStore, "summarizeRunFromStore");
|
|
9022
|
+
async function pageFanFromStore(store, runId, group, query, stats) {
|
|
9023
|
+
const buckets = stats ?? await store.checkpointStats(runId, {
|
|
9024
|
+
group
|
|
9025
|
+
});
|
|
9026
|
+
const { offset, limit } = fanPageBounds(query);
|
|
9027
|
+
if (!query.failedOnly) {
|
|
9028
|
+
const total2 = buckets.reduce((n, b) => n + b.count, 0);
|
|
9029
|
+
const items = total2 > offset ? await store.queryCheckpoints(runId, {
|
|
9030
|
+
group,
|
|
9031
|
+
offset,
|
|
9032
|
+
limit
|
|
9033
|
+
}) : [];
|
|
9034
|
+
return {
|
|
9035
|
+
items,
|
|
9036
|
+
total: total2
|
|
9037
|
+
};
|
|
9038
|
+
}
|
|
9039
|
+
const failedStatus = buckets.filter((b) => b.status === "failed").reduce((n, b) => n + b.count, 0);
|
|
9040
|
+
const children = await failedChildren(store, runId, group, buckets);
|
|
9041
|
+
const total = failedStatus + children.failed.length;
|
|
9042
|
+
if (total <= offset) return {
|
|
9043
|
+
items: [],
|
|
9044
|
+
total
|
|
9045
|
+
};
|
|
9046
|
+
if (children.failed.length === 0) {
|
|
9047
|
+
return {
|
|
9048
|
+
items: await store.queryCheckpoints(runId, {
|
|
9049
|
+
group,
|
|
9050
|
+
statuses: [
|
|
9051
|
+
"failed"
|
|
9052
|
+
],
|
|
9053
|
+
offset,
|
|
9054
|
+
limit
|
|
9055
|
+
}),
|
|
9056
|
+
total
|
|
9057
|
+
};
|
|
9058
|
+
}
|
|
9059
|
+
const failedRows = await store.queryCheckpoints(runId, {
|
|
9060
|
+
group,
|
|
9061
|
+
statuses: [
|
|
9062
|
+
"failed"
|
|
9063
|
+
],
|
|
9064
|
+
limit: offset + limit
|
|
9065
|
+
});
|
|
9066
|
+
return {
|
|
9067
|
+
items: mergeBySeq(failedRows, children.failed).slice(offset, offset + limit),
|
|
9068
|
+
total
|
|
9069
|
+
};
|
|
9070
|
+
}
|
|
9071
|
+
__name(pageFanFromStore, "pageFanFromStore");
|
|
9072
|
+
function hash(text) {
|
|
9073
|
+
let h1 = 3735928559;
|
|
9074
|
+
let h2 = 1103547991;
|
|
9075
|
+
for (let i = 0; i < text.length; i++) {
|
|
9076
|
+
const ch = text.charCodeAt(i);
|
|
9077
|
+
h1 = Math.imul(h1 ^ ch, 2654435761);
|
|
9078
|
+
h2 = Math.imul(h2 ^ ch, 1597334677);
|
|
9079
|
+
}
|
|
9080
|
+
h1 = Math.imul(h1 ^ h1 >>> 16, 2246822507) ^ Math.imul(h2 ^ h2 >>> 13, 3266489909);
|
|
9081
|
+
h2 = Math.imul(h2 ^ h2 >>> 16, 2246822507) ^ Math.imul(h1 ^ h1 >>> 13, 3266489909);
|
|
9082
|
+
return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16);
|
|
9083
|
+
}
|
|
9084
|
+
__name(hash, "hash");
|
|
9085
|
+
function statsFingerprint(stats) {
|
|
9086
|
+
return stats.map((b) => [
|
|
9087
|
+
b.group ?? "",
|
|
9088
|
+
b.status,
|
|
9089
|
+
b.nameKind,
|
|
9090
|
+
b.count,
|
|
9091
|
+
b.minSeq,
|
|
9092
|
+
b.maxSeq,
|
|
9093
|
+
b.minName,
|
|
9094
|
+
b.maxName,
|
|
9095
|
+
b.minEnqueuedAt.getTime(),
|
|
9096
|
+
b.minStartedAt.getTime(),
|
|
9097
|
+
b.maxStartedAt.getTime(),
|
|
9098
|
+
b.maxFinishedAt.getTime(),
|
|
9099
|
+
b.sumAttempts
|
|
9100
|
+
].join("")).sort().join("");
|
|
9101
|
+
}
|
|
9102
|
+
__name(statsFingerprint, "statsFingerprint");
|
|
9103
|
+
function runDetailEtag(run, stats, over) {
|
|
9104
|
+
return `"rd-${hash(`${over}${JSON.stringify(run)}${statsFingerprint(stats)}`)}"`;
|
|
9105
|
+
}
|
|
9106
|
+
__name(runDetailEtag, "runDetailEtag");
|
|
9107
|
+
function fanPageEtag(group, stats, query) {
|
|
9108
|
+
const { offset, limit } = fanPageBounds(query);
|
|
9109
|
+
const key = [
|
|
9110
|
+
group,
|
|
9111
|
+
offset,
|
|
9112
|
+
limit,
|
|
9113
|
+
query.failedOnly === true
|
|
9114
|
+
].join("");
|
|
9115
|
+
return `"fp-${hash(`${key}${statsFingerprint(stats)}`)}"`;
|
|
9116
|
+
}
|
|
9117
|
+
__name(fanPageEtag, "fanPageEtag");
|
|
9118
|
+
function etagMatches(ifNoneMatch, etag) {
|
|
9119
|
+
if (!ifNoneMatch) return false;
|
|
9120
|
+
const bare = /* @__PURE__ */ __name((tag) => tag.trim().replace(/^W\//, ""), "bare");
|
|
9121
|
+
const target = bare(etag);
|
|
9122
|
+
return ifNoneMatch.split(",").some((tag) => tag.trim() === "*" || bare(tag) === target);
|
|
9123
|
+
}
|
|
9124
|
+
__name(etagMatches, "etagMatches");
|
|
9125
|
+
|
|
8101
9126
|
// src/codec-state-store.ts
|
|
8102
9127
|
var CodecStateStore = class {
|
|
8103
9128
|
static {
|
|
@@ -8126,11 +9151,19 @@ var CodecStateStore = class {
|
|
|
8126
9151
|
/** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
|
|
8127
9152
|
* plain-save fallback still kicks in through the wrapper. */
|
|
8128
9153
|
saveCheckpointIfPending;
|
|
9154
|
+
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
9155
|
+
saveCheckpointsIfPending;
|
|
8129
9156
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
8130
9157
|
countCheckpoints;
|
|
8131
9158
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
8132
9159
|
* when the inner store prunes, so the retention poller still sees (and warns about) its absence. */
|
|
8133
9160
|
pruneTerminalRuns;
|
|
9161
|
+
/** Forwarded verbatim — the buckets aggregate names, statuses and timestamps, never a payload.
|
|
9162
|
+
* Bound only when the inner store aggregates, so the run detail's fallback still applies. */
|
|
9163
|
+
checkpointStats;
|
|
9164
|
+
/** Filters on unencoded columns, then decodes the rows like {@link listCheckpoints}. Bound only
|
|
9165
|
+
* when the inner store has it. */
|
|
9166
|
+
queryCheckpoints;
|
|
8134
9167
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store counts, so the
|
|
8135
9168
|
* engine's facet/listing fallback still applies through the wrapper. */
|
|
8136
9169
|
countRuns;
|
|
@@ -8157,8 +9190,14 @@ var CodecStateStore = class {
|
|
|
8157
9190
|
if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
|
|
8158
9191
|
const saveIfPending = inner.saveCheckpointIfPending;
|
|
8159
9192
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
9193
|
+
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
9194
|
+
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
8160
9195
|
const countCps = inner.countCheckpoints;
|
|
8161
9196
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
9197
|
+
const stats = inner.checkpointStats;
|
|
9198
|
+
if (stats) this.checkpointStats = (runId, opts) => stats.call(inner, runId, opts);
|
|
9199
|
+
const queryCps = inner.queryCheckpoints;
|
|
9200
|
+
if (queryCps) this.queryCheckpoints = async (runId, query) => (await queryCps.call(inner, runId, query)).map((c) => this.decCp(c));
|
|
8162
9201
|
const prune = inner.pruneTerminalRuns;
|
|
8163
9202
|
if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
|
|
8164
9203
|
const count = inner.countRuns;
|
|
@@ -8366,8 +9405,11 @@ function collectMetrics(engine) {
|
|
|
8366
9405
|
started: 0,
|
|
8367
9406
|
completed: 0,
|
|
8368
9407
|
failed: 0,
|
|
8369
|
-
suspended: 0
|
|
9408
|
+
suspended: 0,
|
|
9409
|
+
routed: 0,
|
|
9410
|
+
unrouted: 0
|
|
8370
9411
|
},
|
|
9412
|
+
routedByCode: {},
|
|
8371
9413
|
steps: {
|
|
8372
9414
|
started: 0,
|
|
8373
9415
|
completed: 0,
|
|
@@ -8409,6 +9451,15 @@ function collectMetrics(engine) {
|
|
|
8409
9451
|
case "run.suspended":
|
|
8410
9452
|
m.runs.suspended += 1;
|
|
8411
9453
|
break;
|
|
9454
|
+
case "run.routed": {
|
|
9455
|
+
m.runs.routed += 1;
|
|
9456
|
+
const code = e.error?.code ?? "*";
|
|
9457
|
+
m.routedByCode[code] = (m.routedByCode[code] ?? 0) + 1;
|
|
9458
|
+
break;
|
|
9459
|
+
}
|
|
9460
|
+
case "run.unrouted":
|
|
9461
|
+
m.runs.unrouted += 1;
|
|
9462
|
+
break;
|
|
8412
9463
|
case "step.started":
|
|
8413
9464
|
m.steps.started += 1;
|
|
8414
9465
|
break;
|
|
@@ -8435,6 +9486,8 @@ function collectMetrics(engine) {
|
|
|
8435
9486
|
for (const [k, v] of Object.entries(m.steps)) lines.push(`durable_steps_total{event="${k}"} ${v}`);
|
|
8436
9487
|
lines.push("# TYPE durable_runs_by_workflow_total counter");
|
|
8437
9488
|
for (const [name, v] of Object.entries(m.byWorkflow)) for (const [event, n] of Object.entries(v)) lines.push(`durable_runs_by_workflow_total{workflow="${name}",event="${event}"} ${n}`);
|
|
9489
|
+
lines.push("# TYPE durable_runs_routed_total counter");
|
|
9490
|
+
for (const [code, n] of Object.entries(m.routedByCode)) lines.push(`durable_runs_routed_total{code="${code}"} ${n}`);
|
|
8438
9491
|
lines.push("# TYPE durable_step_duration_ms_sum counter");
|
|
8439
9492
|
lines.push(`durable_step_duration_ms_sum ${m.stepDuration.sumMs}`);
|
|
8440
9493
|
lines.push("# TYPE durable_step_duration_count counter");
|
|
@@ -8605,6 +9658,11 @@ var InMemoryStateStore = class {
|
|
|
8605
9658
|
* so this lets the benchmark contrast "fetch N, scan in JS" against "fetch the k that match".
|
|
8606
9659
|
*/
|
|
8607
9660
|
lastCheckpointRowsRead = 0;
|
|
9661
|
+
/**
|
|
9662
|
+
* Test/observability hook: checkpoint rows materialized since construction, across every
|
|
9663
|
+
* checkpoint read that returns rows — what a run-detail benchmark compares between paths.
|
|
9664
|
+
*/
|
|
9665
|
+
checkpointRowsRead = 0;
|
|
8608
9666
|
key(runId, seq) {
|
|
8609
9667
|
return `${runId}:${seq}`;
|
|
8610
9668
|
}
|
|
@@ -8730,6 +9788,16 @@ var InMemoryStateStore = class {
|
|
|
8730
9788
|
});
|
|
8731
9789
|
return true;
|
|
8732
9790
|
}
|
|
9791
|
+
async saveCheckpointsIfPending(checkpoints) {
|
|
9792
|
+
return checkpoints.map((checkpoint) => {
|
|
9793
|
+
const key = this.key(checkpoint.runId, checkpoint.seq);
|
|
9794
|
+
if (this.checkpoints.get(key)?.status !== "pending") return false;
|
|
9795
|
+
this.checkpoints.set(key, {
|
|
9796
|
+
...checkpoint
|
|
9797
|
+
});
|
|
9798
|
+
return true;
|
|
9799
|
+
});
|
|
9800
|
+
}
|
|
8733
9801
|
async listIncompleteRuns(namespace) {
|
|
8734
9802
|
return [
|
|
8735
9803
|
...this.runs.values()
|
|
@@ -9009,6 +10077,23 @@ var InMemoryStateStore = class {
|
|
|
9009
10077
|
...cp
|
|
9010
10078
|
}));
|
|
9011
10079
|
this.lastCheckpointRowsRead = rows.length;
|
|
10080
|
+
this.checkpointRowsRead += rows.length;
|
|
10081
|
+
return rows;
|
|
10082
|
+
}
|
|
10083
|
+
runCheckpoints(runId) {
|
|
10084
|
+
return [
|
|
10085
|
+
...this.checkpoints.values()
|
|
10086
|
+
].filter((cp) => cp.runId === runId);
|
|
10087
|
+
}
|
|
10088
|
+
async checkpointStats(runId, opts) {
|
|
10089
|
+
return checkpointStatsOf(this.runCheckpoints(runId), opts);
|
|
10090
|
+
}
|
|
10091
|
+
async queryCheckpoints(runId, query) {
|
|
10092
|
+
const rows = queryCheckpointsOf(this.runCheckpoints(runId), query).map((cp) => ({
|
|
10093
|
+
...cp
|
|
10094
|
+
}));
|
|
10095
|
+
this.lastCheckpointRowsRead = rows.length;
|
|
10096
|
+
this.checkpointRowsRead += rows.length;
|
|
9012
10097
|
return rows;
|
|
9013
10098
|
}
|
|
9014
10099
|
async getLatestCheckpointByName(runId, name) {
|
|
@@ -9036,6 +10121,7 @@ var InMemoryStateStore = class {
|
|
|
9036
10121
|
...cp
|
|
9037
10122
|
}));
|
|
9038
10123
|
this.lastCheckpointRowsRead = rows.length;
|
|
10124
|
+
this.checkpointRowsRead += rows.length;
|
|
9039
10125
|
return rows;
|
|
9040
10126
|
}
|
|
9041
10127
|
};
|
|
@@ -9087,6 +10173,7 @@ var InMemoryTransport = class {
|
|
|
9087
10173
|
};
|
|
9088
10174
|
// Annotate the CommonJS export names for ESM import in node:
|
|
9089
10175
|
0 && (module.exports = {
|
|
10176
|
+
CHILD_EDGE_PREFIXES,
|
|
9090
10177
|
CONTROL_FLOW_SIGNAL,
|
|
9091
10178
|
CURRENT_PROTOCOL_VERSION,
|
|
9092
10179
|
CodecStateStore,
|
|
@@ -9104,8 +10191,10 @@ var InMemoryTransport = class {
|
|
|
9104
10191
|
EVENT_TOKEN_PREFIX,
|
|
9105
10192
|
Entities,
|
|
9106
10193
|
EventAccumulators,
|
|
10194
|
+
FAN_SUMMARY_FAILURES,
|
|
9107
10195
|
FatalError,
|
|
9108
10196
|
GatherError,
|
|
10197
|
+
HANDOFF_SEQ_BASE,
|
|
9109
10198
|
InMemoryAdmissionBackend,
|
|
9110
10199
|
InMemoryStateStore,
|
|
9111
10200
|
InMemoryTransport,
|
|
@@ -9125,6 +10214,7 @@ var InMemoryTransport = class {
|
|
|
9125
10214
|
SINGLETON_ADMITTED_TAG,
|
|
9126
10215
|
STATE_STORE,
|
|
9127
10216
|
STATE_STORE_CANONICAL,
|
|
10217
|
+
SUMMARY_CHILD_LIMIT,
|
|
9128
10218
|
ScheduleAlreadyExistsError,
|
|
9129
10219
|
ScheduleClient,
|
|
9130
10220
|
ScheduleNotFoundError,
|
|
@@ -9139,9 +10229,11 @@ var InMemoryTransport = class {
|
|
|
9139
10229
|
WorkflowEngine,
|
|
9140
10230
|
WorkflowSuspended,
|
|
9141
10231
|
aggregateAnnouncements,
|
|
10232
|
+
aggregatesCheckpoints,
|
|
9142
10233
|
announcementsOf,
|
|
9143
10234
|
applyAttributeQuery,
|
|
9144
10235
|
assertConcurrency,
|
|
10236
|
+
assertRoutes,
|
|
9145
10237
|
attributeColumnFor,
|
|
9146
10238
|
attributeOperand,
|
|
9147
10239
|
attributePredicateOperands,
|
|
@@ -9151,9 +10243,14 @@ var InMemoryTransport = class {
|
|
|
9151
10243
|
buildWorkflowDirectory,
|
|
9152
10244
|
canRoute,
|
|
9153
10245
|
capableWorkers,
|
|
10246
|
+
checkpointNameKind,
|
|
10247
|
+
checkpointStatsOf,
|
|
10248
|
+
childIdsOf,
|
|
9154
10249
|
classifyWaiterToken,
|
|
9155
10250
|
clearDurableExecutionWrappers,
|
|
9156
10251
|
clearTerminalSingletonAdmission,
|
|
10252
|
+
codeAndDetails,
|
|
10253
|
+
codeOf,
|
|
9157
10254
|
collectMetrics,
|
|
9158
10255
|
concurrencyTag,
|
|
9159
10256
|
controlPlaneDescriptor,
|
|
@@ -9166,23 +10263,32 @@ var InMemoryTransport = class {
|
|
|
9166
10263
|
debug,
|
|
9167
10264
|
describeWorker,
|
|
9168
10265
|
descriptorHash,
|
|
10266
|
+
detailsOf,
|
|
9169
10267
|
error,
|
|
10268
|
+
etagMatches,
|
|
9170
10269
|
eventMatchOf,
|
|
9171
10270
|
eventMatches,
|
|
9172
10271
|
eventNameOf,
|
|
9173
10272
|
eventPrefix,
|
|
9174
10273
|
eventToken,
|
|
9175
10274
|
facetOrigin,
|
|
10275
|
+
fanPageBounds,
|
|
10276
|
+
fanPageEtag,
|
|
10277
|
+
foldedCount,
|
|
10278
|
+
handoffHandlerId,
|
|
10279
|
+
handoffName,
|
|
9176
10280
|
hasDurableExecutionWrappers,
|
|
9177
10281
|
heartbeatStatus,
|
|
9178
10282
|
indexWaitersByRun,
|
|
9179
10283
|
info,
|
|
9180
10284
|
isEngineMintedTag,
|
|
9181
10285
|
isEventToken,
|
|
10286
|
+
isFailedMember,
|
|
9182
10287
|
isLegacyDescriptor,
|
|
9183
10288
|
isNonEmptyRunScope,
|
|
9184
10289
|
isWorkflowControlFlowSignal,
|
|
9185
10290
|
listSignalWaitersOfRuns,
|
|
10291
|
+
liveHandoff,
|
|
9186
10292
|
log,
|
|
9187
10293
|
matchesAttributes,
|
|
9188
10294
|
mergeRunFacetRows,
|
|
@@ -9193,16 +10299,23 @@ var InMemoryTransport = class {
|
|
|
9193
10299
|
normalizeAttributeRows,
|
|
9194
10300
|
normalizeDescriptor,
|
|
9195
10301
|
observedAnnouncements,
|
|
10302
|
+
onErrorRunId,
|
|
10303
|
+
pageFan,
|
|
10304
|
+
pageFanFromStore,
|
|
9196
10305
|
parentRunIdOf,
|
|
9197
10306
|
parseDuration,
|
|
9198
10307
|
partitionOfToken,
|
|
9199
10308
|
planDispatch,
|
|
9200
10309
|
prevCronFireMs,
|
|
10310
|
+
queryCheckpointsOf,
|
|
9201
10311
|
readSearchAttributes,
|
|
9202
10312
|
requiredCapabilities,
|
|
9203
10313
|
resolveConcurrency,
|
|
9204
10314
|
resolveRouting,
|
|
9205
10315
|
resolveRunWaiting,
|
|
10316
|
+
retiredHandoffName,
|
|
10317
|
+
routeFor,
|
|
10318
|
+
runDetailEtag,
|
|
9206
10319
|
runDurableExecution,
|
|
9207
10320
|
runInStepLogger,
|
|
9208
10321
|
runInStepScope,
|
|
@@ -9222,6 +10335,9 @@ var InMemoryTransport = class {
|
|
|
9222
10335
|
sub,
|
|
9223
10336
|
subEvent,
|
|
9224
10337
|
subProcess,
|
|
10338
|
+
summarizeFan,
|
|
10339
|
+
summarizeRunFromStore,
|
|
10340
|
+
summarizeTimeline,
|
|
9225
10341
|
tenantGroup,
|
|
9226
10342
|
useDurableExecution,
|
|
9227
10343
|
warn,
|