@dudousxd/nestjs-durable-core 0.77.0 → 0.79.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -22,6 +22,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
22
22
  // src/index.ts
23
23
  var index_exports = {};
24
24
  __export(index_exports, {
25
+ CHILD_EDGE_PREFIXES: () => CHILD_EDGE_PREFIXES,
25
26
  CONTROL_FLOW_SIGNAL: () => CONTROL_FLOW_SIGNAL,
26
27
  CURRENT_PROTOCOL_VERSION: () => CURRENT_PROTOCOL_VERSION,
27
28
  CodecStateStore: () => CodecStateStore,
@@ -39,8 +40,10 @@ __export(index_exports, {
39
40
  EVENT_TOKEN_PREFIX: () => EVENT_TOKEN_PREFIX,
40
41
  Entities: () => Entities,
41
42
  EventAccumulators: () => EventAccumulators,
43
+ FAN_SUMMARY_FAILURES: () => FAN_SUMMARY_FAILURES,
42
44
  FatalError: () => FatalError,
43
45
  GatherError: () => GatherError,
46
+ HANDOFF_SEQ_BASE: () => HANDOFF_SEQ_BASE,
44
47
  InMemoryAdmissionBackend: () => InMemoryAdmissionBackend,
45
48
  InMemoryStateStore: () => InMemoryStateStore,
46
49
  InMemoryTransport: () => InMemoryTransport,
@@ -60,6 +63,7 @@ __export(index_exports, {
60
63
  SINGLETON_ADMITTED_TAG: () => SINGLETON_ADMITTED_TAG,
61
64
  STATE_STORE: () => STATE_STORE,
62
65
  STATE_STORE_CANONICAL: () => STATE_STORE_CANONICAL,
66
+ SUMMARY_CHILD_LIMIT: () => SUMMARY_CHILD_LIMIT,
63
67
  ScheduleAlreadyExistsError: () => ScheduleAlreadyExistsError,
64
68
  ScheduleClient: () => ScheduleClient,
65
69
  ScheduleNotFoundError: () => ScheduleNotFoundError,
@@ -74,9 +78,11 @@ __export(index_exports, {
74
78
  WorkflowEngine: () => WorkflowEngine,
75
79
  WorkflowSuspended: () => WorkflowSuspended,
76
80
  aggregateAnnouncements: () => aggregateAnnouncements,
81
+ aggregatesCheckpoints: () => aggregatesCheckpoints,
77
82
  announcementsOf: () => announcementsOf,
78
83
  applyAttributeQuery: () => applyAttributeQuery,
79
84
  assertConcurrency: () => assertConcurrency,
85
+ assertRoutes: () => assertRoutes,
80
86
  attributeColumnFor: () => attributeColumnFor,
81
87
  attributeOperand: () => attributeOperand,
82
88
  attributePredicateOperands: () => attributePredicateOperands,
@@ -86,9 +92,14 @@ __export(index_exports, {
86
92
  buildWorkflowDirectory: () => buildWorkflowDirectory,
87
93
  canRoute: () => canRoute,
88
94
  capableWorkers: () => capableWorkers,
95
+ checkpointNameKind: () => checkpointNameKind,
96
+ checkpointStatsOf: () => checkpointStatsOf,
97
+ childIdsOf: () => childIdsOf,
89
98
  classifyWaiterToken: () => classifyWaiterToken,
90
99
  clearDurableExecutionWrappers: () => clearDurableExecutionWrappers,
91
100
  clearTerminalSingletonAdmission: () => clearTerminalSingletonAdmission,
101
+ codeAndDetails: () => codeAndDetails,
102
+ codeOf: () => codeOf,
92
103
  collectMetrics: () => collectMetrics,
93
104
  concurrencyTag: () => concurrencyTag,
94
105
  controlPlaneDescriptor: () => controlPlaneDescriptor,
@@ -101,23 +112,32 @@ __export(index_exports, {
101
112
  debug: () => debug,
102
113
  describeWorker: () => describeWorker,
103
114
  descriptorHash: () => descriptorHash,
115
+ detailsOf: () => detailsOf,
104
116
  error: () => error,
117
+ etagMatches: () => etagMatches,
105
118
  eventMatchOf: () => eventMatchOf,
106
119
  eventMatches: () => eventMatches,
107
120
  eventNameOf: () => eventNameOf,
108
121
  eventPrefix: () => eventPrefix,
109
122
  eventToken: () => eventToken,
110
123
  facetOrigin: () => facetOrigin,
124
+ fanPageBounds: () => fanPageBounds,
125
+ fanPageEtag: () => fanPageEtag,
126
+ foldedCount: () => foldedCount,
127
+ handoffHandlerId: () => handoffHandlerId,
128
+ handoffName: () => handoffName,
111
129
  hasDurableExecutionWrappers: () => hasDurableExecutionWrappers,
112
130
  heartbeatStatus: () => heartbeatStatus,
113
131
  indexWaitersByRun: () => indexWaitersByRun,
114
132
  info: () => info,
115
133
  isEngineMintedTag: () => isEngineMintedTag,
116
134
  isEventToken: () => isEventToken,
135
+ isFailedMember: () => isFailedMember,
117
136
  isLegacyDescriptor: () => isLegacyDescriptor,
118
137
  isNonEmptyRunScope: () => isNonEmptyRunScope,
119
138
  isWorkflowControlFlowSignal: () => isWorkflowControlFlowSignal,
120
139
  listSignalWaitersOfRuns: () => listSignalWaitersOfRuns,
140
+ liveHandoff: () => liveHandoff,
121
141
  log: () => log,
122
142
  matchesAttributes: () => matchesAttributes,
123
143
  mergeRunFacetRows: () => mergeRunFacetRows,
@@ -128,16 +148,23 @@ __export(index_exports, {
128
148
  normalizeAttributeRows: () => normalizeAttributeRows,
129
149
  normalizeDescriptor: () => normalizeDescriptor,
130
150
  observedAnnouncements: () => observedAnnouncements,
151
+ onErrorRunId: () => onErrorRunId,
152
+ pageFan: () => pageFan,
153
+ pageFanFromStore: () => pageFanFromStore,
131
154
  parentRunIdOf: () => parentRunIdOf,
132
155
  parseDuration: () => parseDuration,
133
156
  partitionOfToken: () => partitionOfToken,
134
157
  planDispatch: () => planDispatch,
135
158
  prevCronFireMs: () => prevCronFireMs,
159
+ queryCheckpointsOf: () => queryCheckpointsOf,
136
160
  readSearchAttributes: () => readSearchAttributes,
137
161
  requiredCapabilities: () => requiredCapabilities,
138
162
  resolveConcurrency: () => resolveConcurrency,
139
163
  resolveRouting: () => resolveRouting,
140
164
  resolveRunWaiting: () => resolveRunWaiting,
165
+ retiredHandoffName: () => retiredHandoffName,
166
+ routeFor: () => routeFor,
167
+ runDetailEtag: () => runDetailEtag,
141
168
  runDurableExecution: () => runDurableExecution,
142
169
  runInStepLogger: () => runInStepLogger,
143
170
  runInStepScope: () => runInStepScope,
@@ -157,6 +184,9 @@ __export(index_exports, {
157
184
  sub: () => sub,
158
185
  subEvent: () => subEvent,
159
186
  subProcess: () => subProcess,
187
+ summarizeFan: () => summarizeFan,
188
+ summarizeRunFromStore: () => summarizeRunFromStore,
189
+ summarizeTimeline: () => summarizeTimeline,
160
190
  tenantGroup: () => tenantGroup,
161
191
  useDurableExecution: () => useDurableExecution,
162
192
  warn: () => warn,
@@ -428,10 +458,16 @@ var FatalError = class extends Error {
428
458
  __name(this, "FatalError");
429
459
  }
430
460
  code;
431
- constructor(message, code) {
461
+ /** Structured, JSON-serializable detail (e.g. validation issues). It travels with the failure to
462
+ * the run's error, a parent's `allSettled` item and an onError handler's input. */
463
+ details;
464
+ /** Read by the step result builders, so a dispatched step's retry policy honours it too. */
465
+ retryable = false;
466
+ constructor(message, code, details) {
432
467
  super(message);
433
468
  this.name = "FatalError";
434
469
  this.code = code;
470
+ this.details = details;
435
471
  }
436
472
  };
437
473
  var ConcurrencyLimitError = class extends Error {
@@ -509,6 +545,8 @@ var NonDeterminismError = class extends Error {
509
545
  static {
510
546
  __name(this, "NonDeterminismError");
511
547
  }
548
+ /** Marks the failure as a broken deploy, which an `onError` route never takes. */
549
+ code = "non_determinism";
512
550
  runId;
513
551
  seq;
514
552
  constructor(runId, seq, expected, recorded) {
@@ -920,6 +958,9 @@ async function runStepHandler(task, handler) {
920
958
  ...typeof e?.code === "string" ? {
921
959
  code: e.code
922
960
  } : {},
961
+ ...e?.details !== void 0 ? {
962
+ details: e.details
963
+ } : {},
923
964
  ...typeof e?.retryable === "boolean" ? {
924
965
  retryable: e.retryable
925
966
  } : {}
@@ -996,11 +1037,36 @@ __name(isFailedCompletion, "isFailedCompletion");
996
1037
  function unwrapCompletion(payload, label) {
997
1038
  const c = payload;
998
1039
  if (c && typeof c === "object" && "ok" in c && c.ok === false) {
999
- throw new FatalError(`${label} failed: ${c.error}`);
1040
+ throw new FatalError(`${label} failed: ${c.error}`, c.code, c.details);
1000
1041
  }
1001
1042
  return c?.value;
1002
1043
  }
1003
1044
  __name(unwrapCompletion, "unwrapCompletion");
1045
+ function failedCompletion(error2) {
1046
+ return {
1047
+ ok: false,
1048
+ error: error2.message,
1049
+ ...error2.code !== void 0 ? {
1050
+ code: error2.code
1051
+ } : {},
1052
+ ...error2.details !== void 0 ? {
1053
+ details: error2.details
1054
+ } : {}
1055
+ };
1056
+ }
1057
+ __name(failedCompletion, "failedCompletion");
1058
+ function completionError(c) {
1059
+ return {
1060
+ message: c.error,
1061
+ ...c.code !== void 0 ? {
1062
+ code: c.code
1063
+ } : {},
1064
+ ...c.details !== void 0 ? {
1065
+ details: c.details
1066
+ } : {}
1067
+ };
1068
+ }
1069
+ __name(completionError, "completionError");
1004
1070
 
1005
1071
  // src/concurrency.ts
1006
1072
  var DEFAULT_CONCURRENCY_STATUSES = [
@@ -1246,8 +1312,9 @@ function allChildId(runId, firstSeq, index, wave) {
1246
1312
  return `${allBase(runId, firstSeq, wave)}${index}`;
1247
1313
  }
1248
1314
  __name(allChildId, "allChildId");
1249
- function allGroup(firstSeq, mode) {
1250
- return mode === "failFast" ? `all:${firstSeq}:failFast` : `all:${firstSeq}`;
1315
+ function allGroup(firstSeq, size, mode) {
1316
+ const group = `all:${firstSeq}:n${size}`;
1317
+ return mode === "failFast" ? `${group}:failFast` : group;
1251
1318
  }
1252
1319
  __name(allGroup, "allGroup");
1253
1320
  function stepFanGroup(firstSeq, size, mode) {
@@ -1256,14 +1323,26 @@ function stepFanGroup(firstSeq, size, mode) {
1256
1323
  }
1257
1324
  __name(stepFanGroup, "stepFanGroup");
1258
1325
  function parseStepFanGroup(group) {
1259
- const m = group ? /^steps:(\d+):(\d+)(?::failFast)?$/.exec(group) : null;
1260
- if (!m) return void 0;
1326
+ const fan = parseFanGroup(group);
1327
+ if (fan?.kind !== "steps" || fan.size === void 0) return void 0;
1261
1328
  return {
1262
- firstSeq: Number(m[1]),
1263
- size: Number(m[2])
1329
+ firstSeq: fan.firstSeq,
1330
+ size: fan.size
1264
1331
  };
1265
1332
  }
1266
1333
  __name(parseStepFanGroup, "parseStepFanGroup");
1334
+ function parseFanGroup(group) {
1335
+ const m = group ? /^(all|steps):(\d+)(?::n?(\d+))?(:failFast)?$/.exec(group) : null;
1336
+ if (!m) return void 0;
1337
+ if (m[1] === "steps" && m[3] === void 0) return void 0;
1338
+ return {
1339
+ kind: m[1],
1340
+ firstSeq: Number(m[2]),
1341
+ size: m[3] === void 0 ? void 0 : Number(m[3]),
1342
+ failFast: m[4] !== void 0
1343
+ };
1344
+ }
1345
+ __name(parseFanGroup, "parseFanGroup");
1267
1346
  function allWait(runId, firstSeq, wave) {
1268
1347
  const base = allBase(runId, firstSeq, wave);
1269
1348
  return {
@@ -1308,12 +1387,12 @@ function allWaves(size, concurrency) {
1308
1387
  };
1309
1388
  }
1310
1389
  __name(allWaves, "allWaves");
1311
- var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+)$/;
1390
+ var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+|onError\.\d+)$/;
1312
1391
  function parentRunIdOf(runId) {
1313
1392
  return ENGINE_CHILD_ID.exec(runId)?.[1];
1314
1393
  }
1315
1394
  __name(parentRunIdOf, "parentRunIdOf");
1316
- var GROUP = /^all:(\d+)(:failFast)?$/;
1395
+ var GROUP = /^all:(\d+)(?::n\d+)?(:failFast)?$/;
1317
1396
  var REST = /^(?:w(\d+)\.)?\d+$/;
1318
1397
  function fanOfWaiter(waiter) {
1319
1398
  const group = waiter.parallelGroup ? GROUP.exec(waiter.parallelGroup) : null;
@@ -1393,7 +1472,11 @@ function canonicalizeRegistrations(regs) {
1393
1472
  requires: r.requires ? [
1394
1473
  ...new Set(r.requires)
1395
1474
  ].sort() : null,
1396
- origin: r.origin ?? null
1475
+ origin: r.origin ?? null,
1476
+ // Conditional, so a registration that states no routes keeps the hash it had before routes existed.
1477
+ ...r.onError !== void 0 ? {
1478
+ onError: canonicalRoutes(r.onError)
1479
+ } : {}
1397
1480
  }));
1398
1481
  const byJson = /* @__PURE__ */ new Map();
1399
1482
  for (const entry of canonical) byJson.set(JSON.stringify(entry), entry);
@@ -1402,6 +1485,20 @@ function canonicalizeRegistrations(regs) {
1402
1485
  ].sort().map((k) => byJson.get(k));
1403
1486
  }
1404
1487
  __name(canonicalizeRegistrations, "canonicalizeRegistrations");
1488
+ function canonicalRoutes(routes) {
1489
+ const byJson = /* @__PURE__ */ new Map();
1490
+ for (const r of routes) {
1491
+ const entry = {
1492
+ code: r.code ?? null,
1493
+ workflow: r.workflow
1494
+ };
1495
+ byJson.set(JSON.stringify(entry), entry);
1496
+ }
1497
+ return [
1498
+ ...byJson.keys()
1499
+ ].sort().map((k) => byJson.get(k));
1500
+ }
1501
+ __name(canonicalRoutes, "canonicalRoutes");
1405
1502
  function canonicalizeForHash(d) {
1406
1503
  const set = /* @__PURE__ */ __name((xs) => [
1407
1504
  ...new Set(xs)
@@ -1436,13 +1533,13 @@ function fnv1a64Hex(input) {
1436
1533
  const OFFSET = 0xcbf29ce484222325n;
1437
1534
  const PRIME = 0x100000001b3n;
1438
1535
  const MASK = 0xffffffffffffffffn;
1439
- let hash = OFFSET;
1536
+ let hash2 = OFFSET;
1440
1537
  for (let i = 0; i < input.length; i += 1) {
1441
- hash ^= BigInt(input.charCodeAt(i) & 255);
1442
- hash ^= BigInt(input.charCodeAt(i) >> 8 & 255);
1443
- hash = hash * PRIME & MASK;
1538
+ hash2 ^= BigInt(input.charCodeAt(i) & 255);
1539
+ hash2 ^= BigInt(input.charCodeAt(i) >> 8 & 255);
1540
+ hash2 = hash2 * PRIME & MASK;
1444
1541
  }
1445
- return hash.toString(16).padStart(16, "0");
1542
+ return hash2.toString(16).padStart(16, "0");
1446
1543
  }
1447
1544
  __name(fnv1a64Hex, "fnv1a64Hex");
1448
1545
  function descriptorHash(descriptor) {
@@ -1558,6 +1655,20 @@ function aggregateAnnouncements(descriptors) {
1558
1655
  const requires = [
1559
1656
  ...requiresByJson.values()
1560
1657
  ].sort((x, y) => x.join(",").localeCompare(y.join(",")));
1658
+ const routesByJson = /* @__PURE__ */ new Map();
1659
+ for (const a of announcements) {
1660
+ if (a.onError === void 0) continue;
1661
+ const canonical = canonicalRoutes(a.onError).map((r) => r.code === null ? {
1662
+ workflow: r.workflow
1663
+ } : {
1664
+ code: r.code,
1665
+ workflow: r.workflow
1666
+ });
1667
+ routesByJson.set(JSON.stringify(canonical), canonical);
1668
+ }
1669
+ const onError = [
1670
+ ...routesByJson.keys()
1671
+ ].sort().map((k) => routesByJson.get(k) ?? []);
1561
1672
  const disagreements = [];
1562
1673
  if (groups.length > 1) disagreements.push({
1563
1674
  axis: "group",
@@ -1573,6 +1684,14 @@ function aggregateAnnouncements(descriptors) {
1573
1684
  values: requires.map((r) => r.join(","))
1574
1685
  });
1575
1686
  }
1687
+ if (onError.length > 1) {
1688
+ disagreements.push({
1689
+ axis: "onError",
1690
+ values: [
1691
+ ...routesByJson.keys()
1692
+ ].sort()
1693
+ });
1694
+ }
1576
1695
  const first = announcements[0];
1577
1696
  if (!first) continue;
1578
1697
  out.push({
@@ -1585,6 +1704,7 @@ function aggregateAnnouncements(descriptors) {
1585
1704
  groups,
1586
1705
  origins,
1587
1706
  requires,
1707
+ onError,
1588
1708
  runtimes: [
1589
1709
  ...new Set(announcements.map((a) => a.runtime))
1590
1710
  ].sort(),
@@ -1618,6 +1738,7 @@ function observedAnnouncements(heartbeats, covered) {
1618
1738
  ],
1619
1739
  origins: [],
1620
1740
  requires: [],
1741
+ onError: [],
1621
1742
  runtimes: [],
1622
1743
  instances: [
1623
1744
  ...instances
@@ -1976,6 +2097,13 @@ function planDispatch(requires, liveDescriptors, cp, token) {
1976
2097
  __name(planDispatch, "planDispatch");
1977
2098
 
1978
2099
  // src/interfaces.ts
2100
+ function checkpointNameKind(name) {
2101
+ if (name.startsWith("signal:child:")) return "child";
2102
+ if (name.startsWith("fan:")) return "folded";
2103
+ if (name.startsWith("spawn:")) return "spawn";
2104
+ return "step";
2105
+ }
2106
+ __name(checkpointNameKind, "checkpointNameKind");
1979
2107
  var TERMINAL_RUN_STATUSES = [
1980
2108
  "completed",
1981
2109
  "failed",
@@ -1988,6 +2116,71 @@ function isNonEmptyRunScope(scope) {
1988
2116
  }
1989
2117
  __name(isNonEmptyRunScope, "isNonEmptyRunScope");
1990
2118
 
2119
+ // src/on-error.ts
2120
+ var UNROUTABLE_CODES = /* @__PURE__ */ new Set([
2121
+ "non_determinism",
2122
+ "nested_workflow_call"
2123
+ ]);
2124
+ var HANDOFF_SEQ_BASE = 1e9;
2125
+ var onErrorRunId = /* @__PURE__ */ __name((runId, attempt) => `${runId}.onError.${attempt}`, "onErrorRunId");
2126
+ var handoffName = /* @__PURE__ */ __name((handlerId) => `signal:child:${handlerId}`, "handoffName");
2127
+ var retiredHandoffName = /* @__PURE__ */ __name((handlerId) => `onError:retried:${handlerId}`, "retiredHandoffName");
2128
+ function handoffHandlerId(runId, cp) {
2129
+ const prefix = `signal:child:${runId}.onError.`;
2130
+ return cp.seq >= HANDOFF_SEQ_BASE && cp.name.startsWith(prefix) ? cp.name.slice("signal:child:".length) : void 0;
2131
+ }
2132
+ __name(handoffHandlerId, "handoffHandlerId");
2133
+ function liveHandoff(runId, checkpoints) {
2134
+ let live;
2135
+ for (const cp of checkpoints) {
2136
+ if (handoffHandlerId(runId, cp) !== void 0 && (!live || cp.seq > live.seq)) live = cp;
2137
+ }
2138
+ return live;
2139
+ }
2140
+ __name(liveHandoff, "liveHandoff");
2141
+ function routeFor(routes, error2) {
2142
+ if (!routes?.length || error2.code && UNROUTABLE_CODES.has(error2.code)) return void 0;
2143
+ return (error2.code ? routes.find((r) => r.code === error2.code) : void 0) ?? routes.find((r) => r.code === void 0);
2144
+ }
2145
+ __name(routeFor, "routeFor");
2146
+ function assertRoutes(workflow, routes) {
2147
+ const seen = /* @__PURE__ */ new Set();
2148
+ for (const r of routes) {
2149
+ if (seen.has(r.code)) {
2150
+ throw new Error(r.code === void 0 ? `workflow ${workflow}: more than one onError route without a code` : `workflow ${workflow}: more than one onError route for code "${r.code}"`);
2151
+ }
2152
+ seen.add(r.code);
2153
+ }
2154
+ }
2155
+ __name(assertRoutes, "assertRoutes");
2156
+ function codeOf(err) {
2157
+ const e = err;
2158
+ return [
2159
+ e?.code,
2160
+ e?.stepError?.code,
2161
+ e?.error?.code
2162
+ ].find((c) => typeof c === "string");
2163
+ }
2164
+ __name(codeOf, "codeOf");
2165
+ function detailsOf(err) {
2166
+ const e = err;
2167
+ return e?.details ?? e?.stepError?.details ?? e?.error?.details;
2168
+ }
2169
+ __name(detailsOf, "detailsOf");
2170
+ function codeAndDetails(err) {
2171
+ const code = codeOf(err);
2172
+ const details = detailsOf(err);
2173
+ return {
2174
+ ...code !== void 0 ? {
2175
+ code
2176
+ } : {},
2177
+ ...details !== void 0 ? {
2178
+ details
2179
+ } : {}
2180
+ };
2181
+ }
2182
+ __name(codeAndDetails, "codeAndDetails");
2183
+
1991
2184
  // src/tenant-group.ts
1992
2185
  function tenantGroup(baseGroup, tenant) {
1993
2186
  return tenant !== void 0 && tenant !== "" && tenant !== "default" ? `${baseGroup}@${tenant}` : baseGroup;
@@ -3047,7 +3240,8 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3047
3240
  kind: "local",
3048
3241
  error: {
3049
3242
  message: err instanceof Error ? err.message : String(err),
3050
- stack: err instanceof Error ? err.stack : void 0
3243
+ stack: err instanceof Error ? err.stack : void 0,
3244
+ ...codeAndDetails(err)
3051
3245
  },
3052
3246
  events,
3053
3247
  attempts: attempt,
@@ -3303,17 +3497,17 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3303
3497
  }
3304
3498
  ]));
3305
3499
  }, "child");
3306
- const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
3500
+ const fan = /* @__PURE__ */ __name(async (workflow2, inputs, opts, settle) => {
3307
3501
  if (inputs.length === 0) return [];
3308
3502
  if (stepNameOf(workflow2) !== void 0) {
3309
- return allSteps(workflow2, inputs, opts);
3503
+ return allSteps(workflow2, inputs, opts, settle);
3310
3504
  }
3311
- const mode = opts?.mode ?? "waitAll";
3505
+ const mode = settle ? "waitAll" : opts?.mode ?? "waitAll";
3312
3506
  const name = workflowName(workflow2);
3313
3507
  const waves = allWaves(inputs.length, opts?.concurrency);
3314
3508
  const positions = inputs.map(() => pos.next());
3315
3509
  const firstSeq = positions[0];
3316
- const group = allGroup(firstSeq, mode);
3510
+ const group = allGroup(firstSeq, inputs.length, mode);
3317
3511
  const waveOf = /* @__PURE__ */ __name((i) => waves ? Math.floor(i / waves.waveSize) : void 0, "waveOf");
3318
3512
  const id = /* @__PURE__ */ __name((i) => allChildId(runId, firstSeq, i, waveOf(i)), "id");
3319
3513
  const { checkpoints: existing, folded } = await readFan(positions, [
@@ -3430,24 +3624,45 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3430
3624
  throw new WorkflowSuspended(void 0, fanJoin(active.map((k) => allWait(runId, firstSeq, waves ? k : void 0))));
3431
3625
  }
3432
3626
  const outputs = [];
3627
+ const results = [];
3433
3628
  const failures = [];
3434
3629
  for (let i = 0; i < inputs.length; i += 1) {
3435
3630
  const c = existing[i]?.output;
3436
3631
  if (c && typeof c === "object" && "ok" in c && c.ok === false) {
3632
+ const { message, ...rest } = completionError({
3633
+ ...c,
3634
+ error: c.error ?? "unknown"
3635
+ });
3636
+ const failure = {
3637
+ id: id(i),
3638
+ error: message,
3639
+ ...rest
3640
+ };
3437
3641
  failures.push({
3438
3642
  index: i,
3439
- id: id(i),
3440
- error: c.error ?? "unknown"
3643
+ ...failure
3644
+ });
3645
+ results.push({
3646
+ status: "rejected",
3647
+ ...failure
3441
3648
  });
3442
3649
  outputs.push(void 0);
3443
3650
  } else {
3444
- outputs.push(c?.value);
3651
+ const value = c?.value;
3652
+ results.push({
3653
+ status: "fulfilled",
3654
+ value
3655
+ });
3656
+ outputs.push(value);
3445
3657
  }
3446
3658
  }
3659
+ if (settle) return results;
3447
3660
  if (failures.length > 0) throw new GatherError(failures, inputs.length);
3448
3661
  return outputs;
3449
- }, "all");
3450
- const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts) => {
3662
+ }, "fan");
3663
+ const all = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, false), "all");
3664
+ const allSettled = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, true), "allSettled");
3665
+ const allSteps = /* @__PURE__ */ __name(async (ref, inputs, opts, settle) => {
3451
3666
  if (opts?.queue !== void 0) {
3452
3667
  throw new Error("ctx.all: a flow-control queue is not supported on a step fan");
3453
3668
  }
@@ -3458,7 +3673,8 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3458
3673
  };
3459
3674
  const positions = inputs.map(() => pos.next());
3460
3675
  return host.callRemoteAll(runId, positions, def, inputs, {
3461
- mode: opts?.mode ?? "waitAll",
3676
+ mode: settle ? "waitAll" : opts?.mode ?? "waitAll",
3677
+ settle,
3462
3678
  concurrency: opts?.concurrency,
3463
3679
  transport: opts?.transport,
3464
3680
  admission: {
@@ -3776,6 +3992,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
3776
3992
  task: guard("task", task),
3777
3993
  child: guard("child", child),
3778
3994
  all: guard("all", all),
3995
+ allSettled: guard("allSettled", allSettled),
3779
3996
  startChild: guard("startChild", startChild),
3780
3997
  breakpoint: guard("breakpoint", breakpoint),
3781
3998
  webhook: guardedWebhook,
@@ -3801,9 +4018,11 @@ function mergeTags(staticTags, runTags) {
3801
4018
  __name(mergeTags, "mergeTags");
3802
4019
  var BREAKPOINT = "breakpoint";
3803
4020
  var isBreakpoint = /* @__PURE__ */ __name((cp) => cp.status === "pending" && cp.name.startsWith(BREAKPOINT), "isBreakpoint");
4021
+ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
3804
4022
  var VERSION_UNDECLARED_TAG = "version:undeclared";
3805
4023
  var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
3806
4024
  var STEP_FAN_CHUNK = 200;
4025
+ var RESULT_BATCH_MAX = 256;
3807
4026
  function shuffle(items) {
3808
4027
  for (let i = items.length - 1; i > 0; i -= 1) {
3809
4028
  const j = Math.floor(Math.random() * (i + 1));
@@ -3911,6 +4130,8 @@ var WorkflowEngine = class {
3911
4130
  updateValidators = /* @__PURE__ */ new Map();
3912
4131
  /** Runs being cancelled WITH saga compensation — see `cancel({ compensate: true })`. */
3913
4132
  cancelRequested = /* @__PURE__ */ new Set();
4133
+ /** {@link announcedRoutes} reads, by group + workflow + version. */
4134
+ announcedRoutesCache = /* @__PURE__ */ new Map();
3914
4135
  /**
3915
4136
  * Seqs of the dispatched remote steps a turn suspended on, by run, while that turn executes on
3916
4137
  * THIS instance — so the turn can re-check them once its lease is released (see
@@ -3932,6 +4153,8 @@ var WorkflowEngine = class {
3932
4153
  stepFansFiredHere = /* @__PURE__ */ new Map();
3933
4154
  /** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
3934
4155
  resultChains = /* @__PURE__ */ new Map();
4156
+ /** Each run's results waiting to be settled on this instance — see {@link completeRemoteResult}. */
4157
+ resultQueues = /* @__PURE__ */ new Map();
3935
4158
  /** Flow-control admission backend for remote steps (see {@link registerQueue}). */
3936
4159
  admission;
3937
4160
  /** Runs on THIS instance blocked on admission, by queue — woken early on a freed-slot signal. */
@@ -4138,6 +4361,7 @@ var WorkflowEngine = class {
4138
4361
  name,
4139
4362
  version,
4140
4363
  fn,
4364
+ onError: this.checkedRoutes(name, opts?.onError),
4141
4365
  tags: opts?.tags,
4142
4366
  singleton: opts?.singleton,
4143
4367
  concurrency: opts?.concurrency,
@@ -4207,6 +4431,7 @@ var WorkflowEngine = class {
4207
4431
  executionTimeoutMs: opts.executionTimeout != null ? parseDuration(opts.executionTimeout) : void 0,
4208
4432
  validateInput: opts.validateInput,
4209
4433
  requires: opts.requires,
4434
+ onError: this.checkedRoutes(name, opts.onError),
4210
4435
  remote: {
4211
4436
  group: opts.group,
4212
4437
  executor: opts.executor
@@ -4984,6 +5209,24 @@ var WorkflowEngine = class {
4984
5209
  if (resurrecting) {
4985
5210
  const isFailureCompletion = /* @__PURE__ */ __name((v) => typeof v === "object" && v !== null && v.ok === false, "isFailureCompletion");
4986
5211
  for (const cp of await this.store.listCheckpoints(runId)) {
5212
+ const handlerId = handoffHandlerId(runId, cp);
5213
+ if (handlerId !== void 0) {
5214
+ await this.store.takeSignalWaiter(`child:${handlerId}`);
5215
+ await this.store.saveCheckpoint({
5216
+ ...cp,
5217
+ name: retiredHandoffName(handlerId),
5218
+ status: "completed"
5219
+ });
5220
+ const tags = run.tags?.filter((t) => !t.startsWith("onError:"));
5221
+ if (tags && tags.length !== run.tags?.length) {
5222
+ await this.store.updateRun(runId, {
5223
+ tags,
5224
+ updatedAt: /* @__PURE__ */ new Date()
5225
+ });
5226
+ run.tags = tags;
5227
+ }
5228
+ continue;
5229
+ }
4987
5230
  if (cp.status === "failed") {
4988
5231
  await this.store.saveCheckpoint({
4989
5232
  ...cp,
@@ -5128,13 +5371,13 @@ var WorkflowEngine = class {
5128
5371
  * `start` when a call site needs the outcome — `await start(...); const r = await waitForRun(id)`.
5129
5372
  */
5130
5373
  waitForRun(runId, opts) {
5131
- const TERMINAL = /* @__PURE__ */ new Set([
5374
+ const TERMINAL2 = /* @__PURE__ */ new Set([
5132
5375
  "completed",
5133
5376
  "failed",
5134
5377
  "cancelled",
5135
5378
  "dead"
5136
5379
  ]);
5137
- const isSettled = /* @__PURE__ */ __name((run) => opts?.until === "terminal" ? TERMINAL.has(run.status) : run.status !== "pending" && run.status !== "running" && run.status !== "cancelling" && !(run.status === "suspended" && run.awaitingDecisionTaskId !== void 0), "isSettled");
5380
+ const isSettled = /* @__PURE__ */ __name((run) => opts?.until === "terminal" ? TERMINAL2.has(run.status) : run.status !== "pending" && run.status !== "running" && run.status !== "cancelling" && !(run.status === "suspended" && run.awaitingDecisionTaskId !== void 0), "isSettled");
5138
5381
  const toResult = /* @__PURE__ */ __name((run) => ({
5139
5382
  runId,
5140
5383
  status: run.status,
@@ -5389,10 +5632,7 @@ var WorkflowEngine = class {
5389
5632
  async redriveSettledStepFans(runId, fans) {
5390
5633
  for (const fan of fans) {
5391
5634
  if (this.stepFansFiredHere.has(`${runId}:${fan.from}`)) continue;
5392
- const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(runId, fan.from, fan.to, [
5393
- "pending"
5394
- ]) : (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= fan.from && c.seq <= fan.to && c.status === "pending").length;
5395
- if (pending > 0) continue;
5635
+ if (await this.stepFanPending(runId, fan.from, fan.to) > 0) continue;
5396
5636
  const nowMs = this.clock();
5397
5637
  if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
5398
5638
  return;
@@ -6102,6 +6342,209 @@ var WorkflowEngine = class {
6102
6342
  * us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
6103
6343
  */
6104
6344
  /**
6345
+ * The registered onError routes whose handler workflow nothing can run: not registered on this
6346
+ * engine, and no live worker group serves it. Meant for a boot-time warning: a typo in a handler's
6347
+ * name would otherwise surface only when the first run fails.
6348
+ */
6349
+ async unservedOnErrorRoutes() {
6350
+ const unserved = [];
6351
+ let groups;
6352
+ for (const registered of this.workflows.values()) {
6353
+ for (const route of registered.onError ?? []) {
6354
+ if (this.latest.has(route.workflow)) continue;
6355
+ groups ??= await this.pool.listWorkerGroups().catch(() => []);
6356
+ const token = sanitizeQueueToken(route.workflow);
6357
+ if (groups.some((g) => g === token || g.startsWith(`${token}@`))) continue;
6358
+ unserved.push({
6359
+ workflow: registered.name,
6360
+ route
6361
+ });
6362
+ }
6363
+ }
6364
+ return unserved;
6365
+ }
6366
+ /** Validate a workflow's {@link OnErrorRoute}s at registration. */
6367
+ checkedRoutes(name, routes) {
6368
+ if (!routes?.length) return void 0;
6369
+ assertRoutes(name, routes);
6370
+ return routes;
6371
+ }
6372
+ /**
6373
+ * Route a failed run to its {@link OnErrorRoute} handler, read from the workflow's local registration
6374
+ * or, for one served by convention, from what its live workers announce. Starts the handler run and
6375
+ * waits for it the way a parent waits for a `ctx.child`: a `child:<handlerId>` waiter plus a
6376
+ * `running` placeholder checkpoint carrying the handler's name and input, so the handler's terminal
6377
+ * state wakes the run (buffered and reconciled like any child). From then on every execution of the
6378
+ * run only follows the handler ({@link followHandoff}); the body never runs again. Undefined when no
6379
+ * route matches: the run fails as usual.
6380
+ *
6381
+ * The placeholder sits past every seq the body used and is left out of {@link remoteHistory}.
6382
+ */
6383
+ async routeFailure(run, error2) {
6384
+ const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
6385
+ const routes = registered ? registered.onError : await this.announcedRoutes(run, error2);
6386
+ const route = routeFor(routes, error2);
6387
+ if (!route) return void 0;
6388
+ const checkpoints = await this.handoffCheckpoints(run.id);
6389
+ const live = liveHandoff(run.id, checkpoints);
6390
+ if (live) return this.followHandoff(run, live);
6391
+ const attempt = checkpoints.length;
6392
+ const seq = HANDOFF_SEQ_BASE + attempt;
6393
+ const handlerId = onErrorRunId(run.id, attempt);
6394
+ const handler = {
6395
+ workflow: route.workflow,
6396
+ input: {
6397
+ runId: run.id,
6398
+ workflow: run.workflow,
6399
+ input: run.input,
6400
+ error: error2
6401
+ }
6402
+ };
6403
+ await this.store.putSignalWaiter({
6404
+ token: `child:${handlerId}`,
6405
+ runId: run.id,
6406
+ seq
6407
+ });
6408
+ await this.store.saveCheckpoint({
6409
+ ...instantCheckpoint({
6410
+ runId: run.id,
6411
+ seq,
6412
+ name: handoffName(handlerId),
6413
+ kind: "signal",
6414
+ status: "running"
6415
+ }),
6416
+ input: handler
6417
+ });
6418
+ const tag = `onError:${route.code ?? "*"}`;
6419
+ if (!run.tags?.includes(tag)) {
6420
+ const tags = [
6421
+ ...run.tags ?? [],
6422
+ tag
6423
+ ];
6424
+ await this.store.updateRun(run.id, {
6425
+ tags,
6426
+ updatedAt: /* @__PURE__ */ new Date()
6427
+ });
6428
+ run.tags = tags;
6429
+ }
6430
+ this.emit({
6431
+ type: "run.routed",
6432
+ runId: run.id,
6433
+ workflow: run.workflow,
6434
+ namespace: run.namespace,
6435
+ name: route.workflow,
6436
+ error: error2
6437
+ });
6438
+ this.startChildDeferred(handler.workflow, handler.input, handlerId, {
6439
+ namespace: run.namespace
6440
+ });
6441
+ return {
6442
+ kind: "suspended"
6443
+ };
6444
+ }
6445
+ /** A run's handoff checkpoints, live and retired: the only ones at {@link HANDOFF_SEQ_BASE} and up. */
6446
+ async handoffCheckpoints(runId) {
6447
+ const handoffs = await this.store.listCheckpointsByNamePrefix?.(runId, [
6448
+ `signal:child:${runId}.onError.`,
6449
+ `onError:retried:${runId}.onError.`
6450
+ ]);
6451
+ return (handoffs ?? await this.store.listCheckpoints(runId)).filter((cp) => cp.seq >= HANDOFF_SEQ_BASE);
6452
+ }
6453
+ /**
6454
+ * Where a run that handed its failure off stands: the handler's outcome once it settled (from the
6455
+ * wake's checkpoint, or off the handler run when the wake never arrived), else still waiting. Starts
6456
+ * the handler if the turn that handed off crashed before its deferred start ran.
6457
+ */
6458
+ async followHandoff(run, handoff) {
6459
+ if (handoff.status === "completed") {
6460
+ const outcome = handoff.output;
6461
+ return outcome.ok ? {
6462
+ kind: "completed",
6463
+ output: outcome.value
6464
+ } : {
6465
+ kind: "failed",
6466
+ error: completionError(outcome)
6467
+ };
6468
+ }
6469
+ const handlerId = handoffHandlerId(run.id, handoff);
6470
+ const handlerRun = await this.store.getRun(handlerId);
6471
+ if (handlerRun?.status === "completed") return {
6472
+ kind: "completed",
6473
+ output: handlerRun.output
6474
+ };
6475
+ if (handlerRun && TERMINAL_RUN_STATUSES.includes(handlerRun.status)) {
6476
+ return {
6477
+ kind: "failed",
6478
+ error: handlerRun.error ?? {
6479
+ message: `onError handler ${handlerId} ${handlerRun.status}`
6480
+ }
6481
+ };
6482
+ }
6483
+ if (!handlerRun) {
6484
+ const handler = handoff.input;
6485
+ this.startChildDeferred(handler.workflow, handler.input, handlerId, {
6486
+ namespace: run.namespace
6487
+ });
6488
+ }
6489
+ return {
6490
+ kind: "suspended"
6491
+ };
6492
+ }
6493
+ /**
6494
+ * The onError routes the live workers announce for a workflow the engine serves by convention: the
6495
+ * one set they agree on, or none when they state none or cannot be read. Workers that announce
6496
+ * different sets (a rollout in progress) route nothing, and the run's failure is reported as a
6497
+ * `run.unrouted` event. Announcers of another version of the workflow are ignored. The read is
6498
+ * cached for {@link ANNOUNCED_ROUTES_TTL_MS}, so a burst of failures costs one descriptor read.
6499
+ */
6500
+ async announcedRoutes(run, error2) {
6501
+ const group = tenantGroup(sanitizeQueueToken(run.workflow), run.namespace);
6502
+ const key = `${group}
6503
+ ${run.workflow}
6504
+ ${run.workflowVersion}`;
6505
+ const now = this.clock();
6506
+ let cached = this.announcedRoutesCache.get(key);
6507
+ if (!cached || now - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
6508
+ cached = {
6509
+ at: now,
6510
+ sets: await this.readAnnouncedRoutes(run, group)
6511
+ };
6512
+ this.announcedRoutesCache.set(key, cached);
6513
+ }
6514
+ if (cached.sets.length > 1) {
6515
+ this.emit({
6516
+ type: "run.unrouted",
6517
+ runId: run.id,
6518
+ workflow: run.workflow,
6519
+ namespace: run.namespace,
6520
+ error: error2,
6521
+ reason: `live workers announce ${cached.sets.length} different onError route sets`
6522
+ });
6523
+ return void 0;
6524
+ }
6525
+ return cached.sets[0];
6526
+ }
6527
+ /** Every distinct onError route set the live workers of `group` announce for the run's workflow. */
6528
+ async readAnnouncedRoutes(run, group) {
6529
+ const transport = await this.pool.transportWithLiveGroup(group);
6530
+ if (!transport?.readWorkerDescriptors) return [];
6531
+ let descriptors;
6532
+ try {
6533
+ descriptors = await transport.readWorkerDescriptors(group);
6534
+ } catch {
6535
+ return [];
6536
+ }
6537
+ const sets = /* @__PURE__ */ new Map();
6538
+ for (const announced of aggregateAnnouncements(descriptors)) {
6539
+ if (announced.name !== run.workflow) continue;
6540
+ if (announced.version !== void 0 && announced.version !== run.workflowVersion) continue;
6541
+ for (const routes of announced.onError) sets.set(JSON.stringify(routes), routes);
6542
+ }
6543
+ return [
6544
+ ...sets.values()
6545
+ ];
6546
+ }
6547
+ /**
6105
6548
  * Apply a terminal/suspended transition once: persist the new run status, emit the matching
6106
6549
  * lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
6107
6550
  * executors funnel their completed/failed/suspended outcomes through here so a status update can
@@ -6135,9 +6578,12 @@ var WorkflowEngine = class {
6135
6578
  };
6136
6579
  }
6137
6580
  if (outcome.kind === "failed") {
6581
+ const handed = await this.routeFailure(run, outcome.error);
6582
+ if (handed && handed.kind !== "failed") return this.settleRun(run, handed);
6583
+ const error2 = handed?.error ?? outcome.error;
6138
6584
  await this.store.updateRun(run.id, {
6139
6585
  status: "failed",
6140
- error: outcome.error,
6586
+ error: error2,
6141
6587
  updatedAt
6142
6588
  });
6143
6589
  this.emit({
@@ -6145,16 +6591,13 @@ var WorkflowEngine = class {
6145
6591
  runId: run.id,
6146
6592
  workflow: run.workflow,
6147
6593
  namespace: run.namespace,
6148
- error: outcome.error
6149
- });
6150
- void this.notifyParent(run.id, {
6151
- ok: false,
6152
- error: outcome.error.message
6594
+ error: error2
6153
6595
  });
6596
+ void this.notifyParent(run.id, failedCompletion(error2));
6154
6597
  return {
6155
6598
  runId: run.id,
6156
6599
  status: "failed",
6157
- error: outcome.error
6600
+ error: error2
6158
6601
  };
6159
6602
  }
6160
6603
  if (outcome.kind === "blocked") return this.parkBlocked(run, outcome.blocked);
@@ -6243,7 +6686,14 @@ var WorkflowEngine = class {
6243
6686
  }
6244
6687
  const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
6245
6688
  if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
6246
- const history = await this.remoteHistory(run.id);
6689
+ const checkpoints = await this.store.listCheckpoints(run.id);
6690
+ const handoff = liveHandoff(run.id, checkpoints);
6691
+ if (handoff) {
6692
+ const result = await this.settleRun(run, await this.followHandoff(run, handoff));
6693
+ await this.store.releaseRunLock(run.id);
6694
+ return result;
6695
+ }
6696
+ const history = await this.remoteHistory(run.id, checkpoints);
6247
6697
  if (remote.executor.dispatch) {
6248
6698
  const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
6249
6699
  const silenceMs = this.remoteAdvanceSilenceMs;
@@ -6567,8 +7017,8 @@ var WorkflowEngine = class {
6567
7017
  }
6568
7018
  }
6569
7019
  /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
6570
- async remoteHistory(runId) {
6571
- const checkpoints = await this.store.listCheckpoints(runId);
7020
+ async remoteHistory(runId, listed) {
7021
+ const checkpoints = listed ?? await this.store.listCheckpoints(runId);
6572
7022
  const kindOf = {
6573
7023
  remote: "call",
6574
7024
  local: "step",
@@ -6578,15 +7028,14 @@ var WorkflowEngine = class {
6578
7028
  const events = [];
6579
7029
  for (const cp of checkpoints) {
6580
7030
  if (cp.status === "completed" || cp.status === "failed") {
7031
+ if (cp.seq >= HANDOFF_SEQ_BASE) continue;
6581
7032
  if (cp.kind === "signal" && cp.name.startsWith("signal:child:")) {
6582
7033
  const completion = cp.output;
6583
7034
  events.push({
6584
7035
  seq: cp.seq,
6585
7036
  kind: "child",
6586
7037
  output: completion?.ok ? completion.value : void 0,
6587
- error: completion && completion.ok === false ? {
6588
- message: completion.error
6589
- } : void 0
7038
+ error: completion && completion.ok === false ? completionError(completion) : void 0
6590
7039
  });
6591
7040
  continue;
6592
7041
  }
@@ -6867,6 +7316,10 @@ var WorkflowEngine = class {
6867
7316
  this.awaitedStepFans.set(run.id, awaitedFans);
6868
7317
  const ctx = createWorkflowCtx(this.ctxHostFor(replay, run.namespace), run.id, compensations, run.workflow, registered?.searchAttributesSchema);
6869
7318
  try {
7319
+ const handoff = liveHandoff(run.id, snapshot);
7320
+ if (handoff && !this.cancelRequested.has(run.id)) {
7321
+ return await this.settleRun(run, await this.followHandoff(run, handoff));
7322
+ }
6870
7323
  const output = await runInWorkflowCtx(ctx, () => fn(ctx, run.input));
6871
7324
  if (this.cancelRequested.has(run.id)) {
6872
7325
  return await this.compensateAndCancel(run, compensations, replay);
@@ -6929,7 +7382,8 @@ var WorkflowEngine = class {
6929
7382
  }
6930
7383
  const error2 = {
6931
7384
  message: err instanceof Error ? err.message : String(err),
6932
- stack: err instanceof Error ? err.stack : void 0
7385
+ stack: err instanceof Error ? err.stack : void 0,
7386
+ ...codeAndDetails(err)
6933
7387
  };
6934
7388
  const pending = await this.unwindCompensations(run, compensations, replay);
6935
7389
  if (pending) return await this.settleRun(run, {
@@ -7565,7 +8019,8 @@ var WorkflowEngine = class {
7565
8019
  const failure = {
7566
8020
  index: i,
7567
8021
  id: stepId(runId, seq),
7568
- error: err.message
8022
+ error: err.message,
8023
+ ...codeAndDetails(err)
7569
8024
  };
7570
8025
  if (opts.mode === "failFast") throw new GatherError([
7571
8026
  failure
@@ -7587,6 +8042,24 @@ var WorkflowEngine = class {
7587
8042
  });
7588
8043
  throw new WorkflowSuspended(wakeAt);
7589
8044
  }
8045
+ if (opts.settle) {
8046
+ const failed = new Map(failures.map((f) => [
8047
+ f.index,
8048
+ f
8049
+ ]));
8050
+ return Array.from(outputs, (value, i) => {
8051
+ const f = failed.get(i);
8052
+ if (!f) return {
8053
+ status: "fulfilled",
8054
+ value
8055
+ };
8056
+ const { index: _index, ...rejected } = f;
8057
+ return {
8058
+ status: "rejected",
8059
+ ...rejected
8060
+ };
8061
+ });
8062
+ }
7590
8063
  if (failures.length > 0) throw new GatherError(failures, inputs.length);
7591
8064
  return outputs;
7592
8065
  }
@@ -7658,6 +8131,25 @@ var WorkflowEngine = class {
7658
8131
  await this.store.saveCheckpoint(checkpoint);
7659
8132
  return true;
7660
8133
  }
8134
+ /**
8135
+ * {@link savePendingCheckpoint} for checkpoints at distinct seqs of one run: one
8136
+ * {@link StateStore.saveCheckpointsIfPending} call when the store has it, else one write each.
8137
+ * Never rejects as a whole: a write that failed is reported as that entry's error, and the batch
8138
+ * call rejecting means it wrote nothing, so the entries are then written one by one.
8139
+ */
8140
+ async savePendingCheckpoints(checkpoints) {
8141
+ if (checkpoints.length > 1 && this.store.saveCheckpointsIfPending) {
8142
+ try {
8143
+ const wrote = await this.store.saveCheckpointsIfPending(checkpoints);
8144
+ return checkpoints.map((_, i) => wrote[i] === true);
8145
+ } catch {
8146
+ }
8147
+ }
8148
+ const settled = await Promise.allSettled(checkpoints.map((cp) => this.savePendingCheckpoint(cp)));
8149
+ return settled.map((s) => s.status === "fulfilled" ? s.value : {
8150
+ error: s.reason
8151
+ });
8152
+ }
7661
8153
  /** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
7662
8154
  * dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
7663
8155
  * {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
@@ -7696,57 +8188,161 @@ var WorkflowEngine = class {
7696
8188
  * result (the dispatching one may be gone), so the run is crash/scale-safe. Resolves whether the
7697
8189
  * run should now be resumed: not for a duplicate or late delivery (the checkpoint isn't `pending`),
7698
8190
  * a run already finished, or a step fan whose barrier is still closed.
8191
+ *
8192
+ * Results of one run are settled in batches: the first starts a batch, and the ones that arrive
8193
+ * while it is being written form the next. Under load a wide fan's results are written by one
8194
+ * {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked once
8195
+ * per batch, not once per result. A result resolves only once its own write has committed, so the
8196
+ * transport acknowledges nothing that is not durable yet.
7699
8197
  */
7700
- async completeRemoteResult(result) {
7701
- const cp = await this.store.getCheckpoint(result.runId, result.seq);
7702
- if (!cp || cp.status !== "pending") return false;
7703
- void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
7704
- const run = await this.store.getRun(result.runId);
7705
- if (run && (run.status === "cancelled" || run.status === "completed")) return false;
8198
+ completeRemoteResult(result) {
8199
+ return new Promise((resolve, reject) => {
8200
+ const entry = {
8201
+ result,
8202
+ resolve,
8203
+ reject
8204
+ };
8205
+ const queued = this.resultQueues.get(result.runId);
8206
+ if (queued) {
8207
+ queued.push(entry);
8208
+ return;
8209
+ }
8210
+ const queue = [
8211
+ entry
8212
+ ];
8213
+ this.resultQueues.set(result.runId, queue);
8214
+ void this.drainResultQueue(result.runId, queue);
8215
+ });
8216
+ }
8217
+ /** Settle `queue` (a run's queued results) batch by batch until it is empty, then retire it. */
8218
+ async drainResultQueue(runId, queue) {
8219
+ await Promise.resolve();
8220
+ while (queue.length > 0) {
8221
+ const batch = queue.splice(0, RESULT_BATCH_MAX);
8222
+ let outcomes;
8223
+ try {
8224
+ outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
8225
+ } catch (err) {
8226
+ outcomes = batch.map(() => ({
8227
+ error: err
8228
+ }));
8229
+ }
8230
+ batch.forEach((q, i) => {
8231
+ const outcome = outcomes[i] ?? false;
8232
+ if (typeof outcome === "boolean") q.resolve(outcome);
8233
+ else q.reject(outcome.error);
8234
+ });
8235
+ }
8236
+ this.resultQueues.delete(runId);
8237
+ }
8238
+ /**
8239
+ * Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
8240
+ * that resume replays every write of the batch, so a second would only replay it again. A result
8241
+ * whose read or write failed resolves its error, and the transport redelivers just that one.
8242
+ */
8243
+ async completeRemoteResults(runId, results) {
8244
+ const outcomes = results.map(() => false);
8245
+ const seen = /* @__PURE__ */ new Set();
8246
+ const live = results.flatMap((r, i) => {
8247
+ if (seen.has(r.seq)) return [];
8248
+ seen.add(r.seq);
8249
+ return [
8250
+ i
8251
+ ];
8252
+ });
8253
+ const reads = await Promise.allSettled(live.map((i) => this.store.getCheckpoint(runId, results[i].seq)));
8254
+ const pending = [];
8255
+ reads.forEach((read, k) => {
8256
+ const index = live[k];
8257
+ if (read.status === "rejected") outcomes[index] = {
8258
+ error: read.reason
8259
+ };
8260
+ else if (read.value?.status === "pending") pending.push({
8261
+ index,
8262
+ cp: read.value
8263
+ });
8264
+ });
8265
+ if (pending.length === 0) return outcomes;
8266
+ for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
8267
+ const run = await this.store.getRun(runId);
8268
+ if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
7706
8269
  const finishedAt = /* @__PURE__ */ new Date();
7707
- const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
7708
- const claimed = await this.savePendingCheckpoint({
7709
- ...cp,
7710
- status: result.status,
7711
- output: result.status === "completed" ? result.output : cp.output,
7712
- error: result.error,
7713
- events: result.events ?? cp.events,
7714
- startedAt,
7715
- finishedAt
8270
+ const writes = pending.map(({ index, cp }) => {
8271
+ const result = results[index];
8272
+ return {
8273
+ ...cp,
8274
+ status: result.status,
8275
+ output: result.status === "completed" ? result.output : cp.output,
8276
+ error: result.error,
8277
+ events: result.events ?? cp.events,
8278
+ startedAt: result.startedAt ? new Date(result.startedAt) : cp.startedAt,
8279
+ finishedAt
8280
+ };
7716
8281
  });
7717
- if (!claimed) return false;
7718
- this.noteSettledHere(`${result.runId}:${result.seq}`);
7719
- this.emit({
7720
- type: result.status === "completed" ? "step.completed" : "step.failed",
7721
- runId: result.runId,
7722
- seq: result.seq,
7723
- name: cp.name,
7724
- kind: cp.kind,
7725
- output: result.output,
7726
- error: result.error,
7727
- queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
7728
- durationMs: finishedAt.getTime() - startedAt.getTime()
8282
+ const claims = await this.savePendingCheckpoints(writes);
8283
+ const settled = [];
8284
+ claims.forEach((claim, k) => {
8285
+ const { index, cp } = pending[k];
8286
+ if (claim === true) settled.push({
8287
+ index,
8288
+ cp,
8289
+ result: results[index]
8290
+ });
8291
+ else if (claim !== false) outcomes[index] = claim;
7729
8292
  });
7730
- return !await this.stepFanStillRunning(cp, result);
8293
+ for (const { cp, result } of settled) {
8294
+ this.noteSettledHere(`${runId}:${result.seq}`);
8295
+ const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
8296
+ this.emit({
8297
+ type: result.status === "completed" ? "step.completed" : "step.failed",
8298
+ runId,
8299
+ seq: result.seq,
8300
+ name: cp.name,
8301
+ kind: cp.kind,
8302
+ output: result.output,
8303
+ error: result.error,
8304
+ queueMs: startedAt.getTime() - cp.enqueuedAt.getTime(),
8305
+ durationMs: finishedAt.getTime() - startedAt.getTime()
8306
+ });
8307
+ }
8308
+ const barriers = /* @__PURE__ */ new Map();
8309
+ let carrier;
8310
+ for (const { index, cp, result } of settled) {
8311
+ if (!await this.stepFanStillRunning(cp, result, barriers)) carrier = index;
8312
+ }
8313
+ if (carrier !== void 0) outcomes[carrier] = true;
8314
+ return outcomes;
7731
8315
  }
7732
8316
  /**
7733
8317
  * The join barrier of a step fan: a result for one of its steps resumes the parent only once no
7734
8318
  * step of the fan is still `pending` (a settled wave), or on a failure (failFast surfaces it, and a
7735
8319
  * retry needs the parent to schedule it). Without it every result would replay the whole parent,
7736
- * which makes a fan of N steps cost O(N²) checkpoint reads.
8320
+ * which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
8321
+ * for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
8322
+ * re-suspends, a lost one strands the run.
7737
8323
  */
7738
- async stepFanStillRunning(cp, result) {
8324
+ async stepFanStillRunning(cp, result, barriers) {
7739
8325
  const fan = parseStepFanGroup(cp.parallelGroup);
7740
8326
  if (!fan) return false;
7741
8327
  if (result.status === "failed") return false;
7742
- const to = fan.firstSeq + fan.size - 1;
7743
- const pending = this.store.countCheckpoints ? await this.store.countCheckpoints(result.runId, fan.firstSeq, to, [
7744
- "pending"
7745
- ]) : (await this.store.listCheckpoints(result.runId)).filter((c) => c.seq >= fan.firstSeq && c.seq <= to && c.status === "pending").length;
7746
- if (pending > 0) return true;
8328
+ let closed = barriers.get(fan.firstSeq);
8329
+ if (!closed) {
8330
+ closed = this.stepFanPending(result.runId, fan.firstSeq, fan.firstSeq + fan.size - 1).then((pending) => pending > 0, () => false);
8331
+ barriers.set(fan.firstSeq, closed);
8332
+ }
8333
+ if (await closed) return true;
7747
8334
  this.noteRecent(this.stepFansFiredHere, `${result.runId}:${fan.firstSeq}`);
7748
8335
  return false;
7749
8336
  }
8337
+ /** How many steps of a step fan's seq range are still `pending`. */
8338
+ async stepFanPending(runId, from, to) {
8339
+ if (this.store.countCheckpoints) {
8340
+ return this.store.countCheckpoints(runId, from, to, [
8341
+ "pending"
8342
+ ]);
8343
+ }
8344
+ return (await this.store.listCheckpoints(runId)).filter((c) => c.seq >= from && c.seq <= to && c.status === "pending").length;
8345
+ }
7750
8346
  /**
7751
8347
  * Apply a remote workflow-turn {@link WorkflowDecision} durably — runs on WHATEVER instance receives
7752
8348
  * it over the broker (the dispatching one may be gone, or a point-to-point broker handed it to a
@@ -8098,6 +8694,435 @@ var RunGateway = class {
8098
8694
  }
8099
8695
  };
8100
8696
 
8697
+ // src/run-detail-summary.ts
8698
+ var FAN_SUMMARY_FAILURES = 20;
8699
+ var SUMMARY_CHILD_LIMIT = 200;
8700
+ var FAN_PAGE_MAX = 500;
8701
+ function foldedCount(cp) {
8702
+ if (!cp.name.startsWith(FOLDED_PREFIX)) return void 0;
8703
+ const values = cp.output?.values;
8704
+ return Array.isArray(values) ? values.length : 0;
8705
+ }
8706
+ __name(foldedCount, "foldedCount");
8707
+ function isFailedMember(cp) {
8708
+ if (cp.status === "failed") return true;
8709
+ if (!cp.name.startsWith("signal:child:") || cp.status !== "completed") return false;
8710
+ return cp.output?.ok === false;
8711
+ }
8712
+ __name(isFailedMember, "isFailedMember");
8713
+ function fanLabel(members) {
8714
+ if (members.some((cp) => cp.name.startsWith("signal:child:") || foldedCount(cp) !== void 0)) {
8715
+ return "child runs";
8716
+ }
8717
+ const first = members[0]?.name ?? "";
8718
+ return members.every((cp) => cp.name === first) ? first : "steps";
8719
+ }
8720
+ __name(fanLabel, "fanLabel");
8721
+ function summarizeFan(group, members) {
8722
+ const counts = {};
8723
+ const failures = [];
8724
+ let size = 0;
8725
+ let startedAt;
8726
+ let finishedAt;
8727
+ let firstSeq = Number.POSITIVE_INFINITY;
8728
+ let lastSeq = Number.NEGATIVE_INFINITY;
8729
+ for (const cp of members) {
8730
+ firstSeq = Math.min(firstSeq, cp.seq);
8731
+ lastSeq = Math.max(lastSeq, cp.seq);
8732
+ const folded = foldedCount(cp);
8733
+ const n = folded ?? 1;
8734
+ size += n;
8735
+ const status = isFailedMember(cp) ? "failed" : cp.status;
8736
+ counts[status] = (counts[status] ?? 0) + n;
8737
+ if (status === "failed" && failures.length < FAN_SUMMARY_FAILURES) failures.push(cp);
8738
+ if (cp.status !== "pending" && cp.status !== "running") {
8739
+ if (!startedAt || cp.startedAt < startedAt) startedAt = cp.startedAt;
8740
+ if (!finishedAt || cp.finishedAt > finishedAt) finishedAt = cp.finishedAt;
8741
+ } else if (!startedAt || cp.enqueuedAt < startedAt) {
8742
+ startedAt = cp.enqueuedAt;
8743
+ }
8744
+ }
8745
+ const parsed = parseFanGroup(group);
8746
+ const total = parsed?.size;
8747
+ return {
8748
+ group,
8749
+ label: fanLabel(members),
8750
+ firstSeq,
8751
+ lastSeq,
8752
+ size,
8753
+ total,
8754
+ notStarted: total === void 0 ? void 0 : Math.max(0, total - size),
8755
+ failFast: parsed?.failFast ?? false,
8756
+ counts,
8757
+ startedAt,
8758
+ finishedAt,
8759
+ failures
8760
+ };
8761
+ }
8762
+ __name(summarizeFan, "summarizeFan");
8763
+ function summarizeTimeline(timeline, over) {
8764
+ const byGroup = /* @__PURE__ */ new Map();
8765
+ for (const cp of timeline) {
8766
+ if (cp.parallelGroup === void 0) continue;
8767
+ const members = byGroup.get(cp.parallelGroup);
8768
+ if (members) members.push(cp);
8769
+ else byGroup.set(cp.parallelGroup, [
8770
+ cp
8771
+ ]);
8772
+ }
8773
+ const summarized = /* @__PURE__ */ new Set();
8774
+ const fans = [];
8775
+ for (const [group, members] of byGroup) {
8776
+ if (members.length <= over && (parseFanGroup(group)?.size ?? 0) <= over) continue;
8777
+ summarized.add(group);
8778
+ fans.push(summarizeFan(group, members));
8779
+ }
8780
+ if (fans.length === 0) return {
8781
+ timeline,
8782
+ fans
8783
+ };
8784
+ return {
8785
+ timeline: timeline.filter((cp) => cp.parallelGroup === void 0 || !summarized.has(cp.parallelGroup)),
8786
+ fans: fans.sort((a, b) => a.firstSeq - b.firstSeq)
8787
+ };
8788
+ }
8789
+ __name(summarizeTimeline, "summarizeTimeline");
8790
+ function pageFan(timeline, group, query) {
8791
+ const matching = timeline.filter((cp) => cp.parallelGroup === group && (!query.failedOnly || isFailedMember(cp)));
8792
+ const { offset, limit } = fanPageBounds(query);
8793
+ return {
8794
+ items: matching.slice(offset, offset + limit),
8795
+ total: matching.length
8796
+ };
8797
+ }
8798
+ __name(pageFan, "pageFan");
8799
+ function fanPageBounds(query) {
8800
+ const offset = query.offset !== void 0 && Number.isFinite(query.offset) ? Math.max(0, Math.floor(query.offset)) : 0;
8801
+ const limit = query.limit !== void 0 && Number.isFinite(query.limit) ? Math.min(FAN_PAGE_MAX, Math.max(1, Math.floor(query.limit))) : 100;
8802
+ return {
8803
+ offset,
8804
+ limit
8805
+ };
8806
+ }
8807
+ __name(fanPageBounds, "fanPageBounds");
8808
+
8809
+ // src/run-detail-store.ts
8810
+ function aggregatesCheckpoints(store) {
8811
+ return store.checkpointStats !== void 0 && store.queryCheckpoints !== void 0;
8812
+ }
8813
+ __name(aggregatesCheckpoints, "aggregatesCheckpoints");
8814
+ function checkpointStatsOf(checkpoints, opts) {
8815
+ const buckets = /* @__PURE__ */ new Map();
8816
+ for (const cp of checkpoints) {
8817
+ const group = cp.parallelGroup ?? null;
8818
+ if (opts?.group !== void 0 && group !== opts.group) continue;
8819
+ const nameKind = checkpointNameKind(cp.name);
8820
+ const key = JSON.stringify([
8821
+ group,
8822
+ cp.status,
8823
+ nameKind
8824
+ ]);
8825
+ const enqueuedAt = cp.enqueuedAt ?? cp.startedAt;
8826
+ const bucket = buckets.get(key);
8827
+ if (!bucket) {
8828
+ buckets.set(key, {
8829
+ group,
8830
+ status: cp.status,
8831
+ nameKind,
8832
+ count: 1,
8833
+ minSeq: cp.seq,
8834
+ maxSeq: cp.seq,
8835
+ minName: cp.name,
8836
+ maxName: cp.name,
8837
+ minEnqueuedAt: enqueuedAt,
8838
+ minStartedAt: cp.startedAt,
8839
+ maxStartedAt: cp.startedAt,
8840
+ maxFinishedAt: cp.finishedAt,
8841
+ sumAttempts: cp.attempts
8842
+ });
8843
+ continue;
8844
+ }
8845
+ bucket.count += 1;
8846
+ bucket.minSeq = Math.min(bucket.minSeq, cp.seq);
8847
+ bucket.maxSeq = Math.max(bucket.maxSeq, cp.seq);
8848
+ if (cp.name < bucket.minName) bucket.minName = cp.name;
8849
+ if (cp.name > bucket.maxName) bucket.maxName = cp.name;
8850
+ if (enqueuedAt < bucket.minEnqueuedAt) bucket.minEnqueuedAt = enqueuedAt;
8851
+ if (cp.startedAt < bucket.minStartedAt) bucket.minStartedAt = cp.startedAt;
8852
+ if (cp.startedAt > bucket.maxStartedAt) bucket.maxStartedAt = cp.startedAt;
8853
+ if (cp.finishedAt > bucket.maxFinishedAt) bucket.maxFinishedAt = cp.finishedAt;
8854
+ bucket.sumAttempts += cp.attempts;
8855
+ }
8856
+ return [
8857
+ ...buckets.values()
8858
+ ];
8859
+ }
8860
+ __name(checkpointStatsOf, "checkpointStatsOf");
8861
+ function queryCheckpointsOf(checkpoints, query) {
8862
+ const excluded = new Set(query.excludeGroups ?? []);
8863
+ const matching = checkpoints.filter((cp) => (query.group === void 0 || cp.parallelGroup === query.group) && (cp.parallelGroup === void 0 || !excluded.has(cp.parallelGroup)) && (query.statuses === void 0 || query.statuses.includes(cp.status)) && (query.namePrefixes === void 0 || query.namePrefixes.some((prefix) => cp.name.startsWith(prefix)))).sort((a, b) => a.seq - b.seq);
8864
+ const offset = query.offset ?? 0;
8865
+ return matching.slice(offset, query.limit === void 0 ? void 0 : offset + query.limit);
8866
+ }
8867
+ __name(queryCheckpointsOf, "queryCheckpointsOf");
8868
+ var TERMINAL = /* @__PURE__ */ __name((status) => status !== "pending" && status !== "running", "TERMINAL");
8869
+ function isWide(group, rows, over) {
8870
+ return rows > over || (parseFanGroup(group)?.size ?? 0) > over;
8871
+ }
8872
+ __name(isWide, "isWide");
8873
+ function mergeBySeq(a, b) {
8874
+ return [
8875
+ ...a,
8876
+ ...b
8877
+ ].sort((x, y) => x.seq - y.seq);
8878
+ }
8879
+ __name(mergeBySeq, "mergeBySeq");
8880
+ async function failedChildren(store, runId, group, buckets) {
8881
+ if (!buckets.some((b) => b.nameKind === "child" && b.status === "completed")) {
8882
+ return {
8883
+ completed: [],
8884
+ failed: []
8885
+ };
8886
+ }
8887
+ const completed = await store.queryCheckpoints(runId, {
8888
+ group,
8889
+ statuses: [
8890
+ "completed"
8891
+ ],
8892
+ namePrefixes: [
8893
+ "signal:child:"
8894
+ ]
8895
+ });
8896
+ return {
8897
+ completed,
8898
+ failed: completed.filter(isFailedMember)
8899
+ };
8900
+ }
8901
+ __name(failedChildren, "failedChildren");
8902
+ async function summarizeFanFromStats(store, runId, group, buckets, folded) {
8903
+ const counts = {};
8904
+ const add = /* @__PURE__ */ __name((status, n) => {
8905
+ counts[status] = (counts[status] ?? 0) + n;
8906
+ }, "add");
8907
+ let size = 0;
8908
+ let startedAt;
8909
+ let finishedAt;
8910
+ let firstSeq = Number.POSITIVE_INFINITY;
8911
+ let lastSeq = Number.NEGATIVE_INFINITY;
8912
+ let minName;
8913
+ let maxName;
8914
+ let childRuns = false;
8915
+ for (const b of buckets) {
8916
+ firstSeq = Math.min(firstSeq, b.minSeq);
8917
+ lastSeq = Math.max(lastSeq, b.maxSeq);
8918
+ if (minName === void 0 || b.minName < minName) minName = b.minName;
8919
+ if (maxName === void 0 || b.maxName > maxName) maxName = b.maxName;
8920
+ if (b.nameKind === "child" || b.nameKind === "folded") childRuns = true;
8921
+ if (TERMINAL(b.status)) {
8922
+ if (!startedAt || b.minStartedAt < startedAt) startedAt = b.minStartedAt;
8923
+ if (!finishedAt || b.maxFinishedAt > finishedAt) finishedAt = b.maxFinishedAt;
8924
+ } else if (!startedAt || b.minEnqueuedAt < startedAt) {
8925
+ startedAt = b.minEnqueuedAt;
8926
+ }
8927
+ if (b.nameKind === "folded") continue;
8928
+ if (b.nameKind === "child" && b.status === "completed") continue;
8929
+ add(b.status, b.count);
8930
+ size += b.count;
8931
+ }
8932
+ for (const cp of folded) {
8933
+ const n = foldedCount(cp) ?? 1;
8934
+ add(isFailedMember(cp) ? "failed" : cp.status, n);
8935
+ size += n;
8936
+ }
8937
+ const children = await failedChildren(store, runId, group, buckets);
8938
+ const failedChildSet = new Set(children.failed);
8939
+ for (const cp of children.completed) add(failedChildSet.has(cp) ? "failed" : "completed", 1);
8940
+ size += children.completed.length;
8941
+ const failedStatusRows = buckets.some((b) => b.status === "failed") ? await store.queryCheckpoints(runId, {
8942
+ group,
8943
+ statuses: [
8944
+ "failed"
8945
+ ],
8946
+ limit: FAN_SUMMARY_FAILURES
8947
+ }) : [];
8948
+ const failures = mergeBySeq(failedStatusRows, children.failed).slice(0, FAN_SUMMARY_FAILURES);
8949
+ const parsed = parseFanGroup(group);
8950
+ const total = parsed?.size;
8951
+ return {
8952
+ group,
8953
+ label: childRuns ? "child runs" : minName === maxName ? minName ?? "" : "steps",
8954
+ firstSeq,
8955
+ lastSeq,
8956
+ size,
8957
+ total,
8958
+ notStarted: total === void 0 ? void 0 : Math.max(0, total - size),
8959
+ failFast: parsed?.failFast ?? false,
8960
+ counts,
8961
+ startedAt,
8962
+ finishedAt,
8963
+ failures
8964
+ };
8965
+ }
8966
+ __name(summarizeFanFromStats, "summarizeFanFromStats");
8967
+ async function firstChildIds(store, runId) {
8968
+ const ids = [];
8969
+ for (let offset = 0; ids.length < SUMMARY_CHILD_LIMIT; offset += SUMMARY_CHILD_LIMIT) {
8970
+ const page = await store.queryCheckpoints(runId, {
8971
+ namePrefixes: CHILD_EDGE_PREFIXES,
8972
+ offset,
8973
+ limit: SUMMARY_CHILD_LIMIT
8974
+ });
8975
+ ids.push(...childIdsOf(page));
8976
+ if (page.length < SUMMARY_CHILD_LIMIT) break;
8977
+ }
8978
+ return ids.slice(0, SUMMARY_CHILD_LIMIT);
8979
+ }
8980
+ __name(firstChildIds, "firstChildIds");
8981
+ async function summarizeRunFromStore(store, runId, over, stats) {
8982
+ const all = stats ?? await store.checkpointStats(runId);
8983
+ const byGroup = /* @__PURE__ */ new Map();
8984
+ let edgeRows = 0;
8985
+ let hasFolded = false;
8986
+ for (const b of all) {
8987
+ if (b.nameKind === "child" || b.nameKind === "spawn") edgeRows += b.count;
8988
+ if (b.nameKind === "folded") hasFolded = true;
8989
+ if (b.group === null) continue;
8990
+ const buckets = byGroup.get(b.group);
8991
+ if (buckets) buckets.push(b);
8992
+ else byGroup.set(b.group, [
8993
+ b
8994
+ ]);
8995
+ }
8996
+ const wide = [
8997
+ ...byGroup
8998
+ ].filter(([group, buckets]) => isWide(group, buckets.reduce((n, b) => n + b.count, 0), over));
8999
+ const [timeline, folded, children] = await Promise.all([
9000
+ store.queryCheckpoints(runId, {
9001
+ excludeGroups: wide.map(([group]) => group)
9002
+ }),
9003
+ hasFolded ? store.queryCheckpoints(runId, {
9004
+ namePrefixes: [
9005
+ FOLDED_PREFIX
9006
+ ]
9007
+ }) : [],
9008
+ edgeRows > 0 || hasFolded ? firstChildIds(store, runId) : []
9009
+ ]);
9010
+ const fans = await Promise.all(wide.map(([group, buckets]) => summarizeFanFromStats(store, runId, group, buckets, folded.filter((cp) => cp.parallelGroup === group))));
9011
+ const childCount = folded.reduce((n, cp) => n + childIdsOf([
9012
+ cp
9013
+ ]).length, edgeRows);
9014
+ return {
9015
+ timeline,
9016
+ fans: fans.sort((a, b) => a.firstSeq - b.firstSeq),
9017
+ children,
9018
+ childCount
9019
+ };
9020
+ }
9021
+ __name(summarizeRunFromStore, "summarizeRunFromStore");
9022
+ async function pageFanFromStore(store, runId, group, query, stats) {
9023
+ const buckets = stats ?? await store.checkpointStats(runId, {
9024
+ group
9025
+ });
9026
+ const { offset, limit } = fanPageBounds(query);
9027
+ if (!query.failedOnly) {
9028
+ const total2 = buckets.reduce((n, b) => n + b.count, 0);
9029
+ const items = total2 > offset ? await store.queryCheckpoints(runId, {
9030
+ group,
9031
+ offset,
9032
+ limit
9033
+ }) : [];
9034
+ return {
9035
+ items,
9036
+ total: total2
9037
+ };
9038
+ }
9039
+ const failedStatus = buckets.filter((b) => b.status === "failed").reduce((n, b) => n + b.count, 0);
9040
+ const children = await failedChildren(store, runId, group, buckets);
9041
+ const total = failedStatus + children.failed.length;
9042
+ if (total <= offset) return {
9043
+ items: [],
9044
+ total
9045
+ };
9046
+ if (children.failed.length === 0) {
9047
+ return {
9048
+ items: await store.queryCheckpoints(runId, {
9049
+ group,
9050
+ statuses: [
9051
+ "failed"
9052
+ ],
9053
+ offset,
9054
+ limit
9055
+ }),
9056
+ total
9057
+ };
9058
+ }
9059
+ const failedRows = await store.queryCheckpoints(runId, {
9060
+ group,
9061
+ statuses: [
9062
+ "failed"
9063
+ ],
9064
+ limit: offset + limit
9065
+ });
9066
+ return {
9067
+ items: mergeBySeq(failedRows, children.failed).slice(offset, offset + limit),
9068
+ total
9069
+ };
9070
+ }
9071
+ __name(pageFanFromStore, "pageFanFromStore");
9072
+ function hash(text) {
9073
+ let h1 = 3735928559;
9074
+ let h2 = 1103547991;
9075
+ for (let i = 0; i < text.length; i++) {
9076
+ const ch = text.charCodeAt(i);
9077
+ h1 = Math.imul(h1 ^ ch, 2654435761);
9078
+ h2 = Math.imul(h2 ^ ch, 1597334677);
9079
+ }
9080
+ h1 = Math.imul(h1 ^ h1 >>> 16, 2246822507) ^ Math.imul(h2 ^ h2 >>> 13, 3266489909);
9081
+ h2 = Math.imul(h2 ^ h2 >>> 16, 2246822507) ^ Math.imul(h1 ^ h1 >>> 13, 3266489909);
9082
+ return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16);
9083
+ }
9084
+ __name(hash, "hash");
9085
+ function statsFingerprint(stats) {
9086
+ return stats.map((b) => [
9087
+ b.group ?? "",
9088
+ b.status,
9089
+ b.nameKind,
9090
+ b.count,
9091
+ b.minSeq,
9092
+ b.maxSeq,
9093
+ b.minName,
9094
+ b.maxName,
9095
+ b.minEnqueuedAt.getTime(),
9096
+ b.minStartedAt.getTime(),
9097
+ b.maxStartedAt.getTime(),
9098
+ b.maxFinishedAt.getTime(),
9099
+ b.sumAttempts
9100
+ ].join("")).sort().join("");
9101
+ }
9102
+ __name(statsFingerprint, "statsFingerprint");
9103
+ function runDetailEtag(run, stats, over) {
9104
+ return `"rd-${hash(`${over}${JSON.stringify(run)}${statsFingerprint(stats)}`)}"`;
9105
+ }
9106
+ __name(runDetailEtag, "runDetailEtag");
9107
+ function fanPageEtag(group, stats, query) {
9108
+ const { offset, limit } = fanPageBounds(query);
9109
+ const key = [
9110
+ group,
9111
+ offset,
9112
+ limit,
9113
+ query.failedOnly === true
9114
+ ].join("");
9115
+ return `"fp-${hash(`${key}${statsFingerprint(stats)}`)}"`;
9116
+ }
9117
+ __name(fanPageEtag, "fanPageEtag");
9118
+ function etagMatches(ifNoneMatch, etag) {
9119
+ if (!ifNoneMatch) return false;
9120
+ const bare = /* @__PURE__ */ __name((tag) => tag.trim().replace(/^W\//, ""), "bare");
9121
+ const target = bare(etag);
9122
+ return ifNoneMatch.split(",").some((tag) => tag.trim() === "*" || bare(tag) === target);
9123
+ }
9124
+ __name(etagMatches, "etagMatches");
9125
+
8101
9126
  // src/codec-state-store.ts
8102
9127
  var CodecStateStore = class {
8103
9128
  static {
@@ -8126,11 +9151,19 @@ var CodecStateStore = class {
8126
9151
  /** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
8127
9152
  * plain-save fallback still kicks in through the wrapper. */
8128
9153
  saveCheckpointIfPending;
9154
+ /** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
9155
+ saveCheckpointsIfPending;
8129
9156
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
8130
9157
  countCheckpoints;
8131
9158
  /** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
8132
9159
  * when the inner store prunes, so the retention poller still sees (and warns about) its absence. */
8133
9160
  pruneTerminalRuns;
9161
+ /** Forwarded verbatim — the buckets aggregate names, statuses and timestamps, never a payload.
9162
+ * Bound only when the inner store aggregates, so the run detail's fallback still applies. */
9163
+ checkpointStats;
9164
+ /** Filters on unencoded columns, then decodes the rows like {@link listCheckpoints}. Bound only
9165
+ * when the inner store has it. */
9166
+ queryCheckpoints;
8134
9167
  /** Forwarded verbatim — a count reads no payload. Bound only when the inner store counts, so the
8135
9168
  * engine's facet/listing fallback still applies through the wrapper. */
8136
9169
  countRuns;
@@ -8157,8 +9190,14 @@ var CodecStateStore = class {
8157
9190
  if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
8158
9191
  const saveIfPending = inner.saveCheckpointIfPending;
8159
9192
  if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
9193
+ const saveManyIfPending = inner.saveCheckpointsIfPending;
9194
+ if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
8160
9195
  const countCps = inner.countCheckpoints;
8161
9196
  if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
9197
+ const stats = inner.checkpointStats;
9198
+ if (stats) this.checkpointStats = (runId, opts) => stats.call(inner, runId, opts);
9199
+ const queryCps = inner.queryCheckpoints;
9200
+ if (queryCps) this.queryCheckpoints = async (runId, query) => (await queryCps.call(inner, runId, query)).map((c) => this.decCp(c));
8162
9201
  const prune = inner.pruneTerminalRuns;
8163
9202
  if (prune) this.pruneTerminalRuns = (policy, now, limit) => prune.call(inner, policy, now, limit);
8164
9203
  const count = inner.countRuns;
@@ -8366,8 +9405,11 @@ function collectMetrics(engine) {
8366
9405
  started: 0,
8367
9406
  completed: 0,
8368
9407
  failed: 0,
8369
- suspended: 0
9408
+ suspended: 0,
9409
+ routed: 0,
9410
+ unrouted: 0
8370
9411
  },
9412
+ routedByCode: {},
8371
9413
  steps: {
8372
9414
  started: 0,
8373
9415
  completed: 0,
@@ -8409,6 +9451,15 @@ function collectMetrics(engine) {
8409
9451
  case "run.suspended":
8410
9452
  m.runs.suspended += 1;
8411
9453
  break;
9454
+ case "run.routed": {
9455
+ m.runs.routed += 1;
9456
+ const code = e.error?.code ?? "*";
9457
+ m.routedByCode[code] = (m.routedByCode[code] ?? 0) + 1;
9458
+ break;
9459
+ }
9460
+ case "run.unrouted":
9461
+ m.runs.unrouted += 1;
9462
+ break;
8412
9463
  case "step.started":
8413
9464
  m.steps.started += 1;
8414
9465
  break;
@@ -8435,6 +9486,8 @@ function collectMetrics(engine) {
8435
9486
  for (const [k, v] of Object.entries(m.steps)) lines.push(`durable_steps_total{event="${k}"} ${v}`);
8436
9487
  lines.push("# TYPE durable_runs_by_workflow_total counter");
8437
9488
  for (const [name, v] of Object.entries(m.byWorkflow)) for (const [event, n] of Object.entries(v)) lines.push(`durable_runs_by_workflow_total{workflow="${name}",event="${event}"} ${n}`);
9489
+ lines.push("# TYPE durable_runs_routed_total counter");
9490
+ for (const [code, n] of Object.entries(m.routedByCode)) lines.push(`durable_runs_routed_total{code="${code}"} ${n}`);
8438
9491
  lines.push("# TYPE durable_step_duration_ms_sum counter");
8439
9492
  lines.push(`durable_step_duration_ms_sum ${m.stepDuration.sumMs}`);
8440
9493
  lines.push("# TYPE durable_step_duration_count counter");
@@ -8605,6 +9658,11 @@ var InMemoryStateStore = class {
8605
9658
  * so this lets the benchmark contrast "fetch N, scan in JS" against "fetch the k that match".
8606
9659
  */
8607
9660
  lastCheckpointRowsRead = 0;
9661
+ /**
9662
+ * Test/observability hook: checkpoint rows materialized since construction, across every
9663
+ * checkpoint read that returns rows — what a run-detail benchmark compares between paths.
9664
+ */
9665
+ checkpointRowsRead = 0;
8608
9666
  key(runId, seq) {
8609
9667
  return `${runId}:${seq}`;
8610
9668
  }
@@ -8730,6 +9788,16 @@ var InMemoryStateStore = class {
8730
9788
  });
8731
9789
  return true;
8732
9790
  }
9791
+ async saveCheckpointsIfPending(checkpoints) {
9792
+ return checkpoints.map((checkpoint) => {
9793
+ const key = this.key(checkpoint.runId, checkpoint.seq);
9794
+ if (this.checkpoints.get(key)?.status !== "pending") return false;
9795
+ this.checkpoints.set(key, {
9796
+ ...checkpoint
9797
+ });
9798
+ return true;
9799
+ });
9800
+ }
8733
9801
  async listIncompleteRuns(namespace) {
8734
9802
  return [
8735
9803
  ...this.runs.values()
@@ -9009,6 +10077,23 @@ var InMemoryStateStore = class {
9009
10077
  ...cp
9010
10078
  }));
9011
10079
  this.lastCheckpointRowsRead = rows.length;
10080
+ this.checkpointRowsRead += rows.length;
10081
+ return rows;
10082
+ }
10083
+ runCheckpoints(runId) {
10084
+ return [
10085
+ ...this.checkpoints.values()
10086
+ ].filter((cp) => cp.runId === runId);
10087
+ }
10088
+ async checkpointStats(runId, opts) {
10089
+ return checkpointStatsOf(this.runCheckpoints(runId), opts);
10090
+ }
10091
+ async queryCheckpoints(runId, query) {
10092
+ const rows = queryCheckpointsOf(this.runCheckpoints(runId), query).map((cp) => ({
10093
+ ...cp
10094
+ }));
10095
+ this.lastCheckpointRowsRead = rows.length;
10096
+ this.checkpointRowsRead += rows.length;
9012
10097
  return rows;
9013
10098
  }
9014
10099
  async getLatestCheckpointByName(runId, name) {
@@ -9036,6 +10121,7 @@ var InMemoryStateStore = class {
9036
10121
  ...cp
9037
10122
  }));
9038
10123
  this.lastCheckpointRowsRead = rows.length;
10124
+ this.checkpointRowsRead += rows.length;
9039
10125
  return rows;
9040
10126
  }
9041
10127
  };
@@ -9087,6 +10173,7 @@ var InMemoryTransport = class {
9087
10173
  };
9088
10174
  // Annotate the CommonJS export names for ESM import in node:
9089
10175
  0 && (module.exports = {
10176
+ CHILD_EDGE_PREFIXES,
9090
10177
  CONTROL_FLOW_SIGNAL,
9091
10178
  CURRENT_PROTOCOL_VERSION,
9092
10179
  CodecStateStore,
@@ -9104,8 +10191,10 @@ var InMemoryTransport = class {
9104
10191
  EVENT_TOKEN_PREFIX,
9105
10192
  Entities,
9106
10193
  EventAccumulators,
10194
+ FAN_SUMMARY_FAILURES,
9107
10195
  FatalError,
9108
10196
  GatherError,
10197
+ HANDOFF_SEQ_BASE,
9109
10198
  InMemoryAdmissionBackend,
9110
10199
  InMemoryStateStore,
9111
10200
  InMemoryTransport,
@@ -9125,6 +10214,7 @@ var InMemoryTransport = class {
9125
10214
  SINGLETON_ADMITTED_TAG,
9126
10215
  STATE_STORE,
9127
10216
  STATE_STORE_CANONICAL,
10217
+ SUMMARY_CHILD_LIMIT,
9128
10218
  ScheduleAlreadyExistsError,
9129
10219
  ScheduleClient,
9130
10220
  ScheduleNotFoundError,
@@ -9139,9 +10229,11 @@ var InMemoryTransport = class {
9139
10229
  WorkflowEngine,
9140
10230
  WorkflowSuspended,
9141
10231
  aggregateAnnouncements,
10232
+ aggregatesCheckpoints,
9142
10233
  announcementsOf,
9143
10234
  applyAttributeQuery,
9144
10235
  assertConcurrency,
10236
+ assertRoutes,
9145
10237
  attributeColumnFor,
9146
10238
  attributeOperand,
9147
10239
  attributePredicateOperands,
@@ -9151,9 +10243,14 @@ var InMemoryTransport = class {
9151
10243
  buildWorkflowDirectory,
9152
10244
  canRoute,
9153
10245
  capableWorkers,
10246
+ checkpointNameKind,
10247
+ checkpointStatsOf,
10248
+ childIdsOf,
9154
10249
  classifyWaiterToken,
9155
10250
  clearDurableExecutionWrappers,
9156
10251
  clearTerminalSingletonAdmission,
10252
+ codeAndDetails,
10253
+ codeOf,
9157
10254
  collectMetrics,
9158
10255
  concurrencyTag,
9159
10256
  controlPlaneDescriptor,
@@ -9166,23 +10263,32 @@ var InMemoryTransport = class {
9166
10263
  debug,
9167
10264
  describeWorker,
9168
10265
  descriptorHash,
10266
+ detailsOf,
9169
10267
  error,
10268
+ etagMatches,
9170
10269
  eventMatchOf,
9171
10270
  eventMatches,
9172
10271
  eventNameOf,
9173
10272
  eventPrefix,
9174
10273
  eventToken,
9175
10274
  facetOrigin,
10275
+ fanPageBounds,
10276
+ fanPageEtag,
10277
+ foldedCount,
10278
+ handoffHandlerId,
10279
+ handoffName,
9176
10280
  hasDurableExecutionWrappers,
9177
10281
  heartbeatStatus,
9178
10282
  indexWaitersByRun,
9179
10283
  info,
9180
10284
  isEngineMintedTag,
9181
10285
  isEventToken,
10286
+ isFailedMember,
9182
10287
  isLegacyDescriptor,
9183
10288
  isNonEmptyRunScope,
9184
10289
  isWorkflowControlFlowSignal,
9185
10290
  listSignalWaitersOfRuns,
10291
+ liveHandoff,
9186
10292
  log,
9187
10293
  matchesAttributes,
9188
10294
  mergeRunFacetRows,
@@ -9193,16 +10299,23 @@ var InMemoryTransport = class {
9193
10299
  normalizeAttributeRows,
9194
10300
  normalizeDescriptor,
9195
10301
  observedAnnouncements,
10302
+ onErrorRunId,
10303
+ pageFan,
10304
+ pageFanFromStore,
9196
10305
  parentRunIdOf,
9197
10306
  parseDuration,
9198
10307
  partitionOfToken,
9199
10308
  planDispatch,
9200
10309
  prevCronFireMs,
10310
+ queryCheckpointsOf,
9201
10311
  readSearchAttributes,
9202
10312
  requiredCapabilities,
9203
10313
  resolveConcurrency,
9204
10314
  resolveRouting,
9205
10315
  resolveRunWaiting,
10316
+ retiredHandoffName,
10317
+ routeFor,
10318
+ runDetailEtag,
9206
10319
  runDurableExecution,
9207
10320
  runInStepLogger,
9208
10321
  runInStepScope,
@@ -9222,6 +10335,9 @@ var InMemoryTransport = class {
9222
10335
  sub,
9223
10336
  subEvent,
9224
10337
  subProcess,
10338
+ summarizeFan,
10339
+ summarizeRunFromStore,
10340
+ summarizeTimeline,
9225
10341
  tenantGroup,
9226
10342
  useDurableExecution,
9227
10343
  warn,