@intx/workflow-host 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +56 -10
  2. package/dist/adapters/repo-store.d.ts +22 -1
  3. package/dist/adapters/repo-store.js +53 -53
  4. package/dist/adapters/spawn-child.d.ts +71 -42
  5. package/dist/adapters/spawn-child.js +83 -77
  6. package/dist/adapters/step-invoker.js +84 -7
  7. package/dist/child/env-bootstrap.d.ts +20 -6
  8. package/dist/child/env-bootstrap.js +9 -1
  9. package/dist/child/index.d.ts +2 -1
  10. package/dist/child/parked-correlations.d.ts +42 -0
  11. package/dist/child/parked-correlations.js +80 -0
  12. package/dist/child/proxy-repo-store.d.ts +3 -2
  13. package/dist/child/proxy-repo-store.js +2 -0
  14. package/dist/child/run-child.d.ts +107 -13
  15. package/dist/child/run-child.js +290 -108
  16. package/dist/child/self-discovery.d.ts +10 -0
  17. package/dist/child/self-discovery.js +25 -1
  18. package/dist/child/verified-definition-loader.d.ts +33 -0
  19. package/dist/child/verified-definition-loader.js +43 -0
  20. package/dist/conversation-text.d.ts +23 -0
  21. package/dist/conversation-text.js +56 -0
  22. package/dist/index.d.ts +4 -3
  23. package/dist/index.js +3 -2
  24. package/dist/ipc/control-channel.d.ts +58 -0
  25. package/dist/ipc/control-channel.js +94 -1
  26. package/dist/ipc/event-channel.d.ts +32 -1
  27. package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
  28. package/dist/mail-bus/hub-transport-adapter.js +9 -5
  29. package/dist/seams/scheduler.d.ts +4 -6
  30. package/dist/seams/scheduler.js +74 -93
  31. package/dist/supervisor/cancel-signing.d.ts +2 -2
  32. package/dist/supervisor/cancel-signing.js +1 -1
  33. package/dist/supervisor/credentials.d.ts +11 -10
  34. package/dist/supervisor/credentials.js +7 -7
  35. package/dist/supervisor/dispatch-attribution.js +1 -1
  36. package/dist/supervisor/drain-timeout.d.ts +2 -2
  37. package/dist/supervisor/drain-timeout.js +1 -1
  38. package/dist/supervisor/index.d.ts +3 -3
  39. package/dist/supervisor/index.js +2 -2
  40. package/dist/supervisor/recycle.d.ts +5 -2
  41. package/dist/supervisor/recycle.js +18 -7
  42. package/dist/supervisor/run-event-compaction.d.ts +5 -5
  43. package/dist/supervisor/run-event-compaction.js +5 -5
  44. package/dist/supervisor/spawn-env.d.ts +2 -2
  45. package/dist/supervisor/spawn-env.js +1 -1
  46. package/dist/supervisor/supervisor.d.ts +82 -25
  47. package/dist/supervisor/supervisor.js +1313 -410
  48. package/dist/supervisor/terminal-commit.d.ts +36 -0
  49. package/dist/supervisor/terminal-commit.js +134 -0
  50. package/dist/supervisor/types.d.ts +150 -23
  51. package/dist/workflow-definition-loader.d.ts +131 -0
  52. package/dist/workflow-definition-loader.js +316 -0
  53. package/package.json +12 -11
@@ -43,49 +43,73 @@ import { type } from "arktype";
43
43
  import { getLogger } from "@intx/log";
44
44
  import { sampleStructuralCounters, forceRepack, } from "./dispatch-attribution.js";
45
45
  import { generateKeyPair } from "@intx/crypto";
46
- import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed, readOwnedMessageIds, replayProcessingToInbox as defaultReplayProcessingToInbox, DEFAULT_CONSUMED_RETENTION_MS, } from "@intx/hub-sessions/substrate";
47
- import { base64Decode, base64Encode, hexEncode } from "@intx/types";
46
+ import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed, readOwnedMessageIds, readWorkflowRunLifecycle, replayProcessingToInbox as defaultReplayProcessingToInbox, StaleInboxEnqueueError, DEFAULT_CONSUMED_RETENTION_MS, } from "@intx/hub-sessions/substrate";
47
+ import { base64Decode, base64Encode, deriveMessageId, deriveWorkflowRunId, signalName, } from "@intx/types";
48
48
  import { RepoId } from "@intx/types/sidecar";
49
49
  import { createControlChannelSender, generateChannelId, generateHmacKey, receiveControlChannel, receiveEventChannel, } from "../ipc/index.js";
50
50
  import { assembleCredentialsSnapshot, } from "./credentials.js";
51
51
  import { commitCancelRequested } from "./cancel-signing.js";
52
+ import { commitRunFailed } from "./terminal-commit.js";
52
53
  import { buildChildSpawnEnv } from "./spawn-env.js";
53
54
  import { compactRunEvents } from "./run-event-compaction.js";
55
+ import { extractConversationText } from "../conversation-text.js";
54
56
  import { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, } from "./drain-timeout.js";
55
57
  import { createRecyclePolicy, triggerRecycle, } from "./recycle.js";
56
58
  import { createTerminalBroadcaster, } from "./terminal-broadcaster.js";
57
59
  import { DEFAULT_KILL_TIMEOUT_MS, DEFAULT_READY_TIMEOUT_MS, defaultClearTimer, defaultSetTimer, killChildHandle, waitDeadline, } from "./child-termination.js";
58
60
  const logger = getLogger(["workflow-host", "supervisor"]);
59
61
  /**
60
- * Default watchdog timeout for the supervisor's
61
- * `synchronouslyDispatchTerminalWrite`. The handler holds the
62
- * `substrate.write.response` back to the child until the dispatch
63
- * loop's `markConsumed` settles for the matching terminal event; an
64
- * unbounded wait would chain into a child / runtime / dispatch loop
65
- * deadlock if `markConsumed` never armed (bug in the dispatch loop, a
66
- * torn-down cohort, a stalled inbox primitive). 30s sits between the
67
- * recycle path's `DEFAULT_KILL_TIMEOUT_MS` (5s, a hard process-level
68
- * kill cap) and `DEFAULT_DRAIN_TIMEOUT_MS` (60s, the per-deployment
69
- * drain budget) -- generous enough to absorb a slow legitimate
70
- * markConsumed, tight enough to surface a real deadlock long before
71
- * the drainTimeout would otherwise mask it.
62
+ * Default crash-loop bound: the supervisor stops respawning and latches
63
+ * the deployment once the workflow-process child exits unexpectedly this
64
+ * many times within `DEFAULT_CRASH_LOOP_WINDOW_MS`. Overridable via
65
+ * `WorkflowSupervisorBindings.crashLoopMaxCount`.
72
66
  */
73
- export const DEFAULT_TERMINAL_WRITE_WATCHDOG_MS = 30_000;
67
+ export const DEFAULT_CRASH_LOOP_MAX_COUNT = 3;
74
68
  /**
75
- * Raised when a `pendingMerges` entry or a
76
- * `markConsumedCompletionWaiters` waiter is rejected because the
77
- * cohort it was registered against has been aborted (cohort transition
78
- * during a recycle, or a supervisor shutdown). Callers awaiting the
79
- * resolved value receive an instance of this error so the failure mode
80
- * is recognisable from a generic substrate-merge or markConsumed
81
- * failure.
69
+ * Default sliding window (ms) over which `DEFAULT_CRASH_LOOP_MAX_COUNT`
70
+ * unexpected exits latch the deployment. Overridable via
71
+ * `WorkflowSupervisorBindings.crashLoopWindowMs`.
82
72
  */
83
- export class MergeAbortedError extends Error {
84
- constructor(reason) {
85
- super(`supervisor cohort aborted before completion: ${reason}`);
86
- this.name = "MergeAbortedError";
87
- }
88
- }
73
+ export const DEFAULT_CRASH_LOOP_WINDOW_MS = 60_000;
74
+ /**
75
+ * Default stable-run duration (ms): once a respawned child stays up this
76
+ * long, the crash counter resets so flapping followed by stability does
77
+ * not permanently latch. Overridable via
78
+ * `WorkflowSupervisorBindings.crashLoopStableResetMs`.
79
+ */
80
+ export const DEFAULT_CRASH_LOOP_STABLE_RESET_MS = 60_000;
81
+ /**
82
+ * Default initial respawn backoff (ms): the wait before the first respawn
83
+ * after an unexpected exit. Overridable via
84
+ * `WorkflowSupervisorBindings.respawnBackoffInitialMs`.
85
+ */
86
+ export const DEFAULT_RESPAWN_BACKOFF_INITIAL_MS = 1_000;
87
+ /**
88
+ * Default cap (ms) on the exponential respawn backoff. Kept below
89
+ * `DEFAULT_CRASH_LOOP_WINDOW_MS` so a slow flapper's crashes still fall
90
+ * within the window and latch the guard. Overridable via
91
+ * `WorkflowSupervisorBindings.respawnBackoffMaxMs`.
92
+ */
93
+ export const DEFAULT_RESPAWN_BACKOFF_MAX_MS = 30_000;
94
+ /**
95
+ * Default watchdog for `reEmitParkedCorrelations`' wait on the child's
96
+ * `parked-correlations.response`. 30s is generous enough for a healthy child
97
+ * to enumerate its in-flight runs and load each parked snapshot, tight
98
+ * enough that a wedged-but-alive child does not hang the reconnect caller
99
+ * until some coarser timeout intervenes.
100
+ */
101
+ export const DEFAULT_PARKED_QUERY_WATCHDOG_MS = 30_000;
102
+ /**
103
+ * Backstop for `waitForRunTerminalOrPark`. A dispatch waits here for the child
104
+ * to park or terminate the run before releasing `markConsumed`; a lost park
105
+ * wake or a wedged child would otherwise hang the deployment's dispatch loop
106
+ * forever. Five minutes is far beyond any healthy per-message dispatch (which
107
+ * settles in well under a second), so this never fires on a legitimately long
108
+ * run without also being a genuine fault -- and when it does fire it is logged
109
+ * loudly and fails the dispatch (the mail is left reclaimable, never consumed
110
+ * on the assumption the run made progress), not silently swallowed.
111
+ */
112
+ export const TERMINAL_OR_PARK_BACKSTOP_MS = 300_000;
89
113
  /**
90
114
  * Construct a per-deployment supervisor. All host-specific
91
115
  * dependencies are pulled in via `bindings`; nothing in the
@@ -94,22 +118,60 @@ export class MergeAbortedError extends Error {
94
118
  export function createWorkflowSupervisor(bindings) {
95
119
  let state = { phase: "idle" };
96
120
  /**
97
- * In-flight runIds the supervisor knows about. A runId enters this
98
- * set when the supervisor forwards a `trigger.fire` for it on the
99
- * control channel; the runId leaves the set when the dispatch
100
- * loop's terminal-event watcher fires `markConsumed`. The drain
101
- * path arms one accumulator per entry here.
121
+ * ALL runIds the current child cohort is driving, regardless of
122
+ * who spawned them: supervisor-dispatched + self-discovered.
123
+ * Populated at `trigger.fire` time and when the child reports
124
+ * `resumed.runs`. Removed on terminal event or cohort teardown.
125
+ *
126
+ * Used by: `drain()` to arm one drainTimeout accumulator per run.
127
+ */
128
+ const cohortRunIds = new Set();
129
+ /**
130
+ * Runs observed terminal in this supervisor process. The terminal control
131
+ * frame follows the durable event commit, but retaining the observation
132
+ * closes the short visibility window before the working tree reflects that
133
+ * commit. Terminal membership is permanent for a deployment: the stable
134
+ * top-level run is never cleared and fired again.
135
+ */
136
+ const terminalRunIds = new Set();
137
+ /**
138
+ * Per-run input channel cache. When a long-lived run parks on an input
139
+ * signal, the child sends `park.notify` with `parkKind: "input"`. The
140
+ * supervisor stores the `correlationId` here so that subsequent mail
141
+ * deliveries can fire `signal.deliver` without re-reading the substrate.
142
+ * Cleared on terminal event or cohort abort.
143
+ */
144
+ const runInputChannels = new Map();
145
+ /**
146
+ * Waiters for dispatch loops blocked on a `park.notify` for a
147
+ * specific runId. When `park.notify` arrives, the handler resolves
148
+ * the waiter so the dispatch loop re-evaluates routing.
149
+ */
150
+ const parkNotifyWaiters = new Map();
151
+ /**
152
+ * Monotonic per-run INPUT-park generation. Bumped on every
153
+ * `park.notify(input)` for a runId. `waitForRunTerminalOrPark` captures a
154
+ * `sinceGen` before its caller's pre-wait awaits and returns `"parked"` when
155
+ * the generation later exceeds it, so the wait keys on the park EDGE rather
156
+ * than `runInputChannels`' LEVEL state: a park that fired during the pre-wait
157
+ * awaits (before the waiter armed, so `resolveParkNotifyWaiter` no-op'd) is
158
+ * still observed, and a stale `runInputChannels` entry from a prior run or
159
+ * incarnation cannot false-positive because it did not bump the generation
160
+ * past `sinceGen`. Incarnation-scoped: cleared with `parkNotifyWaiters` on
161
+ * recycle/teardown so a reused runId cannot carry a stale captured generation
162
+ * across incarnations.
102
163
  */
103
- const inFlightRuns = new Set();
104
- // D2 attribution (measurement-only): the runId the dispatch loop is
164
+ const parkGenerations = new Map();
165
+ // D2 attribution (measurement-only): the messageId the dispatch loop is
105
166
  // currently servicing. Set at `dispatch-start`, cleared after
106
167
  // `reply-produced`. The dispatch loop is strictly serial (one message
107
168
  // in flight at a time -- the sustained interactive case the bench
108
- // drives), so a child-proxied WAL `substrate.write.request` (whose
109
- // `agent-state/<key>/...` preservePrefix carries no runId) is
110
- // unambiguously attributable to this runId. Run-event writes carry the
111
- // runId in their `runs/<runId>/events/` prefix and do not need it.
112
- let currentDispatchRunId = null;
169
+ // drives), so a child-proxied `substrate.write.request` is unambiguously
170
+ // attributable to this message. Both the WAL leg (whose `agent-state/...`
171
+ // prefix names no run) and the run-event leg (whose `runs/<runId>/events/`
172
+ // prefix names only the stable per-deployment run id, not the message)
173
+ // take their per-message key from here.
174
+ let currentDispatchMessageId = null;
113
175
  /**
114
176
  * Per-run drainTimeout accumulators armed by `drain()`. Held so
115
177
  * `shutdown()` can stop every accumulator cleanly before tearing
@@ -134,21 +196,21 @@ export function createWorkflowSupervisor(bindings) {
134
196
  }
135
197
  });
136
198
  const drainTimeoutMs = bindings.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
137
- const terminalWriteWatchdogMs = bindings.terminalWriteWatchdogMs ?? DEFAULT_TERMINAL_WRITE_WATCHDOG_MS;
199
+ const parkedQueryWatchdogMs = bindings.parkedQueryWatchdogMs ?? DEFAULT_PARKED_QUERY_WATCHDOG_MS;
138
200
  // Pure observability: invoke the dispatch-timing hook (when wired) at
139
201
  // the two per-message boundaries the 4.7 latency gate brackets. A
140
202
  // throwing observer is swallowed and logged so a benchmark hook bug
141
203
  // cannot wedge the dispatch loop.
142
- function emitDispatchTiming(runId, marker, atMs) {
204
+ function emitDispatchTiming(messageId, marker, atMs) {
143
205
  const observer = bindings.onDispatchTiming;
144
206
  if (observer === undefined)
145
207
  return;
146
208
  try {
147
- observer({ kind: "roundtrip", runId, marker, atMs });
209
+ observer({ kind: "roundtrip", messageId, marker, atMs });
148
210
  }
149
211
  catch (cause) {
150
212
  const message = cause instanceof Error ? cause.message : String(cause);
151
- logger.warn `onDispatchTiming observer threw for ${runId} (${marker}): ${message}`;
213
+ logger.warn `onDispatchTiming observer threw for ${messageId} (${marker}): ${message}`;
152
214
  }
153
215
  }
154
216
  // D2 per-leg attribution (measurement-only). Emits a paired
@@ -160,14 +222,14 @@ export function createWorkflowSupervisor(bindings) {
160
222
  // observability: a throwing observer is swallowed + logged so a
161
223
  // benchmark hook bug cannot wedge dispatch, and no clock or directory
162
224
  // is sampled when the observer is unwired.
163
- function legMarkStart(runId, leg) {
225
+ function legMarkStart(messageId, leg) {
164
226
  if (bindings.onDispatchTiming === undefined)
165
227
  return 0;
166
228
  const atMs = performance.now();
167
229
  try {
168
230
  bindings.onDispatchTiming({
169
231
  kind: "leg",
170
- runId,
232
+ messageId,
171
233
  leg,
172
234
  phase: "start",
173
235
  atMs,
@@ -175,11 +237,11 @@ export function createWorkflowSupervisor(bindings) {
175
237
  }
176
238
  catch (cause) {
177
239
  const message = cause instanceof Error ? cause.message : String(cause);
178
- logger.warn `onDispatchTiming leg observer threw for ${runId} (${leg} start): ${message}`;
240
+ logger.warn `onDispatchTiming leg observer threw for ${messageId} (${leg} start): ${message}`;
179
241
  }
180
242
  return atMs;
181
243
  }
182
- function legMarkEnd(runId, leg) {
244
+ function legMarkEnd(messageId, leg) {
183
245
  const observer = bindings.onDispatchTiming;
184
246
  if (observer === undefined)
185
247
  return;
@@ -193,12 +255,12 @@ export function createWorkflowSupervisor(bindings) {
193
255
  // surface it on the log and emit the end mark without counters so
194
256
  // the timing slope is still recoverable.
195
257
  const message = cause instanceof Error ? cause.message : String(cause);
196
- logger.warn `structural-counter sample failed for ${runId} (${leg}): ${message}`;
258
+ logger.warn `structural-counter sample failed for ${messageId} (${leg}): ${message}`;
197
259
  }
198
260
  try {
199
261
  observer({
200
262
  kind: "leg",
201
- runId,
263
+ messageId,
202
264
  leg,
203
265
  phase: "end",
204
266
  atMs,
@@ -207,7 +269,7 @@ export function createWorkflowSupervisor(bindings) {
207
269
  }
208
270
  catch (cause) {
209
271
  const message = cause instanceof Error ? cause.message : String(cause);
210
- logger.warn `onDispatchTiming leg observer threw for ${runId} (${leg} end): ${message}`;
272
+ logger.warn `onDispatchTiming leg observer threw for ${messageId} (${leg} end): ${message}`;
211
273
  }
212
274
  }
213
275
  // §10c forced-repack A/B (measurement-only). Absent toggle => never
@@ -244,27 +306,26 @@ export function createWorkflowSupervisor(bindings) {
244
306
  }
245
307
  /**
246
308
  * Classify a child-proxied `substrate.write.request` into the D2 leg it
247
- * represents, plus the runId the per-message OLS fit groups on.
248
- * `runs/<runId>/events/` is the run-event bracket commit (runId from the
249
- * prefix); `agent-state/...` is the D1 conversation WAL append (no runId
250
- * in the prefix -- attributed to the dispatch loop's current serial
251
- * runId). Any other prefix is an unmarked proxied write. Returns `null`
252
- * when no observer is wired (so the supervisor samples nothing) or the
253
- * prefix is not an attributed leg.
309
+ * represents, plus the messageId the per-message OLS fit groups on.
310
+ * `runs/<runId>/events/` is the run-event bracket commit and `agent-state/...`
311
+ * is the D1 conversation WAL append; neither prefix carries the message
312
+ * identity (the run-event prefix names only the stable per-deployment run
313
+ * id), so both are attributed to the dispatch loop's current serial
314
+ * message. Any other prefix is an unmarked proxied write. Returns `null`
315
+ * when no observer is wired (so the supervisor samples nothing), when the
316
+ * prefix is not an attributed leg, or when no message is in flight to
317
+ * attribute it to.
254
318
  */
255
319
  function classifyProxiedWriteLeg(preservePrefix) {
256
320
  if (bindings.onDispatchTiming === undefined)
257
321
  return null;
258
- const runEventMatch = /^runs\/([^/]+)\/events\/$/.exec(preservePrefix);
259
- if (runEventMatch !== null) {
260
- const runId = runEventMatch[1];
261
- if (runId !== undefined)
262
- return { leg: "runevent", runId };
322
+ if (currentDispatchMessageId === null)
323
+ return null;
324
+ if (/^runs\/[^/]+\/events\/$/.test(preservePrefix)) {
325
+ return { leg: "runevent", messageId: currentDispatchMessageId };
263
326
  }
264
327
  if (preservePrefix.startsWith("agent-state/")) {
265
- if (currentDispatchRunId !== null) {
266
- return { leg: "wal", runId: currentDispatchRunId };
267
- }
328
+ return { leg: "wal", messageId: currentDispatchMessageId };
268
329
  }
269
330
  return null;
270
331
  }
@@ -286,7 +347,7 @@ export function createWorkflowSupervisor(bindings) {
286
347
  const deriveMailAuditRef = bindings.deriveMailAuditRef ?? defaultInProcessMailAuditRef;
287
348
  const defaultInboxWritePrincipal = {
288
349
  kind: "supervisor",
289
- deploymentId: bindings.deploymentId,
350
+ anchorRunId: bindings.anchorRunId,
290
351
  };
291
352
  const inboxWritePrincipal = bindings.inboxWritePrincipal ?? defaultInboxWritePrincipal;
292
353
  // Resolve the consumed-dedup retention horizon once at the bindings
@@ -302,6 +363,17 @@ export function createWorkflowSupervisor(bindings) {
302
363
  const readyTimeoutMs = bindings.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS;
303
364
  const readySetTimer = bindings.setTimer ?? defaultSetTimer;
304
365
  const readyClearTimer = bindings.clearTimer ?? defaultClearTimer;
366
+ // Resolve the crash-loop guard bounds once at the bindings edge (the
367
+ // layer that owns the operator config). The stable-run reset timer
368
+ // reuses the same injectable `setTimer`/`clearTimer` pair as the ready
369
+ // handshake, and the wall clock reuses the recycle-policy `now` seam, so
370
+ // tests drive the whole guard deterministically through existing knobs.
371
+ const crashLoopMaxCount = bindings.crashLoopMaxCount ?? DEFAULT_CRASH_LOOP_MAX_COUNT;
372
+ const crashLoopWindowMs = bindings.crashLoopWindowMs ?? DEFAULT_CRASH_LOOP_WINDOW_MS;
373
+ const crashLoopStableResetMs = bindings.crashLoopStableResetMs ?? DEFAULT_CRASH_LOOP_STABLE_RESET_MS;
374
+ const respawnBackoffInitialMs = bindings.respawnBackoffInitialMs ?? DEFAULT_RESPAWN_BACKOFF_INITIAL_MS;
375
+ const respawnBackoffMaxMs = bindings.respawnBackoffMaxMs ?? DEFAULT_RESPAWN_BACKOFF_MAX_MS;
376
+ const crashNow = bindings.recyclePolicyNow ?? defaultNow;
305
377
  /**
306
378
  * Resolved on every successful `enqueueInbox`; the dispatch loop
307
379
  * awaits this promise after a null dequeue so it returns to
@@ -332,12 +404,321 @@ export function createWorkflowSupervisor(bindings) {
332
404
  */
333
405
  let spawnContext = null;
334
406
  let recyclePolicy = null;
335
- let recycleInProgress = false;
407
+ // Mutual-exclusion latch shared by every respawn path (operator/policy/
408
+ // self recycle and crash-respawn). `runRespawn` owns the set/clear; each
409
+ // caller owns the contention read because the two paths diverge on
410
+ // contention (recycle throws, crash-respawn declines silently).
411
+ let respawnInProgress = false;
412
+ // Monotonic child-cohort generation, bumped atomically with each
413
+ // transition to `running` (initial spawn and every respawn's
414
+ // `installNewChild`). The exit-watcher captures the generation it was
415
+ // armed for; a watcher whose generation is no longer current is stale
416
+ // (a newer child already superseded it) and is ignored. Generation plus
417
+ // the phase guard plus `respawnInProgress` together classify every child
418
+ // exit as planned or unexpected without a separate per-handle marker.
419
+ let childGeneration = 0;
420
+ // An unexpected child exit that could not be handled the instant it was
421
+ // observed (a respawn was already in flight). Recorded generation-tagged
422
+ // and drained by `maybeHandleChildExit` once `runRespawn` clears the
423
+ // latch; a pending exit for a superseded generation is dropped as stale.
424
+ let pendingChildExit = null;
425
+ // Monotonic timestamps (ms, from `crashNow`) of recent unexpected exits,
426
+ // pruned to `crashLoopWindowMs`. The crash-loop guard latches when the
427
+ // count within the window reaches `crashLoopMaxCount`.
428
+ const crashTimestamps = [];
429
+ // Handle for the armed stable-run reset timer (or null). Cleared and
430
+ // re-armed on every transition to `running`; cleared on teardown.
431
+ let stableRunResetTimer = null;
432
+ // Current exponential respawn backoff (ms). Advances by doubling on each
433
+ // respawn (capped at `respawnBackoffMaxMs`) and resets to the initial
434
+ // value when a respawned child runs stably. See `waitRespawnBackoff`.
435
+ let respawnBackoffMs = respawnBackoffInitialMs;
436
+ // Every armed backoff wait: the injected timer handle plus the promise
437
+ // resolver, so a shutdown can cancel all of them and unblock the parked
438
+ // respawn coroutines (each then bails at its phase/generation re-check).
439
+ // A SET, not a single slot: more than one coroutine can be parked at once
440
+ // -- a recycle can install a fresh, live child DURING one crash's backoff
441
+ // wait, and that child crashing parks a second coroutine. A single slot
442
+ // would drop the earlier one's timer and leak it past shutdown.
443
+ const respawnBackoffWaits = new Set();
444
+ // =====================================================================
445
+ // CRASH-RESPAWN -- auto-recovery from an unexpected workflow-process exit
446
+ // =====================================================================
447
+ //
448
+ // An UNEXPECTED child exit (crash, OOM, panic, signal -- not a
449
+ // supervisor-initiated shutdown, drain, or recycle) is detected by
450
+ // watching `handle.exited`, the only universal death signal: a clean
451
+ // process death ends the IPC channel readers without a protocol-level
452
+ // crash callback (see `onChildCrash`). `armChildForRunning` arms that
453
+ // watcher and bumps `childGeneration` on every transition to `running`;
454
+ // an exit is classified as unexpected iff its generation is still current
455
+ // AND the phase is still `running` (a planned kill is observed in a
456
+ // non-running phase, or against a superseded generation).
457
+ //
458
+ // On an unexpected exit `handleUnexpectedChildExit` runs: it records the
459
+ // crash against the crash-loop guard and either
460
+ // - RESPAWNS -- after an exponential backoff (`waitRespawnBackoff`,
461
+ // 1s doubling to a 30s cap), it reuses the recycle path's
462
+ // `runRespawn` with a no-op drain (the child is already dead) so the
463
+ // stranded-mail replay and fresh-child spawn happen exactly as a
464
+ // recycle's do; or
465
+ // - LATCHES -- once the guard trips (`crashLoopMaxCount` exits within
466
+ // `crashLoopWindowMs`), it tears down to the terminal `crash-looping`
467
+ // phase and commits a `RunFailed` for the deployment's run so the
468
+ // crash-loop is durably observable as a failed run status.
469
+ //
470
+ // A respawned child that survives `crashLoopStableResetMs`
471
+ // (`armStableRunResetTimer`) resets both the crash counter and the
472
+ // backoff. The backoff wait sits OUTSIDE the `respawnInProgress` latch,
473
+ // so the generation re-check after the wait is what stops a recycle that
474
+ // installed a fresh cohort mid-wait from being respawned a second time.
475
+ // Full policy: `packages/workflow-host/README.md` "Respawn policy".
476
+ // A protocol violation on a live cohort's control or event channel. The
477
+ // channel receiver ends its iterator and invokes this; a clean process
478
+ // death does NOT (it just ends the reader with no crash callback), so
479
+ // the exit-watcher on `handle.exited` is the universal death signal and
480
+ // this path only covers the frame-level violations the receiver detects.
481
+ // On the live (`running`) cohort, force the child down so its `exited`
482
+ // fires the exit-watcher and the crash flows through the SAME
483
+ // respawn/crash-loop path a clean death takes -- a violation that emits a
484
+ // garbage frame just before dying must not divert to `stopped` while a
485
+ // clean SIGKILL respawns. In any other phase the owning lifecycle path
486
+ // (spawn handshake, recycle reap, shutdown) owns teardown.
336
487
  function onChildCrash(reason) {
337
- logger.error `workflow-process control channel crash: {reason}`;
488
+ if (state.phase === "running") {
489
+ logger.error `workflow-process channel crash on live cohort; forcing child down to respawn: {reason}`;
490
+ state.handle.kill();
491
+ return;
492
+ }
493
+ logger.error `workflow-process channel crash: {reason}`;
338
494
  void shutdownInternal({ reason });
339
495
  }
340
- function onMailMessage(rawMessage) {
496
+ // Prune crash timestamps older than the sliding window relative to `nowMs`.
497
+ function pruneCrashTimestamps(nowMs) {
498
+ const cutoff = nowMs - crashLoopWindowMs;
499
+ while (true) {
500
+ const oldest = crashTimestamps[0];
501
+ if (oldest === undefined || oldest > cutoff)
502
+ break;
503
+ crashTimestamps.shift();
504
+ }
505
+ }
506
+ function clearStableRunResetTimer() {
507
+ if (stableRunResetTimer !== null) {
508
+ readyClearTimer(stableRunResetTimer);
509
+ stableRunResetTimer = null;
510
+ }
511
+ }
512
+ // Arm (or re-arm) the stable-run reset for the cohort that just reached
513
+ // `running`. If the child is still that same live cohort when the timer
514
+ // fires, the run has been stable for `crashLoopStableResetMs` and the
515
+ // crash counter is cleared so a flap-then-stabilize sequence does not
516
+ // latch. A crash before the timer fires re-arms it for the next cohort;
517
+ // teardown clears it.
518
+ function armStableRunResetTimer(generation) {
519
+ clearStableRunResetTimer();
520
+ stableRunResetTimer = readySetTimer(() => {
521
+ stableRunResetTimer = null;
522
+ if (generation === childGeneration && state.phase === "running") {
523
+ // The cohort ran stably: reset both the crash counter and the
524
+ // exponential backoff so a flap followed by stability starts over.
525
+ crashTimestamps.length = 0;
526
+ respawnBackoffMs = respawnBackoffInitialMs;
527
+ }
528
+ }, crashLoopStableResetMs);
529
+ }
530
+ // Wait the current respawn backoff before a respawn. The wait uses the
531
+ // injected timer so tests drive it deterministically, and is cancellable:
532
+ // `cancelRespawnBackoffWaits` (called on shutdown) clears the timer and
533
+ // resolves the promise early so the parked respawn coroutine unblocks and
534
+ // bails at its phase/generation re-check rather than sleeping out a full
535
+ // 30s backoff against a torn-down supervisor.
536
+ function waitRespawnBackoff(ms) {
537
+ return new Promise((resolve) => {
538
+ const entry = {
539
+ timer: null,
540
+ resolve,
541
+ };
542
+ entry.timer = readySetTimer(() => {
543
+ respawnBackoffWaits.delete(entry);
544
+ resolve();
545
+ }, ms);
546
+ respawnBackoffWaits.add(entry);
547
+ });
548
+ }
549
+ // Cancel every armed backoff wait. Idempotent: an empty set is a no-op,
550
+ // and each entry's own fire path has already removed it.
551
+ function cancelRespawnBackoffWaits() {
552
+ for (const entry of respawnBackoffWaits) {
553
+ readyClearTimer(entry.timer);
554
+ entry.resolve();
555
+ }
556
+ respawnBackoffWaits.clear();
557
+ }
558
+ // Bump the generation and arm the `handle.exited` watcher for a child
559
+ // that just became the `running` cohort. Called atomically with the
560
+ // state swap to `running` (no await between the swap and this call).
561
+ // `exited` resolving OR rejecting both mean the process is gone. The
562
+ // stable-run reset timer is NOT armed here: on the pristine first spawn
563
+ // there is no crash counter to reset, so it is armed only after a
564
+ // respawn (see `handleUnexpectedChildExit`).
565
+ function armChildForRunning(handle) {
566
+ childGeneration += 1;
567
+ const generation = childGeneration;
568
+ void handle.exited
569
+ .then(() => {
570
+ onChildExited(generation, "workflow-process child exited");
571
+ })
572
+ .catch(() => {
573
+ onChildExited(generation, "workflow-process child exited (exit promise rejected)");
574
+ });
575
+ }
576
+ // Record a child exit and try to act on it. Stale exits (a newer cohort
577
+ // already installed) are dropped.
578
+ function onChildExited(generation, reason) {
579
+ if (generation !== childGeneration)
580
+ return;
581
+ pendingChildExit = { generation, reason };
582
+ maybeHandleChildExit();
583
+ }
584
+ // Drain a recorded child exit if the supervisor is in a state to act on
585
+ // it. Declines (leaving the exit pending) while a respawn is in flight;
586
+ // `runRespawn`'s `finally` re-invokes this after clearing the latch so
587
+ // an exit observed during the respawn is not lost. A pending exit for a
588
+ // superseded generation, or one observed after the deployment left the
589
+ // `running` phase (shutdown/recycle owns that teardown), is dropped.
590
+ function maybeHandleChildExit() {
591
+ if (respawnInProgress)
592
+ return;
593
+ const pending = pendingChildExit;
594
+ if (pending === null)
595
+ return;
596
+ if (pending.generation !== childGeneration) {
597
+ pendingChildExit = null;
598
+ return;
599
+ }
600
+ if (state.phase !== "running")
601
+ return;
602
+ pendingChildExit = null;
603
+ void handleUnexpectedChildExit(pending.reason).catch((cause) => {
604
+ // Fire-and-forget context (the exit-watcher has no caller to catch
605
+ // this). `runRespawn` already ran its own failure teardown to a
606
+ // terminal state before rethrowing, so the deployment is not wedged;
607
+ // surface the failure and stop.
608
+ const message = cause instanceof Error ? cause.message : String(cause);
609
+ logger.error `crash-respawn handling failed; deployment torn down: ${message}`;
610
+ });
611
+ }
612
+ // Handle one unexpected child exit: record it against the crash-loop
613
+ // guard and either latch the deployment (guard tripped) or respawn.
614
+ async function handleUnexpectedChildExit(reason) {
615
+ if (state.phase !== "running" || spawnContext === null) {
616
+ // Raced a shutdown/recycle between the drain check and here; the
617
+ // owning lifecycle path handles teardown.
618
+ return;
619
+ }
620
+ // A cohort that just crashed did not earn a stability reward: disarm its
621
+ // stable-run reset timer up front so it cannot fire during the backoff
622
+ // wait below (where the phase is still `running` and the generation is
623
+ // still this dead cohort's -- exactly the state the timer's own guard
624
+ // reads as "alive and stable") and wrongly clear the crash counter.
625
+ clearStableRunResetTimer();
626
+ // Capture the crashing cohort's generation. If a recycle or policy
627
+ // respawn installs a fresh cohort while the backoff wait below is
628
+ // parked, the generation advances and this handler must NOT respawn
629
+ // that healthy cohort -- the post-wait guard bails on the mismatch.
630
+ const armedGeneration = childGeneration;
631
+ const nowMs = crashNow();
632
+ crashTimestamps.push(nowMs);
633
+ pruneCrashTimestamps(nowMs);
634
+ if (crashTimestamps.length >= crashLoopMaxCount) {
635
+ // Crash-loop latch. The deployment stops respawning and tears down
636
+ // to the terminal `crash-looping` state so a rapidly-flapping child
637
+ // cannot saturate the host.
638
+ const crashCount = crashTimestamps.length;
639
+ logger.error `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms; stopping the deployment (${reason})`;
640
+ await shutdownInternal({
641
+ reason: `crash-loop: ${reason}`,
642
+ terminalPhase: "crash-looping",
643
+ });
644
+ // Commit the RunFailed tombstone AFTER teardown: shutdownInternal has
645
+ // quiesced the drain accumulators (stop + await disposed), so the
646
+ // run-event tree is settled and no escalation commit races this write.
647
+ // This RunFailed is the SOLE durable, externally-queryable signal of
648
+ // the crash-loop (the `crash-looping` phase is in-memory only), so a
649
+ // failure to write it is logged loudly rather than swallowed. Best-
650
+ // effort: the deployment is already terminal, so the write not landing
651
+ // costs observability, not correctness.
652
+ try {
653
+ // `anchorRunId` and the tombstone's `runId` are DISTINCT ids and must
654
+ // not be conflated. `bindings.anchorRunId` is the workflow-run repo
655
+ // slug (`deriveWorkflowRunRepoId`), which the supervisor principal's
656
+ // authz check keys on (`repoId.id === anchorRunId`). The RunFailed must
657
+ // land on the deployment's ONE top-level run, whose id is the local
658
+ // part of the deployment's mail address (`deriveWorkflowRunId`) -- the
659
+ // same id the dispatch loop writes every run event under. For a domain
660
+ // like `integration.interchange` the two ids differ (the repo slug
661
+ // carries a domain suffix), so writing the tombstone under the repo
662
+ // slug would strand it in a run subtree no reader consults.
663
+ await commitRunFailed({
664
+ substrate: bindings.repoStore,
665
+ repoId: bindings.workflowRunRepoId,
666
+ ref: bindings.workflowRunRef,
667
+ anchorRunId: bindings.anchorRunId,
668
+ runId: deriveWorkflowRunId(bindings.deploymentMailAddress),
669
+ at: new Date(nowMs).toISOString(),
670
+ message: `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms`,
671
+ });
672
+ }
673
+ catch (cause) {
674
+ const message = cause instanceof Error ? cause.message : String(cause);
675
+ logger.error `crash-loop RunFailed commit failed; deployment has no durable failure tombstone: ${message}`;
676
+ }
677
+ return;
678
+ }
679
+ const thisBackoffMs = respawnBackoffMs;
680
+ logger.warn `workflow-process exited unexpectedly; respawning after ${String(thisBackoffMs)}ms backoff (${reason})`;
681
+ await waitRespawnBackoff(thisBackoffMs);
682
+ // A recycle/policy respawn, or a shutdown, may have run during the
683
+ // backoff wait. Bail unless THIS dead cohort is still the current
684
+ // running one: the generation guard prevents respawning a healthy
685
+ // cohort a recycle just installed, and the phase guard prevents acting
686
+ // after teardown. No await separates this re-check from `runRespawn`'s
687
+ // synchronous `respawnInProgress` set, so nothing can interleave.
688
+ if (childGeneration !== armedGeneration ||
689
+ state.phase !== "running" ||
690
+ spawnContext === null) {
691
+ return;
692
+ }
693
+ const priorRunning = state;
694
+ const priorContext = spawnContext;
695
+ // Advance the backoff only now that a respawn is actually happening, so
696
+ // a bail above does not inflate the wait for a respawn that never ran.
697
+ respawnBackoffMs = Math.min(respawnBackoffMs * 2, respawnBackoffMaxMs);
698
+ await runRespawn({
699
+ origin: "crash",
700
+ reason,
701
+ prior: priorRunning,
702
+ priorContext,
703
+ // The child is already dead: there is nothing to drain, and
704
+ // `runRespawn`'s kill step is a no-op on a dead handle. The replay
705
+ // step still runs, moving any mail stranded mid-flight back to the
706
+ // inbox tail before dispatch resumes.
707
+ drain: async () => undefined,
708
+ });
709
+ // The respawned child is now the running cohort. Arm the stable-run
710
+ // reset against its generation: if it survives `crashLoopStableResetMs`
711
+ // the crash counter and backoff reset, so a flap followed by stability
712
+ // does not latch on a later, unrelated crash.
713
+ armStableRunResetTimer(childGeneration);
714
+ }
715
+ // Resolves once the inbound mail is durably accepted (its inbox write landed
716
+ // or the message was already durably present); rejects when it was not (a
717
+ // phase where the deployment is not accepting mail, a transient enqueue
718
+ // failure, or a stale refusal). The host propagates that settlement to the
719
+ // wire, so resolution is the durable-receipt ACK signal and rejection is the
720
+ // WITHHOLD signal -- a withheld message is redelivered by the hub.
721
+ async function onMailMessage(rawMessage) {
341
722
  // Every inbound mail flows through the FIFO inbox claim-check
342
723
  // queue, regardless of the supervisor's current phase. The
343
724
  // dispatch loop (started by `spawn()` and restarted by the
@@ -349,16 +730,33 @@ export function createWorkflowSupervisor(bindings) {
349
730
  // the envelope's `receivedAt` prefix on the inbox filename.
350
731
  if (state.phase === "idle" ||
351
732
  state.phase === "stopping" ||
352
- state.phase === "stopped") {
353
- // The host's higher-level lifecycle is already tearing the
354
- // deployment down; the message drops on the floor rather than
355
- // landing in an inbox no live dispatch loop will service.
356
- return;
733
+ state.phase === "stopped" ||
734
+ state.phase === "crash-looping") {
735
+ // The host's higher-level lifecycle is already tearing the deployment
736
+ // down; nothing is enqueued. Reject rather than silently drop so the
737
+ // ack is WITHHELD and the hub redelivers -- a later generation (or a
738
+ // recycle-installed dispatch loop) may accept it, and a permanently
739
+ // torn-down address exhausts the hub's bounded retry budget instead of
740
+ // losing a message a transiently-idle deployment would have taken.
741
+ throw new Error(`inbound mail not accepted: supervisor phase is "${state.phase}"`);
742
+ }
743
+ try {
744
+ await enqueueInboundMail(rawMessage);
745
+ }
746
+ catch (cause) {
747
+ // Both branches WITHHOLD (rethrow); the split only sets log severity so
748
+ // a stale refusal surfaces as its own loud signal rather than blending
749
+ // into ordinary enqueue-failure noise. The ack/withhold decision is the
750
+ // rethrow itself, never this classification.
751
+ if (cause instanceof StaleInboxEnqueueError) {
752
+ logger.error `inbound mail refused as stale, withholding ack (hub will redeliver): ${cause.message}`;
753
+ }
754
+ else {
755
+ const message = cause instanceof Error ? cause.message : String(cause);
756
+ logger.error `enqueueInbox failed, withholding ack (hub will redeliver): ${message}`;
757
+ }
758
+ throw cause;
357
759
  }
358
- void enqueueInboundMail(rawMessage).catch((cause) => {
359
- const message = cause instanceof Error ? cause.message : String(cause);
360
- logger.error `enqueueInbox failed: ${message}`;
361
- });
362
760
  }
363
761
  async function enqueueInboundMail(rawMessage) {
364
762
  const messageId = await deriveMessageId(rawMessage);
@@ -373,11 +771,11 @@ export function createWorkflowSupervisor(bindings) {
373
771
  const rawMessageBase64 = base64Encode(rawMessage);
374
772
  // D2 leg: `enqueueInbox` runs in `onMailMessage` BEFORE dispatch, so
375
773
  // it is paid OUTSIDE the dispatch-start..reply-produced window -- its
376
- // growth is invisible to the 4.7 bracket. The leg mark, keyed by the
377
- // same messageId the dispatch loop later uses as the runId, makes the
378
- // out-of-window cost visible and joinable to the in-window legs.
774
+ // growth is invisible to the 4.7 bracket. This leg mark is keyed by the
775
+ // messageId, the same per-message key every in-window leg uses, so the
776
+ // D2 per-message OLS fit groups the enqueue leg with the rest.
379
777
  legMarkStart(messageId, "enqueue");
380
- await inboxPrimitives.enqueueInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
778
+ const outcome = await inboxPrimitives.enqueueInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
381
779
  address: bindings.deploymentMailAddress,
382
780
  messageId,
383
781
  receivedAt,
@@ -385,7 +783,20 @@ export function createWorkflowSupervisor(bindings) {
385
783
  rawMessage: rawMessageBase64,
386
784
  });
387
785
  legMarkEnd(messageId, "enqueue");
388
- wakeDispatch();
786
+ // Only a fresh enqueue added a new inbox entry; wake the dispatch loop for
787
+ // it alone. An `already-present` outcome landed nothing new -- returning
788
+ // (which acks) without waking is correct, since the earlier delivery of
789
+ // the same messageId already drives dispatch. This resolves for both
790
+ // outcomes: both mean the bytes are durably accounted for, so both ack.
791
+ if (outcome.outcome === "enqueued") {
792
+ wakeDispatch();
793
+ }
794
+ else {
795
+ // A redelivery of a message already durably present: the ack still
796
+ // fires (it is on disk), but no new run is dispatched. Surface it so an
797
+ // at-least-once redelivery being made effectively-once is observable.
798
+ logger.info `inbound mail ${messageId} already durably present (${outcome.reason}); acknowledging without re-dispatch`;
799
+ }
389
800
  }
390
801
  /**
391
802
  * Pump child-initiated upstream control frames after `ready` has
@@ -456,16 +867,98 @@ export function createWorkflowSupervisor(bindings) {
456
867
  // to-one with its cohort's `controlIncoming` iterator: a
457
868
  // buffered `terminal.event` the OLD child emitted before kill
458
869
  // landed must NEVER route to the NEW cohort's broadcaster.
459
- // Without this binding, a stale OLD-cohort frame for a runId
460
- // the NEW cohort happens to be dispatching under the same id
461
- // (the normal recycle/replay case) would falsely settle the
462
- // NEW cohort's `waitForRunTerminal` and commit `markConsumed`
870
+ // Every run shares the stable runId (the deployment mail
871
+ // address), so an OLD-cohort frame and a NEW-cohort run collide
872
+ // on that id by construction (the recycle/replay case); without
873
+ // this binding the stale frame would falsely settle the NEW
874
+ // cohort's `waitForRunTerminalOrPark` and commit `markConsumed`
463
875
  // on a run still in flight. The broadcaster's own `dispose()`
464
876
  // on cohort teardown turns post-dispose notify into a no-op,
465
877
  // so a stale frame dequeued after the cohort was torn down
466
878
  // drops cleanly without leaking into any successor cohort.
467
879
  const event = terminalEventFromPayload(payload.data);
468
880
  cohortBroadcaster.notify(payload.data.runId, event);
881
+ terminalRunIds.add(payload.data.runId);
882
+ // Clean up cohort tracking for the terminated run. Self-discovered
883
+ // runs have no dispatch-loop entry, so their cleanup happens here.
884
+ cohortRunIds.delete(payload.data.runId);
885
+ runInputChannels.delete(payload.data.runId);
886
+ continue;
887
+ }
888
+ if (payload.type === "park.notify") {
889
+ // The workflow-process child reported a control-plane suspension: an
890
+ // agent step parked on a reserved `signalName(correlationId)` channel.
891
+ if (payload.data.parkKind === "input") {
892
+ // Input parks are owned by the supervisor, not the hub. Register
893
+ // cohort membership BEFORE caching the correlationId so a run is
894
+ // never a channel-without-cohort entry (the routing-hygiene
895
+ // invariant); a live run that parked is always in cohortRunIds
896
+ // already, so this is idempotent belt-and-suspenders. Cache the
897
+ // correlationId so the dispatch loop can fire signal.deliver on
898
+ // subsequent mail without a substrate round-trip.
899
+ cohortRunIds.add(payload.data.runId);
900
+ runInputChannels.set(payload.data.runId, {
901
+ correlationId: payload.data.correlationId,
902
+ parkKind: "input",
903
+ });
904
+ // Stop any drain accumulator for a run that has parked. Input parks
905
+ // own this because a drain arms no accumulator for a run that already
906
+ // holds an input channel; an approval park never carries one, so its
907
+ // drain interaction is a separate concern and stays out of this arm.
908
+ const accumulator = drainAccumulators.get(payload.data.runId);
909
+ if (accumulator !== undefined) {
910
+ accumulator.stop();
911
+ drainAccumulators.delete(payload.data.runId);
912
+ }
913
+ }
914
+ else if (payload.data.parkKind === "approval") {
915
+ // Approval parks are hub-registered through the shared
916
+ // `registerSuspension` transform.
917
+ registerSuspension({
918
+ runId: payload.data.runId,
919
+ correlationId: payload.data.correlationId,
920
+ parkKind: "approval",
921
+ ...(payload.data.snapshot !== undefined
922
+ ? { snapshot: payload.data.snapshot }
923
+ : {}),
924
+ });
925
+ }
926
+ else {
927
+ // A `signal-relay` park is relayed down into the body child by the
928
+ // section runtime and is never hub-registered, so it does not ride
929
+ // `park.notify`. One arriving here is a protocol violation; log and
930
+ // drop rather than mis-registering it as an approval.
931
+ logger.error `park.notify for run ${payload.data.runId} carried parkKind=${payload.data.parkKind}, which is not a hub-registered kind; dropping`;
932
+ }
933
+ // A park of ANY kind suspends the run, so a dispatch loop waiting on
934
+ // `waitForRunTerminalOrPark` after firing the trigger (or delivering
935
+ // the last signal) must be released here regardless of park kind. An
936
+ // approval park that only registered its suspension would leave that
937
+ // loop hanging to the terminal-or-park backstop. Bump the park
938
+ // generation BEFORE resolving the waiter: a dispatch loop that captured
939
+ // `sinceGen` for this run must see the newer generation both when its
940
+ // armed waiter fires here and when it re-reads the generation after
941
+ // arming (the check-after-register).
942
+ parkGenerations.set(payload.data.runId, (parkGenerations.get(payload.data.runId) ?? 0) + 1);
943
+ // Wake any dispatch loop waiting for this run to park.
944
+ resolveParkNotifyWaiter(payload.data.runId);
945
+ continue;
946
+ }
947
+ if (payload.type === "parked-correlations.response") {
948
+ // The child answered a `reEmitParkedCorrelations` query. Resolve the
949
+ // awaiting driver; a response with no pending entry (the query already
950
+ // timed out and dropped it) is logged and dropped, never thrown, so it
951
+ // cannot tear the pump down.
952
+ resolveParkedResponse(payload.data);
953
+ continue;
954
+ }
955
+ if (payload.type === "resumed.runs") {
956
+ // The child self-discovered runs from the substrate after reconnect
957
+ // or recycle. Seed cohort tracking so drain accumulators and dispatch
958
+ // routing account for runs the supervisor did not personally fire.
959
+ for (const runId of payload.data.runIds) {
960
+ cohortRunIds.add(runId);
961
+ }
469
962
  continue;
470
963
  }
471
964
  logger.warn `workflow-process upstream control payload ignored: type=${payload.type}`;
@@ -473,22 +966,26 @@ export function createWorkflowSupervisor(bindings) {
473
966
  }
474
967
  const pendingMerges = new Map();
475
968
  /**
476
- * Reject every pending merge round-trip and every
477
- * `markConsumed` completion waiter. Invoked on cohort transitions
478
- * (shutdown, recycle's `installNewChild`) so closures awaiting these
479
- * promises do not outlive the cohort that armed them. Without this,
480
- * a `handleSubstrateWriteRequest` mid-merge or a dispatch-loop
481
- * caller awaiting `markConsumed` would sit on a resolver that the
482
- * dying control channel will never invoke.
969
+ * Reject every pending merge round-trip and every park-notify
970
+ * waiter. Invoked on cohort transitions (shutdown, recycle's
971
+ * `installNewChild`) so closures awaiting these promises do not
972
+ * outlive the cohort that armed them. Without this, a
973
+ * `handleSubstrateWriteRequest` mid-merge or a dispatch loop
974
+ * waiting for park would sit on a resolver that the dying control
975
+ * channel will never invoke.
483
976
  */
484
977
  function rejectCohortAwaiters(reason) {
485
978
  for (const [requestId, entry] of pendingMerges) {
486
979
  pendingMerges.delete(requestId);
487
980
  entry.resolve({ ok: false, reason: `cohort aborted: ${reason}` });
488
981
  }
489
- for (const [runId, waiter] of markConsumedCompletionWaiters.entries()) {
490
- markConsumedCompletionWaiters.delete(runId);
491
- waiter.reject(new MergeAbortedError(`markConsumed waiter (${runId}): ${reason}`));
982
+ for (const [requestId, entry] of pendingParkedQueries) {
983
+ pendingParkedQueries.delete(requestId);
984
+ entry.settle(null);
985
+ }
986
+ for (const [runId, resolve] of parkNotifyWaiters.entries()) {
987
+ parkNotifyWaiters.delete(runId);
988
+ resolve();
492
989
  }
493
990
  }
494
991
  function resolveMergeResponse(data) {
@@ -525,6 +1022,125 @@ export function createWorkflowSupervisor(bindings) {
525
1022
  }
526
1023
  entry.resolve({ ok: false, reason: data.result.reason });
527
1024
  }
1025
+ // Stamp the deployment identity the supervisor owns onto a child-supplied
1026
+ // park and hand it to the host's suspension-register sink. Shared by the
1027
+ // `park.notify` arm (the happy-path emit) and `reEmitParkedCorrelations`
1028
+ // (the re-establishment re-emit). Best-effort: a throwing sink is logged,
1029
+ // not rethrown, so it cannot tear the upstream pump down or abort a re-emit
1030
+ // partway through the parked set. The sink (production: the sidecar) turns
1031
+ // the stamped registration into a `signal.correlation.register` frame the
1032
+ // hub co-writes the run's routing + approval rows from.
1033
+ function registerSuspension(park) {
1034
+ if (bindings.onSuspensionRegister === undefined) {
1035
+ logger.warn `suspension register for runId=${park.runId} but no onSuspensionRegister sink is wired; correlation ${park.correlationId} not registered`;
1036
+ return;
1037
+ }
1038
+ try {
1039
+ bindings.onSuspensionRegister({
1040
+ runId: park.runId,
1041
+ correlationId: park.correlationId,
1042
+ kind: park.parkKind,
1043
+ anchorRunId: bindings.anchorRunId,
1044
+ agentAddress: bindings.deploymentMailAddress,
1045
+ ...(park.snapshot !== undefined
1046
+ ? { approvalSnapshot: park.snapshot }
1047
+ : {}),
1048
+ });
1049
+ }
1050
+ catch (cause) {
1051
+ const message = cause instanceof Error ? cause.message : String(cause);
1052
+ logger.error `onSuspensionRegister sink threw for runId=${park.runId} correlationId=${park.correlationId}: ${message}`;
1053
+ }
1054
+ }
1055
+ const pendingParkedQueries = new Map();
1056
+ let parkedQuerySeq = 0;
1057
+ function resolveParkedResponse(data) {
1058
+ const entry = pendingParkedQueries.get(data.requestId);
1059
+ if (entry === undefined) {
1060
+ logger.warn `parked-correlations.response landed with no pending entry; requestId=${data.requestId} dropped`;
1061
+ return;
1062
+ }
1063
+ pendingParkedQueries.delete(data.requestId);
1064
+ entry.settle(data.parked);
1065
+ }
1066
+ async function reEmitParkedCorrelations() {
1067
+ // No-op (NOT throw) when the child is not addressable. `deliverSignal`
1068
+ // throws on non-running/starting (including recycling) because it points a
1069
+ // write at the dying child's closing pipe and wants the caller to retry;
1070
+ // this driver's contract is the opposite -- the next re-establishment
1071
+ // re-drives it -- so skipping recycling and letting the next spawn's
1072
+ // re-emit cover it is correct here, not a missed guard.
1073
+ if (state.phase !== "running" && state.phase !== "starting") {
1074
+ logger.info `reEmitParkedCorrelations: child not addressable (phase=${state.phase}); skipping`;
1075
+ return;
1076
+ }
1077
+ const controlSender = state.controlSender;
1078
+ const requestId = `pc-${String((parkedQuerySeq += 1))}`;
1079
+ const responded = new Promise((resolve) => {
1080
+ pendingParkedQueries.set(requestId, { settle: resolve });
1081
+ });
1082
+ // Watchdog: a wedged-but-alive child never tears its cohort down, so the
1083
+ // cohort-abort settle would never fire and this await would hang the
1084
+ // re-establishment caller. On expiry, drop the pending entry and return;
1085
+ // the next re-establishment re-drives (the hub co-write is idempotent).
1086
+ let timeoutHandle = null;
1087
+ const watchdog = new Promise((resolve) => {
1088
+ timeoutHandle = setTimeout(() => {
1089
+ timeoutHandle = null;
1090
+ if (pendingParkedQueries.delete(requestId)) {
1091
+ logger.warn `reEmitParkedCorrelations: requestId=${requestId} did not respond within ${String(parkedQueryWatchdogMs)}ms; re-registration retries on the next re-establishment`;
1092
+ }
1093
+ resolve("timeout");
1094
+ }, parkedQueryWatchdogMs);
1095
+ });
1096
+ let outcome;
1097
+ try {
1098
+ await controlSender.send({
1099
+ type: "parked-correlations.request",
1100
+ data: { requestId },
1101
+ });
1102
+ outcome = await Promise.race([responded, watchdog]);
1103
+ }
1104
+ catch (cause) {
1105
+ // The downstream send failed (a closing pipe). Best-effort: drop the
1106
+ // pending entry, log, and return; the next re-establishment re-drives.
1107
+ pendingParkedQueries.delete(requestId);
1108
+ if (timeoutHandle !== null)
1109
+ clearTimeout(timeoutHandle);
1110
+ const message = cause instanceof Error ? cause.message : String(cause);
1111
+ logger.warn `reEmitParkedCorrelations query failed: ${message}; re-registration retries on the next re-establishment`;
1112
+ return;
1113
+ }
1114
+ if (timeoutHandle !== null)
1115
+ clearTimeout(timeoutHandle);
1116
+ // `timeout` (watchdog fired) or `null` (cohort torn down before the child
1117
+ // answered): nothing to re-emit; the next re-establishment re-drives.
1118
+ if (outcome === "timeout" || outcome === null)
1119
+ return;
1120
+ for (const parked of outcome) {
1121
+ if (parked.parkKind === "input") {
1122
+ // Register cohort membership BEFORE the input channel so the run is
1123
+ // never a channel-without-cohort entry: the dispatch loop's routing
1124
+ // hygiene drops exactly such entries, and a live resumed run must not
1125
+ // be mistaken for a dead one and have its channel deleted.
1126
+ cohortRunIds.add(parked.runId);
1127
+ runInputChannels.set(parked.runId, {
1128
+ correlationId: parked.correlationId,
1129
+ parkKind: "input",
1130
+ });
1131
+ }
1132
+ else {
1133
+ registerSuspension({ ...parked, parkKind: "approval" });
1134
+ }
1135
+ }
1136
+ }
1137
+ function resolveParkNotifyWaiter(runId) {
1138
+ const resolve = parkNotifyWaiters.get(runId);
1139
+ if (resolve === undefined)
1140
+ return;
1141
+ parkNotifyWaiters.delete(runId);
1142
+ resolve();
1143
+ }
528
1144
  /**
529
1145
  * OUTBOUND half of mailbox ownership (§3a). The workflow-process child
530
1146
  * never holds the agent's signing key; it forwards the structured
@@ -628,7 +1244,7 @@ export function createWorkflowSupervisor(bindings) {
628
1244
  // is which process owns the substrate write contract.
629
1245
  const writePrincipal = {
630
1246
  kind: "workflow-process",
631
- deploymentId: bindings.deploymentId,
1247
+ anchorRunId: bindings.anchorRunId,
632
1248
  };
633
1249
  // The commit's terminal detection comes from the kind handler's
634
1250
  // typed `newlyTerminalRuns` signal (returned below), not a sniff of
@@ -656,7 +1272,7 @@ export function createWorkflowSupervisor(bindings) {
656
1272
  // benchmark's per-message OLS fit groups on.
657
1273
  const legClassification = classifyProxiedWriteLeg(data.preservePrefix);
658
1274
  if (legClassification !== null) {
659
- legMarkStart(legClassification.runId, legClassification.leg);
1275
+ legMarkStart(legClassification.messageId, legClassification.leg);
660
1276
  }
661
1277
  try {
662
1278
  const { commitSha, newlyTerminalRuns } = await bindings.repoStore.writeTreePreservingPrefix(writePrincipal, validatedRepoId, data.ref, {
@@ -701,22 +1317,11 @@ export function createWorkflowSupervisor(bindings) {
701
1317
  });
702
1318
  // D2 leg end: the substrate commit (hash objects, write tree,
703
1319
  // advance ref under the per-repo lock) just resolved. Stamped here,
704
- // before the terminal-write markConsumed-coupling wait below, so the
705
- // run-event/wal leg measures only its own commit and not the
706
- // dispatch loop's markConsumed (which the `markconsumed` leg owns).
1320
+ // before the response, so the run-event/wal leg measures only its own
1321
+ // commit and not the dispatch loop's markConsumed (which the
1322
+ // `markconsumed` leg owns).
707
1323
  if (legClassification !== null) {
708
- legMarkEnd(legClassification.runId, legClassification.leg);
709
- }
710
- const watchdog = await synchronouslyDispatchTerminalWrite(newlyTerminalRuns);
711
- if (!watchdog.ok) {
712
- await controlSender.send({
713
- type: "substrate.write.response",
714
- data: {
715
- requestId: data.requestId,
716
- result: { ok: false, reason: watchdog.reason },
717
- },
718
- });
719
- return;
1324
+ legMarkEnd(legClassification.messageId, legClassification.leg);
720
1325
  }
721
1326
  await controlSender.send({
722
1327
  type: "substrate.write.response",
@@ -740,7 +1345,7 @@ export function createWorkflowSupervisor(bindings) {
740
1345
  substrate: bindings.repoStore,
741
1346
  repoId: validatedRepoId,
742
1347
  ref: data.ref,
743
- deploymentId: bindings.deploymentId,
1348
+ anchorRunId: bindings.anchorRunId,
744
1349
  runId,
745
1350
  }).catch((cause) => {
746
1351
  logger.warn `compaction of run ${runId} failed: ${cause instanceof Error ? cause.message : String(cause)}`;
@@ -765,129 +1370,6 @@ export function createWorkflowSupervisor(bindings) {
765
1370
  });
766
1371
  }
767
1372
  }
768
- // Per-runId synchronization between the substrate-write handler
769
- // and the dispatch loop's `markConsumed`. The handler arms a
770
- // waiter when it commits a terminal-event blob and waits for the
771
- // dispatch loop to fire `resolveMarkConsumedWaiter(runId)` before
772
- // sending the substrate.write.response back to the child.
773
- const markConsumedCompletionWaiters = new Map();
774
- function resolveMarkConsumedWaiter(runId) {
775
- const waiter = markConsumedCompletionWaiters.get(runId);
776
- if (waiter === undefined)
777
- return;
778
- markConsumedCompletionWaiters.delete(runId);
779
- waiter.resolve();
780
- }
781
- /**
782
- * Hold the substrate.write.response until the dispatch loop's
783
- * markConsumed settles for each run the kind handler reports as newly
784
- * terminal in this commit. Terminal-ness comes from the handler's typed
785
- * `newlyTerminalRuns` signal -- determined authoritatively during
786
- * validation -- not re-derived from the committed path shape, so it
787
- * survives the run-event layout changing (e.g. compaction folding a
788
- * run's per-event files into one combined file). The wait is per-runId
789
- * so multiple runs can proceed concurrently if a future dispatch loop
790
- * ever processes more than one mail in parallel.
791
- *
792
- * A watchdog timeout (`terminalWriteWatchdogMs`) caps each wait so a
793
- * never-arming markConsumed (a bug in the dispatch loop, a torn-down
794
- * cohort, a stalled inbox primitive) does not deadlock the child's
795
- * write -- and therefore the runtime body, and therefore the dispatch
796
- * loop. On expiry the waiter is force-released and a structured failure
797
- * propagates back to the child as
798
- * `{ ok: false, reason: "terminal-write watchdog timeout: ..." }`.
799
- */
800
- async function synchronouslyDispatchTerminalWrite(newlyTerminalRuns) {
801
- const holds = [];
802
- for (const { runId, terminalEventJson } of newlyTerminalRuns) {
803
- if (!inFlightRuns.has(runId))
804
- continue;
805
- holds.push(holdResponseForMarkConsumed(runId, terminalEventJson));
806
- }
807
- if (holds.length === 0)
808
- return { ok: true };
809
- const results = await Promise.all(holds);
810
- return results.find((r) => !r.ok) ?? { ok: true };
811
- }
812
- async function holdResponseForMarkConsumed(runId, terminalEventJson) {
813
- const completed = new Promise((resolve, reject) => {
814
- markConsumedCompletionWaiters.set(runId, { resolve, reject });
815
- });
816
- const broadcaster = activeTerminalBroadcaster();
817
- if (broadcaster !== null) {
818
- const synthetic = synthesizeTerminalEvent(terminalEventJson);
819
- if (synthetic !== null) {
820
- broadcaster.notify(runId, synthetic);
821
- }
822
- }
823
- let timeoutHandle = null;
824
- const watchdog = new Promise((resolve) => {
825
- timeoutHandle = setTimeout(() => {
826
- timeoutHandle = null;
827
- // Force-release the waiter so the dispatch loop's eventual
828
- // resolve does not strand a dangling map entry, then surface
829
- // the structured failure to the caller. The reason text is
830
- // logged through the package logger so the watchdog is not
831
- // silent on the host side.
832
- const stillPending = markConsumedCompletionWaiters.get(runId) !== undefined;
833
- if (stillPending) {
834
- markConsumedCompletionWaiters.delete(runId);
835
- }
836
- const reason = `terminal-write watchdog timeout: markConsumed for runId=${runId} did not settle within ${String(terminalWriteWatchdogMs)}ms`;
837
- logger.error `${reason}`;
838
- resolve({ ok: false, reason });
839
- }, terminalWriteWatchdogMs);
840
- });
841
- const result = await Promise.race([
842
- completed.then(() => ({ ok: true })),
843
- watchdog,
844
- ]);
845
- if (timeoutHandle !== null) {
846
- clearTimeout(timeoutHandle);
847
- }
848
- return result;
849
- }
850
- function synthesizeTerminalEvent(terminalEventJson) {
851
- let parsed;
852
- try {
853
- parsed = JSON.parse(terminalEventJson);
854
- }
855
- catch {
856
- return null;
857
- }
858
- if (typeof parsed !== "object" ||
859
- parsed === null ||
860
- !("type" in parsed) ||
861
- !("seq" in parsed)) {
862
- return null;
863
- }
864
- const body = parsed;
865
- if (typeof body.seq !== "number")
866
- return null;
867
- const at = typeof body.at === "string" ? body.at : new Date().toISOString();
868
- if (body.type === "RunCompleted") {
869
- return { kind: "RunCompleted", seq: body.seq, at };
870
- }
871
- if (body.type === "RunCancelled") {
872
- return { kind: "RunCancelled", seq: body.seq, at };
873
- }
874
- if (body.type === "RunFailed") {
875
- // The wire schema makes `error.message` required when the event
876
- // type is `RunFailed`. An event that doesn't carry one is a
877
- // contract violation upstream of the supervisor; coercing it to an
878
- // empty string would silently hide the producer bug.
879
- if (typeof body.error?.message !== "string") {
880
- throw new Error(`synthesizeTerminalEvent: RunFailed event missing required error.message`);
881
- }
882
- return {
883
- kind: "RunFailed",
884
- seq: body.seq,
885
- at,
886
- error: { message: body.error.message },
887
- };
888
- }
889
- return null;
890
- }
891
1373
  function activeControlSender() {
892
1374
  if (state.phase === "starting" ||
893
1375
  state.phase === "running" ||
@@ -896,14 +1378,6 @@ export function createWorkflowSupervisor(bindings) {
896
1378
  }
897
1379
  return null;
898
1380
  }
899
- function activeTerminalBroadcaster() {
900
- if (state.phase === "starting" ||
901
- state.phase === "running" ||
902
- state.phase === "recycling") {
903
- return state.terminalBroadcaster;
904
- }
905
- return null;
906
- }
907
1381
  async function wireChild(args) {
908
1382
  const controlSender = createControlChannelSender({
909
1383
  privateKeySeed: args.ipcKeypair.privateKey,
@@ -921,10 +1395,11 @@ export function createWorkflowSupervisor(bindings) {
921
1395
  hmacKey: args.hmacKey,
922
1396
  channelId: args.channelId,
923
1397
  reader: args.handle.eventReader,
924
- onCrash: (reason) => {
925
- logger.error `workflow-process event channel crash: {reason}`;
926
- void shutdownInternal({ reason });
927
- },
1398
+ // Route event-channel crashes through the same funnel as
1399
+ // control-channel crashes so both drive the respawn/crash-loop path
1400
+ // uniformly on the live cohort (and defer to the owning lifecycle
1401
+ // path in every other phase).
1402
+ onCrash: onChildCrash,
928
1403
  });
929
1404
  const eventPump = pumpEvents(eventIter, args.onInferenceEvent);
930
1405
  return {
@@ -951,7 +1426,7 @@ export function createWorkflowSupervisor(bindings) {
951
1426
  channelId,
952
1427
  hmacKey,
953
1428
  hostPublicKey: ipcKeypair.publicKey,
954
- deploymentId: bindings.deploymentId,
1429
+ anchorRunId: bindings.anchorRunId,
955
1430
  deploymentMailAddress: bindings.deploymentMailAddress,
956
1431
  stepCount: bindings.stepCount,
957
1432
  definitionHash: opts.definitionHash,
@@ -1029,7 +1504,7 @@ export function createWorkflowSupervisor(bindings) {
1029
1504
  repoStore: bindings.repoStore,
1030
1505
  principal: bindings.readPrincipal,
1031
1506
  stepOrder: opts.stepOrder,
1032
- deploymentId: bindings.deploymentId,
1507
+ anchorRunId: bindings.anchorRunId,
1033
1508
  deriveStepAddress: bindings.deriveStepAddress,
1034
1509
  ...(bindings.deriveStepRepoId !== undefined
1035
1510
  ? { deriveStepRepoId: bindings.deriveStepRepoId }
@@ -1124,19 +1599,27 @@ export function createWorkflowSupervisor(bindings) {
1124
1599
  // same control channel `trigger.fire` uses, so the ordering
1125
1600
  // guarantee (`grants-updated` lands before `trigger.fire`) holds
1126
1601
  // for buffered and post-ready inbound mail alike.
1127
- await wired.wiring.controlSender.send({
1128
- type: "grants-updated",
1129
- data: {
1130
- snapshot: {
1131
- steps: credentialsSnapshot.steps.map((s) => ({
1132
- stepId: s.stepId,
1133
- address: s.address,
1134
- grants: [...s.grants],
1135
- contentHash: s.contentHash,
1136
- })),
1602
+ //
1603
+ // Suppressed when `onRunStart` is wired: that binding makes the
1604
+ // dispatch loop push a per-run snapshot before each `trigger.fire`,
1605
+ // so the spawn-time push would only mask a broken per-run barrier
1606
+ // (the child would already hold grants and never hit its throw-on-
1607
+ // null guard). The per-run push is then the sole grants source.
1608
+ if (bindings.onRunStart === undefined) {
1609
+ await wired.wiring.controlSender.send({
1610
+ type: "grants-updated",
1611
+ data: {
1612
+ snapshot: {
1613
+ steps: credentialsSnapshot.steps.map((s) => ({
1614
+ stepId: s.stepId,
1615
+ address: s.address,
1616
+ grants: [...s.grants],
1617
+ contentHash: s.contentHash,
1618
+ })),
1619
+ },
1137
1620
  },
1138
- },
1139
- });
1621
+ });
1622
+ }
1140
1623
  // Transition to running. The dispatch loop (started below)
1141
1624
  // picks up any pre-ready buffered mail through the FIFO inbox
1142
1625
  // queue rather than through an in-memory buffer; arrival order
@@ -1170,6 +1653,11 @@ export function createWorkflowSupervisor(bindings) {
1170
1653
  dispatchLoop,
1171
1654
  replayDone,
1172
1655
  };
1656
+ // Bump the generation and arm the exit-watcher atomically with the
1657
+ // running transition (no await between the swap above and this call)
1658
+ // so an unexpected exit of this child is classified against the
1659
+ // right generation.
1660
+ armChildForRunning(handle);
1173
1661
  // Kick the dispatch loop in case mail landed in the inbox
1174
1662
  // before the loop's first `await dispatchWake`. A wake against a
1175
1663
  // freshly-minted promise is a no-op; the dispatch loop's first
@@ -1200,6 +1688,18 @@ export function createWorkflowSupervisor(bindings) {
1200
1688
  const message = cause instanceof Error ? cause.message : String(cause);
1201
1689
  logger.error `upstream control pump failed: ${message}`;
1202
1690
  });
1691
+ // Trigger A: re-register every correlation the freshly-ready child is
1692
+ // parked on. A `park.notify` register can be lost while the hub is down
1693
+ // at the original suspend; a child that resumes such a parked run (a
1694
+ // sidecar restart re-spawning this deployment, or a recycle -- see the
1695
+ // matching call in `installNewChild`) re-parks without re-emitting, so
1696
+ // the supervisor re-drives it from every re-establishment. Fire-and-
1697
+ // forget after the pump is armed to route the response: best-effort,
1698
+ // watchdog-bounded, and an empty round-trip when nothing is parked.
1699
+ void reEmitParkedCorrelations().catch((cause) => {
1700
+ const message = cause instanceof Error ? cause.message : String(cause);
1701
+ logger.warn `re-emit of parked correlations on re-establishment failed: ${message}`;
1702
+ });
1203
1703
  // Arm the recycle policy. The policy is a no-op when all bounds
1204
1704
  // are `undefined`; bounds resolution lives inside `createRecyclePolicy`.
1205
1705
  if (bindings.recyclePolicy !== undefined) {
@@ -1318,60 +1818,132 @@ export function createWorkflowSupervisor(bindings) {
1318
1818
  }
1319
1819
  /**
1320
1820
  * Forward one dequeued inbox entry to the child as `trigger.fire`
1321
- * and record its runId as in-flight. The runId is the messageId
1322
- * the envelope carries (one run per trigger fire per discovery
1323
- * Q3.1); the same value is what the dispatch loop waits on via
1324
- * `terminalEventSource`.
1821
+ * and record its runId as in-flight. The runId is the local part of the
1822
+ * deployment's mail address (see `deriveWorkflowRunId`), identifying its one
1823
+ * top-level run; the `messageId` rides alongside it so the child can
1824
+ * recover the trigger's mail bytes by claim-check. The runId is the
1825
+ * same value the dispatch loop waits on via `terminalEventSource`.
1325
1826
  */
1326
- async function forwardDispatchedEntry(sender, messageId, receivedAt) {
1827
+ async function forwardDispatchedEntry(sender, messageId, receivedAt, runId) {
1327
1828
  await sender.send({
1328
1829
  type: "trigger.fire",
1329
1830
  data: {
1330
- runId: messageId,
1831
+ runId,
1331
1832
  messageId,
1332
1833
  receivedAt,
1333
1834
  },
1334
1835
  });
1335
- inFlightRuns.add(messageId);
1336
- return messageId;
1836
+ cohortRunIds.add(runId);
1837
+ return runId;
1838
+ }
1839
+ /**
1840
+ * Push the run's grants snapshot to the child ahead of its
1841
+ * `trigger.fire`. Returns `true` if the barrier FAILED (the caller must
1842
+ * skip the fire; the run has already been settled as `RunFailed`) and
1843
+ * `false` if the barrier passed or is not armed (`onRunStart` unwired,
1844
+ * where `spawn` supplied the snapshot instead).
1845
+ *
1846
+ * The sink is a request/response contract: the supervisor awaits the
1847
+ * returned snapshot and awaits the `grants-updated` send so both land on
1848
+ * the child's control channel before the trigger. A throw from either --
1849
+ * the sink itself or the control send -- is surfaced as a synthesized
1850
+ * `RunFailed` fanned out to this run's broadcaster watcher, never
1851
+ * swallowed, so the run fails deterministically instead of the child
1852
+ * authorizing against a stale or absent snapshot.
1853
+ */
1854
+ async function pushRunGrants(sender, runId, broadcaster) {
1855
+ if (bindings.onRunStart === undefined)
1856
+ return false;
1857
+ try {
1858
+ const snapshot = await bindings.onRunStart({
1859
+ runId,
1860
+ anchorRunId: bindings.anchorRunId,
1861
+ });
1862
+ await sender.send({
1863
+ type: "grants-updated",
1864
+ data: {
1865
+ snapshot: {
1866
+ steps: snapshot.steps.map((s) => ({
1867
+ stepId: s.stepId,
1868
+ address: s.address,
1869
+ grants: [...s.grants],
1870
+ contentHash: s.contentHash,
1871
+ })),
1872
+ },
1873
+ },
1874
+ });
1875
+ // Deliver the deployment's credential material on the same pre-trigger
1876
+ // barrier, so a tool that resolves a credential on the first step already
1877
+ // has it in the child's cell. The material is the decrypted delivery the
1878
+ // hub put on the deploy frame; a later rotation flows through
1879
+ // `deliverCredentials` instead. Absent when the deployment binds none.
1880
+ if (bindings.credentialDelivery !== undefined) {
1881
+ await sender.send({
1882
+ type: "credentials-updated",
1883
+ data: { delivery: bindings.credentialDelivery },
1884
+ });
1885
+ }
1886
+ return false;
1887
+ }
1888
+ catch (cause) {
1889
+ const message = cause instanceof Error ? cause.message : String(cause);
1890
+ logger.error `onRunStart grants barrier failed for run ${runId}; failing the run: ${message}`;
1891
+ broadcaster.notify(runId, {
1892
+ kind: "RunFailed",
1893
+ seq: 0,
1894
+ at: new Date().toISOString(),
1895
+ error: {
1896
+ message: `workflow-host supervisor: run ${runId} not authorized; grants barrier failed before trigger.fire: ${message}`,
1897
+ },
1898
+ });
1899
+ return true;
1900
+ }
1337
1901
  }
1338
1902
  /**
1339
1903
  * One iteration of the dispatch loop: dequeue the FIFO-first inbox
1340
- * entry, forward it as a `trigger.fire`, wait for the corresponding
1341
- * run's terminal event (or for the cohort to abort), then
1342
- * `markConsumed`. Returns `true` if a dispatch landed (caller should
1343
- * loop immediately) and `false` if the inbox was empty (caller
1344
- * should await the next wake).
1904
+ * entry, decide whether to `signal.deliver` or `trigger.fire` (or wait
1905
+ * if the run is in-flight but not yet parked), then `markConsumed`
1906
+ * once the child has taken up the message -- for a `trigger.fire` that
1907
+ * means after the run reaches a terminal event or parks, so the
1908
+ * claim-check entry the child still needs to read is not deleted out
1909
+ * from under it. Returns `true` if a dispatch landed (caller should
1910
+ * loop immediately) and `false` if the inbox was empty (caller should
1911
+ * await the next wake).
1345
1912
  */
1346
1913
  async function dispatchOne(sender, cohortAbort, broadcaster) {
1347
1914
  if (cohortAbort.signal.aborted)
1348
1915
  return false;
1349
- // Subscribe to the terminal broadcaster BEFORE forwarding the
1350
- // trigger.fire so a terminal event the child notifies between
1351
- // forward and subscribe cannot be missed. The broadcaster fires
1352
- // its listeners synchronously inside `notify`; with the subscribe
1353
- // ordered first the listener buffers the event until the
1354
- // dispatch loop's `iter.next()` consumes it.
1355
1916
  const beforeDequeueMs = dispatchTimingEnabled() ? performance.now() : 0;
1356
1917
  const dequeued = await inboxPrimitives.dequeueToProcessing(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress);
1357
1918
  if (dequeued === null)
1358
1919
  return false;
1359
1920
  const envelope = dequeued.envelope;
1360
- const runId = envelope.messageId;
1361
- currentDispatchRunId = runId;
1362
- emitDispatchTiming(runId, "dispatch-start", beforeDequeueMs);
1921
+ const runId = deriveWorkflowRunId(bindings.deploymentMailAddress);
1922
+ const messageId = envelope.messageId;
1923
+ let rejection;
1924
+ const rejectTerminalRun = () => {
1925
+ if (rejection !== undefined)
1926
+ return;
1927
+ rejection = {
1928
+ code: "workflow_run_terminal",
1929
+ message: `Workflow run ${runId} is terminal and cannot be fired again`,
1930
+ };
1931
+ logger.warn `rejecting inbound mail ${messageId}: workflow run ${runId} is terminal`;
1932
+ };
1933
+ currentDispatchMessageId = messageId;
1934
+ emitDispatchTiming(messageId, "dispatch-start", beforeDequeueMs);
1363
1935
  // D2 leg: the claim-check dequeue READ. `dispatch-start` is sampled
1364
1936
  // BEFORE the dequeue (so the roundtrip bracket includes the read);
1365
1937
  // the dequeue leg's own start mark is that same pre-dequeue sample
1366
1938
  // re-stamped under the leg channel, and its end is now (the read just
1367
1939
  // completed). Emitting the start retroactively here -- rather than
1368
- // before the await -- keeps the leg keyed by the runId, which is only
1369
- // known after the dequeue resolves.
1940
+ // before the await -- keeps the leg keyed by the messageId, which is
1941
+ // only known after the dequeue resolves.
1370
1942
  if (bindings.onDispatchTiming !== undefined) {
1371
1943
  try {
1372
1944
  bindings.onDispatchTiming({
1373
1945
  kind: "leg",
1374
- runId,
1946
+ messageId,
1375
1947
  leg: "dequeue",
1376
1948
  phase: "start",
1377
1949
  atMs: beforeDequeueMs,
@@ -1379,28 +1951,212 @@ export function createWorkflowSupervisor(bindings) {
1379
1951
  }
1380
1952
  catch (cause) {
1381
1953
  const message = cause instanceof Error ? cause.message : String(cause);
1382
- logger.warn `onDispatchTiming leg observer threw for ${runId} (dequeue start): ${message}`;
1954
+ logger.warn `onDispatchTiming leg observer threw for ${messageId} (dequeue start): ${message}`;
1955
+ }
1956
+ }
1957
+ legMarkEnd(messageId, "dequeue");
1958
+ // In-memory cohort membership is deliberately not the lifecycle
1959
+ // authority: it is empty for a new run, after a terminal frame, and while
1960
+ // a child is rediscovering a live run after restart. Consult the durable
1961
+ // log before deciding that "no cohort" means "fire". A live log is added
1962
+ // back to cohort tracking so the existing terminal/park wait handles the
1963
+ // recovery window; a terminal log is rejected permanently.
1964
+ if (!cohortRunIds.has(runId)) {
1965
+ const lifecycle = terminalRunIds.has(runId)
1966
+ ? "terminal"
1967
+ : await readWorkflowRunLifecycle(bindings.repoStore, bindings.workflowRunRepoId, runId);
1968
+ if (lifecycle === "terminal") {
1969
+ rejectTerminalRun();
1970
+ }
1971
+ else if (lifecycle === "live") {
1972
+ cohortRunIds.add(runId);
1973
+ }
1974
+ }
1975
+ if (rejection === undefined) {
1976
+ // Subscribe to the terminal broadcaster BEFORE the grants barrier so
1977
+ // a synthetic `RunFailed` from a barrier failure can be captured.
1978
+ const preIter = broadcaster.source(runId)[Symbol.asyncIterator]();
1979
+ // Per-run grants barrier. When `onRunStart` is wired, push this run's
1980
+ // grants snapshot BEFORE the trigger/signal so the child's authorize
1981
+ // closure binds to it rather than throwing on a null snapshot. The push
1982
+ // and the fire share the child's control channel, so a `grants-updated`
1983
+ // awaited here is observed by the child ahead of the trigger. A barrier
1984
+ // failure (the sink throws, or the push fails) fails the run loudly --
1985
+ // a synthesized `RunFailed` fanned out to this run's watcher -- and the
1986
+ // trigger is NOT fired, so no step ever runs against absent grants.
1987
+ const barrierFailed = await pushRunGrants(sender, runId, broadcaster);
1988
+ if (barrierFailed) {
1989
+ // Wait for the synthetic RunFailed before consuming the message.
1990
+ await waitForRunTerminal(preIter, cohortAbort.signal);
1991
+ // Clean up for synthetic barrier failure (real terminal events are
1992
+ // cleaned up by pumpUpstreamControl, but synthetic ones are not).
1993
+ cohortRunIds.delete(runId);
1994
+ runInputChannels.delete(runId);
1995
+ }
1996
+ else {
1997
+ // Dispose the pre-created iterator; the normal path creates fresh
1998
+ // iterators inside the dispatch-decision loop when waiting.
1999
+ if (typeof preIter.return === "function") {
2000
+ await preIter.return();
2001
+ }
2002
+ // Unified dispatch: park → signal.deliver; no live run → trigger.fire;
2003
+ // in-flight but undecided → wait for terminal or park, then re-evaluate.
2004
+ while (!cohortAbort.signal.aborted) {
2005
+ if (terminalRunIds.has(runId)) {
2006
+ rejectTerminalRun();
2007
+ break;
2008
+ }
2009
+ // Capture the park generation BEFORE this iteration's pre-wait awaits
2010
+ // so `waitForRunTerminalOrPark` can accept a strictly-newer park that
2011
+ // fires during them (see the latch there). Re-captured each iteration
2012
+ // so a wait that returned "parked" is not re-counted next time around.
2013
+ const sinceGen = parkGenerations.get(runId) ?? 0;
2014
+ // Routing hygiene (defense-in-depth the latch never depends on): a
2015
+ // runInputChannels entry with no live run in this cohort is a stale
2016
+ // routing hazard left by a dead incarnation. Drop it BEFORE the signal
2017
+ // branch so a fresh mail cannot be routed onto a dead run's
2018
+ // correlation. The resumed.runs invariant (cohortRunIds registered
2019
+ // before its input channel) keeps a LIVE resumed run out of this
2020
+ // branch, so this only ever drops genuinely-dead entries.
2021
+ if (!cohortRunIds.has(runId) && runInputChannels.has(runId)) {
2022
+ runInputChannels.delete(runId);
2023
+ }
2024
+ const inputChannel = runInputChannels.get(runId);
2025
+ if (inputChannel !== undefined) {
2026
+ // Resolve the inbound mail to conversation text HERE, the single
2027
+ // site that knows this payload's provenance is mail, applying the
2028
+ // SAME extraction the turn-1 trigger does (resolveTriggerPayload).
2029
+ // The signal.deliver frame's payload is the resume decision in FINAL
2030
+ // form; deliverSignal's structured signals ship their own payload
2031
+ // unchanged. Done BEFORE minting the terminal watcher so a failure
2032
+ // here cannot leak an un-finalized iterator.
2033
+ let inputText;
2034
+ try {
2035
+ if (envelope.rawMessage === undefined) {
2036
+ throw new Error("inbound mail carries no rawMessage bytes");
2037
+ }
2038
+ inputText = extractConversationText(base64Decode(envelope.rawMessage), envelope.messageId);
2039
+ }
2040
+ catch (cause) {
2041
+ // A malformed turn-2 mail cannot resume the parked agent. DROP it:
2042
+ // log loudly and consume it (break to the post-loop markConsumed)
2043
+ // rather than throwing -- a throw aborts the dispatch without
2044
+ // consuming, and replay re-delivers the same poison mail forever.
2045
+ // The run stays parked on its current correlation, ready for the
2046
+ // next valid mail; one bad mail must not tear down a long-lived
2047
+ // conversation.
2048
+ const message = cause instanceof Error ? cause.message : String(cause);
2049
+ logger.error `signal.deliver for run ${runId}: dropping malformed inbound mail ${envelope.messageId}: ${message}`;
2050
+ break;
2051
+ }
2052
+ // Mint the terminal watcher only now, after the payload resolved, so
2053
+ // a terminal the resumed run reaches right after applying the signal
2054
+ // is not missed; the park watcher is armed inside
2055
+ // waitForRunTerminalOrPark.
2056
+ const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
2057
+ let waitEntered = false;
2058
+ try {
2059
+ await sender.send({
2060
+ type: "signal.deliver",
2061
+ data: {
2062
+ runId,
2063
+ signalName: signalName(inputChannel.correlationId),
2064
+ signalId: envelope.messageId,
2065
+ payload: inputText,
2066
+ },
2067
+ });
2068
+ // Invalidate the cached input channel: its correlation is now
2069
+ // consumed by this delivery, so the NEXT mail must not reuse it.
2070
+ // The resumed run re-parks on a FRESH correlation (a new
2071
+ // park.notify repopulates runInputChannels); a mail arriving before
2072
+ // that re-park waits via the in-flight branch rather than
2073
+ // delivering onto the stale channel. Routing hygiene only -- the
2074
+ // wait keys on the park-generation edge, not this level state.
2075
+ runInputChannels.delete(runId);
2076
+ // Durable-consume contract, mirroring the trigger.fire path: hold
2077
+ // markConsumed until the child has durably taken up the signal --
2078
+ // the resumed run re-parks or reaches a terminal event. That gate
2079
+ // is downstream of durability DESPITE the child's fire-and-forget
2080
+ // SignalReceived writer: the runtime reaches re-park/terminal only
2081
+ // by resuming from the COMMITTED SignalReceived, which its per-run
2082
+ // subscribeKind substrate subscription surfaces only after the
2083
+ // commit lands -- so the substrate subscription IS the ack, and a
2084
+ // failed deliver commit is observed by nothing, never re-parks, and
2085
+ // never releases markConsumed (the mail stays reclaimable). A crash
2086
+ // before the re-park/terminal leaves the claim-check entry in
2087
+ // processing/, so replayProcessingToInbox re-delivers the signal on
2088
+ // restart. On cohort abort the wait returns and the post-loop guard
2089
+ // skips markConsumed.
2090
+ waitEntered = true;
2091
+ await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
2092
+ }
2093
+ finally {
2094
+ // waitForRunTerminalOrPark finalizes the iterator it consumes; the
2095
+ // only leak is when `sender.send` throws before the wait is
2096
+ // entered, so finalize only in that case.
2097
+ if (!waitEntered && typeof iter.return === "function") {
2098
+ await iter.return(undefined).catch(() => {
2099
+ /* best-effort finalisation of the watcher iterator. */
2100
+ });
2101
+ }
2102
+ }
2103
+ break;
2104
+ }
2105
+ if (!cohortRunIds.has(runId)) {
2106
+ // Subscribe the terminal watcher BEFORE the trigger fires. The
2107
+ // broadcaster drops a notify that has no listener (its subscribe-
2108
+ // before-fire contract), so a terminal that lands while
2109
+ // forwardDispatchedEntry is in flight would be lost and the wait
2110
+ // would hang to the backstop.
2111
+ const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
2112
+ let waitEntered = false;
2113
+ try {
2114
+ await forwardDispatchedEntry(sender, envelope.messageId, envelope.receivedAt, runId);
2115
+ // Wait for the child to process this trigger before allowing
2116
+ // `markConsumed` to move the claim-check entry out of
2117
+ // `processing/`. The child reads the trigger payload from that
2118
+ // entry; racing `markConsumed` would delete the entry before the
2119
+ // child resolves it. On cohort abort the wait returns and the
2120
+ // post-loop guard skips markConsumed.
2121
+ waitEntered = true;
2122
+ await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
2123
+ }
2124
+ finally {
2125
+ // waitForRunTerminalOrPark finalizes the iterator it consumes; the
2126
+ // only leak is when forward throws before the wait is
2127
+ // entered, so finalize only in that case.
2128
+ if (!waitEntered && typeof iter.return === "function") {
2129
+ await iter.return(undefined).catch(() => {
2130
+ /* best-effort finalisation of the watcher iterator. */
2131
+ });
2132
+ }
2133
+ }
2134
+ break;
2135
+ }
2136
+ const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
2137
+ const outcome = await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
2138
+ if (outcome === "aborted")
2139
+ break;
2140
+ if (outcome === "terminal") {
2141
+ // This message was waiting for an already-live run to expose its
2142
+ // next input correlation. The run terminated first, so the mail
2143
+ // was never delivered and must not fall through to trigger.fire.
2144
+ rejectTerminalRun();
2145
+ break;
2146
+ }
2147
+ // Continue loop: re-evaluate runInputChannels / cohortRunIds
2148
+ }
1383
2149
  }
1384
2150
  }
1385
- legMarkEnd(runId, "dequeue");
1386
- const iterable = broadcaster.source(runId);
1387
- const iter = iterable[Symbol.asyncIterator]();
1388
- await forwardDispatchedEntry(sender, envelope.messageId, envelope.receivedAt);
1389
- await waitForRunTerminal(iter, cohortAbort.signal);
1390
- emitDispatchTiming(runId, "reply-produced", performance.now());
1391
- inFlightRuns.delete(runId);
1392
2151
  if (cohortAbort.signal.aborted) {
1393
- // The cohort tore down before the terminal event arrived (or
1394
- // alongside it). Skip `markConsumed` so the recycle path's
1395
- // drain-side replay can reclaim the processing entry.
1396
- currentDispatchRunId = null;
1397
- resolveMarkConsumedWaiter(runId);
2152
+ currentDispatchMessageId = null;
1398
2153
  return false;
1399
2154
  }
1400
- // D2 leg: `markConsumed` is paid AFTER `reply-produced` (stamped
1401
- // above), so its growth is invisible to the 4.7 round-trip bracket --
1402
- // the leg mark makes the out-of-window cost visible.
1403
- legMarkStart(runId, "markconsumed");
2155
+ emitDispatchTiming(messageId, "reply-produced", performance.now());
2156
+ // D2 leg: `markConsumed` is paid AFTER `reply-produced`, so its growth
2157
+ // is invisible to the 4.7 round-trip bracket -- the leg mark makes the
2158
+ // out-of-window cost visible.
2159
+ legMarkStart(messageId, "markconsumed");
1404
2160
  try {
1405
2161
  await inboxPrimitives.markConsumed(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
1406
2162
  address: bindings.deploymentMailAddress,
@@ -1408,17 +2164,21 @@ export function createWorkflowSupervisor(bindings) {
1408
2164
  runId,
1409
2165
  consumedAt: Date.now(),
1410
2166
  retentionHorizonMs: consumedRetentionMs,
2167
+ ...(rejection !== undefined ? { rejection } : {}),
1411
2168
  });
1412
2169
  }
1413
2170
  catch (cause) {
1414
- const message = cause instanceof Error ? cause.message : String(cause);
1415
- logger.error `markConsumed failed for run ${runId}: ${message}`;
1416
- }
1417
- legMarkEnd(runId, "markconsumed");
1418
- resolveMarkConsumedWaiter(runId);
1419
- // §10c forced-repack A/B (measurement-only; no-op when unwired).
2171
+ // A markConsumed failure is fatal:
2172
+ // swallowing it treats the dispatch as complete while the mail is NOT
2173
+ // durably recorded consumed, hiding the failure and leaving a mail that
2174
+ // is neither cleanly consumed nor visibly failed. Propagate into the
2175
+ // dispatch fault handler so the failure surfaces and the mail stays
2176
+ // reclaimable.
2177
+ throw new Error(`failed to markConsumed for run ${runId}`, { cause });
2178
+ }
2179
+ legMarkEnd(messageId, "markconsumed");
1420
2180
  maybeRepack(runId);
1421
- currentDispatchRunId = null;
2181
+ currentDispatchMessageId = null;
1422
2182
  return true;
1423
2183
  }
1424
2184
  /**
@@ -1460,6 +2220,94 @@ export function createWorkflowSupervisor(bindings) {
1460
2220
  }
1461
2221
  }
1462
2222
  }
2223
+ /**
2224
+ * Wait until the run's terminal event lands on the cohort
2225
+ * broadcaster's iterator, the child parks the run (bumping the park
2226
+ * generation past `sinceGen`), or the cohort aborts. Returns `"terminal"`
2227
+ * when a terminal event arrived, `"parked"` when the run parked,
2228
+ * and `"aborted"` when the cohort tore down. Throws when the backstop
2229
+ * fires (see `TERMINAL_OR_PARK_BACKSTOP_MS`).
2230
+ *
2231
+ * `sinceGen` is the park generation the CALLER captured before its pre-wait
2232
+ * awaits; the wait accepts only a STRICTLY NEWER park (`generation >
2233
+ * sinceGen`). Keying on that edge -- not `runInputChannels`' level state --
2234
+ * makes the wait's correctness local: a park during the pre-wait awaits is
2235
+ * observed even though its `resolveParkNotifyWaiter` no-op'd, and a stale
2236
+ * channel entry from a prior run or incarnation cannot false-positive.
2237
+ */
2238
+ async function waitForRunTerminalOrPark(iter, abortSignal, runId, sinceGen) {
2239
+ let onAbort = null;
2240
+ const abortPromise = new Promise((resolve) => {
2241
+ if (abortSignal.aborted) {
2242
+ resolve({ source: "abort" });
2243
+ return;
2244
+ }
2245
+ onAbort = () => resolve({ source: "abort" });
2246
+ abortSignal.addEventListener("abort", onAbort, { once: true });
2247
+ });
2248
+ let parkResolve = null;
2249
+ const parkPromise = new Promise((resolve) => {
2250
+ parkResolve = () => resolve({ source: "park" });
2251
+ parkNotifyWaiters.set(runId, parkResolve);
2252
+ });
2253
+ let timeoutHandle;
2254
+ const timeoutPromise = new Promise((resolve) => {
2255
+ timeoutHandle = setTimeout(() => resolve({ source: "timeout" }), TERMINAL_OR_PARK_BACKSTOP_MS);
2256
+ });
2257
+ try {
2258
+ if (abortSignal.aborted)
2259
+ return "aborted";
2260
+ // Check-after-register: read the generation now that the waiter above is
2261
+ // armed, SYNCHRONOUSLY (no await between arming and this read, so no
2262
+ // `park.notify` can interleave). A generation past `sinceGen` means the
2263
+ // run already parked -- during the caller's pre-wait awaits, before the
2264
+ // waiter armed, so `resolveParkNotifyWaiter` no-op'd and the armed
2265
+ // parkPromise would never fire -- and this catches it rather than hanging
2266
+ // to the backstop.
2267
+ if ((parkGenerations.get(runId) ?? 0) > sinceGen)
2268
+ return "parked";
2269
+ const result = await Promise.race([
2270
+ iter.next().then((r) => ({ source: "iter", r })),
2271
+ abortPromise,
2272
+ parkPromise,
2273
+ timeoutPromise,
2274
+ ]);
2275
+ if (result.source === "abort")
2276
+ return "aborted";
2277
+ if (result.source === "park")
2278
+ return "parked";
2279
+ if (result.source === "timeout") {
2280
+ // Backstop against a lost wake or a wedged child: the run neither
2281
+ // parked, terminated, nor aborted within a generous window. Surface it
2282
+ // LOUDLY and throw so the dispatch fails -- the caller does not
2283
+ // markConsumed on a throw, so the mail stays reclaimable in
2284
+ // processing/ and is never consumed on the assumption the run
2285
+ // progressed.
2286
+ logger.error `waitForRunTerminalOrPark backstop fired for run ${runId} after ${TERMINAL_OR_PARK_BACKSTOP_MS}ms; failing the dispatch so the mail stays reclaimable`;
2287
+ throw new Error(`waitForRunTerminalOrPark backstop: run ${runId} did not park or terminate within ${TERMINAL_OR_PARK_BACKSTOP_MS}ms`);
2288
+ }
2289
+ if (result.r.done === true)
2290
+ return "aborted";
2291
+ // A terminal event for this runId arrived; stop waiting.
2292
+ return "terminal";
2293
+ }
2294
+ finally {
2295
+ if (timeoutHandle !== undefined) {
2296
+ clearTimeout(timeoutHandle);
2297
+ }
2298
+ if (parkResolve !== null) {
2299
+ parkNotifyWaiters.delete(runId);
2300
+ }
2301
+ if (onAbort !== null) {
2302
+ abortSignal.removeEventListener("abort", onAbort);
2303
+ }
2304
+ if (typeof iter.return === "function") {
2305
+ await iter.return(undefined).catch(() => {
2306
+ /* swallowed: best-effort finalisation of the watcher iterator. */
2307
+ });
2308
+ }
2309
+ }
2310
+ }
1463
2311
  /**
1464
2312
  * The dispatch loop body. Runs until the cohort aborts; each
1465
2313
  * iteration drains one inbox entry through the FIFO claim-check
@@ -1482,6 +2330,13 @@ export function createWorkflowSupervisor(bindings) {
1482
2330
  return;
1483
2331
  }
1484
2332
  while (!cohortAbort.signal.aborted) {
2333
+ // Capture the wake BEFORE the dispatch iteration (capture-before-check,
2334
+ // same discipline as the park-generation latch). `wakeDispatch` resolves
2335
+ // the CURRENT promise and swaps in a fresh one, so a mail that enqueues
2336
+ // DURING dispatchOne resolves THIS captured promise; capturing it after
2337
+ // dispatchOne would await the fresh, unresolved promise and strand that
2338
+ // mail until some later wake.
2339
+ const wake = dispatchWake.promise;
1485
2340
  let dispatched;
1486
2341
  try {
1487
2342
  dispatched = await dispatchOne(sender, cohortAbort, broadcaster);
@@ -1499,7 +2354,6 @@ export function createWorkflowSupervisor(bindings) {
1499
2354
  continue;
1500
2355
  if (cohortAbort.signal.aborted)
1501
2356
  return;
1502
- const wake = dispatchWake.promise;
1503
2357
  const abortPromise = new Promise((resolve) => {
1504
2358
  if (cohortAbort.signal.aborted) {
1505
2359
  resolve();
@@ -1517,7 +2371,7 @@ export function createWorkflowSupervisor(bindings) {
1517
2371
  substrate: bindings.repoStore,
1518
2372
  repoId: bindings.workflowRunRepoId,
1519
2373
  ref: bindings.workflowRunRef,
1520
- deploymentId: bindings.deploymentId,
2374
+ anchorRunId: bindings.anchorRunId,
1521
2375
  runId: opts.runId,
1522
2376
  origin: opts.origin,
1523
2377
  reason: opts.reason,
@@ -1530,7 +2384,9 @@ export function createWorkflowSupervisor(bindings) {
1530
2384
  await shutdownInternal({ reason: "shutdown requested" });
1531
2385
  }
1532
2386
  async function shutdownInternal(opts) {
1533
- if (state.phase === "idle" || state.phase === "stopped")
2387
+ if (state.phase === "idle" ||
2388
+ state.phase === "stopped" ||
2389
+ state.phase === "crash-looping")
1534
2390
  return;
1535
2391
  const prior = state;
1536
2392
  state = { phase: "stopping" };
@@ -1566,11 +2422,15 @@ export function createWorkflowSupervisor(bindings) {
1566
2422
  }
1567
2423
  }
1568
2424
  drainAccumulators.clear();
2425
+ cohortRunIds.clear();
2426
+ runInputChannels.clear();
2427
+ parkNotifyWaiters.clear();
2428
+ parkGenerations.clear();
1569
2429
  if (prior.phase === "starting" ||
1570
2430
  prior.phase === "running" ||
1571
2431
  prior.phase === "recycling") {
1572
2432
  prior.terminalCohortAbort.abort();
1573
- // Reject every pending merge round-trip and markConsumed waiter
2433
+ // Reject every pending merge round-trip and park-notify waiter
1574
2434
  // so handler closures awaiting them (including fire-and-forget
1575
2435
  // `handleSubstrateWriteRequest` instances) cannot outlive the
1576
2436
  // dying cohort. Without this, the `await new Promise` inside
@@ -1628,6 +2488,24 @@ export function createWorkflowSupervisor(bindings) {
1628
2488
  }
1629
2489
  recyclePolicy = null;
1630
2490
  }
2491
+ // Disarm the crash-loop stable-run reset timer so it cannot fire
2492
+ // against a torn-down supervisor. Also drop any pending child exit
2493
+ // that `maybeHandleChildExit` recorded but declined to act on -- it
2494
+ // leaves one pending when a respawn was in flight, or when the phase
2495
+ // had already left `running`. (The crash-loop latch path does NOT
2496
+ // leave one pending: `maybeHandleChildExit` nulls `pendingChildExit`
2497
+ // before invoking the handler that latches.) A shutdown-initiated
2498
+ // kill of a live child in the `finally` below resolves its
2499
+ // `handle.exited`, so the watcher may re-record a pending exit AFTER
2500
+ // this clear -- harmless: the phase is terminal, so
2501
+ // `maybeHandleChildExit` no-ops on it, and `spawn()` requires `idle`,
2502
+ // so the stale slot is never re-examined.
2503
+ clearStableRunResetTimer();
2504
+ // Cancel every armed respawn backoff wait. The phase was flipped to
2505
+ // `stopping` synchronously above, so each parked respawn coroutine
2506
+ // this unblocks re-checks the phase and bails without respawning.
2507
+ cancelRespawnBackoffWaits();
2508
+ pendingChildExit = null;
1631
2509
  spawnContext = null;
1632
2510
  if (prior.phase === "starting" ||
1633
2511
  prior.phase === "running" ||
@@ -1675,7 +2553,7 @@ export function createWorkflowSupervisor(bindings) {
1675
2553
  /* swallowed for the same reason as above. */
1676
2554
  });
1677
2555
  }
1678
- state = { phase: "stopped" };
2556
+ state = { phase: opts.terminalPhase ?? "stopped" };
1679
2557
  }
1680
2558
  logger.info `supervisor shutdown complete (${opts.reason})`;
1681
2559
  }
@@ -1702,11 +2580,11 @@ export function createWorkflowSupervisor(bindings) {
1702
2580
  */
1703
2581
  async function drainImpl(opts, ctx) {
1704
2582
  // Drain is meaningful only when a workflow-process child is up;
1705
- // calling it from `idle`/`stopping`/`stopped` is a no-op so the
1706
- // higher-level host shutdown sequence can call drain
1707
- // unconditionally without sniffing the phase. The recycle path
1708
- // calls drain via `drainImpl({}, { fromRecycle: true })` and
1709
- // admits `recycling` because the drain step runs against a
2583
+ // calling it from any non-active phase (`idle`/`stopping`/`stopped`/
2584
+ // `crash-looping`) is a no-op so the higher-level host shutdown
2585
+ // sequence can call drain unconditionally without sniffing the phase.
2586
+ // The recycle path calls drain via `drainImpl({}, { fromRecycle: true })`
2587
+ // and admits `recycling` because the drain step runs against a
1710
2588
  // still-live controlSender before the kill lands.
1711
2589
  if (state.phase !== "running" &&
1712
2590
  state.phase !== "starting" &&
@@ -1730,15 +2608,22 @@ export function createWorkflowSupervisor(bindings) {
1730
2608
  // body's cancellation cascade tears the run down without the
1731
2609
  // supervisor having to thread any per-run wiring beyond what the
1732
2610
  // accumulator already encapsulates.
2611
+ //
2612
+ // Runs that are already parked do not need drain escalation; the
2613
+ // runtime parks the run and the supervisor simply stops delivering
2614
+ // new mail. The cohort abort on shutdown/recycle will eventually
2615
+ // tear the run down.
1733
2616
  const cohortSource = perCohortTerminalSource(state.terminalCohortAbort, state.terminalBroadcaster);
1734
- for (const runId of inFlightRuns) {
2617
+ for (const runId of cohortRunIds) {
1735
2618
  if (drainAccumulators.has(runId))
1736
2619
  continue;
2620
+ if (runInputChannels.has(runId))
2621
+ continue;
1737
2622
  const accumulator = accumulatorFactory({
1738
2623
  substrate: bindings.repoStore,
1739
2624
  repoId: bindings.workflowRunRepoId,
1740
2625
  ref: bindings.workflowRunRef,
1741
- deploymentId: bindings.deploymentId,
2626
+ anchorRunId: bindings.anchorRunId,
1742
2627
  runId,
1743
2628
  signAsPrincipal: bindings.signAsPrincipal,
1744
2629
  drainTimeoutMs,
@@ -1752,7 +2637,7 @@ export function createWorkflowSupervisor(bindings) {
1752
2637
  }
1753
2638
  }
1754
2639
  async function recycle(opts) {
1755
- if (recycleInProgress) {
2640
+ if (respawnInProgress) {
1756
2641
  throw new Error("supervisor: recycle already in progress");
1757
2642
  }
1758
2643
  if (state.phase !== "running") {
@@ -1761,10 +2646,43 @@ export function createWorkflowSupervisor(bindings) {
1761
2646
  if (spawnContext === null) {
1762
2647
  throw new Error("supervisor: recycle called without a spawn context; spawn() must complete first");
1763
2648
  }
1764
- recycleInProgress = true;
1765
- const origin = opts.origin ?? "operator";
1766
- const prior = state;
1767
- const priorContext = spawnContext;
2649
+ // The contention read (`respawnInProgress`) stays here in the caller's
2650
+ // precondition zone rather than inside `runRespawn`: an operator double-
2651
+ // recycle is an error and must throw. `runRespawn` sets the latch
2652
+ // synchronously at entry, so this read and that set are never separated
2653
+ // by an await, and no second respawn can interleave between them.
2654
+ return runRespawn({
2655
+ origin: opts.origin ?? "operator",
2656
+ reason: opts.reason,
2657
+ prior: state,
2658
+ priorContext: spawnContext,
2659
+ drain: async (deadlineMs) => {
2660
+ // The recycle path's drain step shares the drain primitive but
2661
+ // bypasses the public surface's `recycling` silent-no-op so the
2662
+ // still-live controlSender (this step runs BEFORE abortPriorCohort
2663
+ // + kill) receives the frame. The public `drain()` silently no-ops
2664
+ // on `recycling` for external callers because the kill/respawn gap
2665
+ // can leave the controlSender dying.
2666
+ await drainImpl({ deadlineMs }, { fromRecycle: true });
2667
+ },
2668
+ });
2669
+ }
2670
+ /**
2671
+ * Shared kill/replay/respawn/install driver. The caller has already
2672
+ * verified the supervisor is `running` with a live spawn context and
2673
+ * snapshotted both as `prior`/`priorContext`; this function transitions
2674
+ * to `recycling`, runs the six-step `triggerRecycle` sequence with the
2675
+ * caller-supplied `drain` step, and swaps in the new cohort via the
2676
+ * inline `installNewChild` callback. The operator/policy/self recycle
2677
+ * path is the sole caller today; the crash-respawn path will call it with
2678
+ * a no-op drain (its child is already dead).
2679
+ */
2680
+ async function runRespawn(args) {
2681
+ // Set synchronously at entry (before any await) so the caller's
2682
+ // contention read and this set cannot be separated by an event-loop
2683
+ // turn; two respawns can never interleave.
2684
+ respawnInProgress = true;
2685
+ const { origin, reason, prior, priorContext, drain } = args;
1768
2686
  // The cohort abort no longer fires up-front. triggerRecycle drives
1769
2687
  // the drain and replay steps against a LIVE cohort first, then
1770
2688
  // invokes `abortPriorCohort` (the callback below) between replay
@@ -1808,16 +2726,7 @@ export function createWorkflowSupervisor(bindings) {
1808
2726
  channelId: prior.channelId,
1809
2727
  eventPump: prior.eventPump,
1810
2728
  },
1811
- drain: async (deadlineMs) => {
1812
- // The recycle path's drain step shares the drain
1813
- // primitive but bypasses the public surface's `recycling`
1814
- // silent-no-op so the still-live controlSender (this
1815
- // step runs BEFORE abortPriorCohort + kill) receives the
1816
- // frame. The public `drain()` silently no-ops on
1817
- // `recycling` for external callers because the
1818
- // kill/respawn gap can leave the controlSender dying.
1819
- await drainImpl({ deadlineMs }, { fromRecycle: true });
1820
- },
2729
+ drain,
1821
2730
  replayProcessingToInbox: async () => {
1822
2731
  await inboxPrimitives.replayProcessingToInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress);
1823
2732
  },
@@ -1828,6 +2737,13 @@ export function createWorkflowSupervisor(bindings) {
1828
2737
  prior.terminalCohortAbort.abort();
1829
2738
  wakeDispatch();
1830
2739
  },
2740
+ // Kept inline rather than extracted: this cohort-swap closes over
2741
+ // the supervisor's full mutable cohort state (drainAccumulators,
2742
+ // cohortRunIds, runInputChannels, parkNotifyWaiters, parkGenerations,
2743
+ // rejectCohortAwaiters, spawnContext, plus `prior`/`priorContext`).
2744
+ // A standalone helper would take all of it as parameters for zero
2745
+ // reuse -- the crash-respawn path reaches this callback transitively
2746
+ // through `runRespawn`, so it needs no separate extraction.
1831
2747
  installNewChild: ({ wiring, credentialsSnapshot, controlIncoming, }) => {
1832
2748
  // Phase guard: a `shutdown()` that landed during the
1833
2749
  // kill/respawn gap (between `subprocessSpawner` and this
@@ -1871,7 +2787,11 @@ export function createWorkflowSupervisor(bindings) {
1871
2787
  accumulator.stop();
1872
2788
  }
1873
2789
  drainAccumulators.clear();
1874
- // Reject every pending merge round-trip and markConsumed
2790
+ cohortRunIds.clear();
2791
+ runInputChannels.clear();
2792
+ parkNotifyWaiters.clear();
2793
+ parkGenerations.clear();
2794
+ // Reject every pending merge round-trip and park-notify
1875
2795
  // waiter registered against the dying cohort so handler
1876
2796
  // closures cannot survive the kill/respawn gap. The new
1877
2797
  // child will re-issue substrate writes through fresh
@@ -1907,6 +2827,12 @@ export function createWorkflowSupervisor(bindings) {
1907
2827
  dispatchLoop: newDispatchLoop,
1908
2828
  replayDone: null,
1909
2829
  };
2830
+ // Bump the generation and arm the exit-watcher for the
2831
+ // respawned child atomically with this running transition, so
2832
+ // the predecessor's watcher (already stale by generation) never
2833
+ // drives a spurious respawn and a crash of THIS child is
2834
+ // classified against the new generation.
2835
+ armChildForRunning(wiring.handle);
1910
2836
  // Cache fresh spawn context with the updated spawnedAt
1911
2837
  // so the policy timer's uptime check resets on recycle.
1912
2838
  const now = bindings.recyclePolicyNow ?? defaultNow;
@@ -1934,6 +2860,16 @@ export function createWorkflowSupervisor(bindings) {
1934
2860
  // entries the previous cohort's replayProcessingToInbox
1935
2861
  // just moved back.
1936
2862
  wakeDispatch();
2863
+ // Trigger A on the recycle seam: the respawned child re-parks any
2864
+ // surviving parked run without re-emitting, and a recycle leaves
2865
+ // the hub link untouched so the reconnect trigger never fires --
2866
+ // so re-drive the re-registration here too. Same fire-and-forget
2867
+ // contract as the spawn seam; the fresh cohort's controlSender is
2868
+ // in `state` now, and its pump (armed above) routes the response.
2869
+ void reEmitParkedCorrelations().catch((cause) => {
2870
+ const message = cause instanceof Error ? cause.message : String(cause);
2871
+ logger.warn `re-emit of parked correlations on re-establishment failed: ${message}`;
2872
+ });
1937
2873
  },
1938
2874
  onCrash: onChildCrash,
1939
2875
  // Edge-resolved once at the supervisor factory; recycle bounds
@@ -1945,7 +2881,7 @@ export function createWorkflowSupervisor(bindings) {
1945
2881
  ...(bindings.recyclePolicyClearTimer !== undefined
1946
2882
  ? { clearTimer: bindings.recyclePolicyClearTimer }
1947
2883
  : {}),
1948
- }, { origin, reason: opts.reason });
2884
+ }, { origin, reason });
1949
2885
  // After the recycle, await the previous cohort's dispatch
1950
2886
  // loop so a teardown coroutine cannot survive past the
1951
2887
  // recycle's return point.
@@ -1963,7 +2899,18 @@ export function createWorkflowSupervisor(bindings) {
1963
2899
  // level shutdown. Tear the prior cohort down through the same
1964
2900
  // path a real shutdown uses so the supervisor reaches a clean
1965
2901
  // `stopped` state, then re-throw so the operator sees the
1966
- // recycle failure and can redeploy.
2902
+ // failure and can redeploy.
2903
+ //
2904
+ // This teardown-to-`stopped` is shared by both callers, and that is
2905
+ // deliberate. A crash-origin respawn whose spawn/wire/handshake fails
2906
+ // is a broken deploy (a bad binary, unreadable credentials), NOT a
2907
+ // flapping child, so it does NOT feed the crash-loop exit counter and
2908
+ // does NOT reach `crash-looping`; conflating a mechanism failure with
2909
+ // a flap would muddy what that counter means. It reaches `stopped`,
2910
+ // the same terminal a failed operator recycle reaches. (Consequence:
2911
+ // this path leaves no `RunFailed` tombstone, unlike the exit-count
2912
+ // latch; the crash-respawn happy path -- a clean child death whose
2913
+ // respawn succeeds -- is what the crash-loop guard bounds.)
1967
2914
  const message = cause instanceof Error ? cause.message : String(cause);
1968
2915
  logger.error `recycle failed; tearing supervisor down: ${message}`;
1969
2916
  await shutdownInternal({
@@ -1977,7 +2924,12 @@ export function createWorkflowSupervisor(bindings) {
1977
2924
  throw cause;
1978
2925
  }
1979
2926
  finally {
1980
- recycleInProgress = false;
2927
+ respawnInProgress = false;
2928
+ // Drain any child exit observed during the respawn. A crash of the
2929
+ // freshly-installed child that raced this respawn's completion was
2930
+ // deferred (respawnInProgress was set); handle it now that the latch
2931
+ // is clear. A pending exit for a superseded generation drops as stale.
2932
+ maybeHandleChildExit();
1981
2933
  }
1982
2934
  return attempt;
1983
2935
  }
@@ -2026,6 +2978,20 @@ export function createWorkflowSupervisor(bindings) {
2026
2978
  },
2027
2979
  });
2028
2980
  }
2981
+ async function deliverCredentials(opts) {
2982
+ // The supervisor is the single producer of `credentials-updated` control
2983
+ // frames. Phase-guarded exactly like `deliverSources`: outside
2984
+ // starting/running the control sender points at a dying child, so a frame
2985
+ // would buffer behind the SIGTERM or write into a closed pipe. Rejecting
2986
+ // surfaces the race so the caller can retry once the recycle completes.
2987
+ if (state.phase !== "running" && state.phase !== "starting") {
2988
+ throw new Error(`supervisor: deliverCredentials called in phase ${state.phase}; expected starting/running`);
2989
+ }
2990
+ await state.controlSender.send({
2991
+ type: "credentials-updated",
2992
+ data: { delivery: opts.delivery },
2993
+ });
2994
+ }
2029
2995
  function getCredentialsSnapshot() {
2030
2996
  if (state.phase === "starting" || state.phase === "running") {
2031
2997
  return state.credentialsSnapshot;
@@ -2040,6 +3006,8 @@ export function createWorkflowSupervisor(bindings) {
2040
3006
  recycle,
2041
3007
  deliverSignal,
2042
3008
  deliverSources,
3009
+ deliverCredentials,
3010
+ reEmitParkedCorrelations,
2043
3011
  getCredentialsSnapshot,
2044
3012
  };
2045
3013
  }
@@ -2097,71 +3065,6 @@ async function pumpEvents(iter, onInferenceEvent) {
2097
3065
  function defaultInProcessMailAuditRef(messageId, _rawMessage) {
2098
3066
  return { store: "in-process", path: messageId };
2099
3067
  }
2100
- /**
2101
- * Derive a stable message identifier from the raw bytes the bus
2102
- * delivered. The RFC 2822 `Message-ID` header (if present) is the
2103
- * canonical identifier the audit log surfaces as
2104
- * `RunStarted.consumedMessageId`; downstream consumers join inbound
2105
- * mail to workflow-run events on this value, so the header parse must
2106
- * win when the sender emitted one. A message that lacks a
2107
- * `Message-ID` header falls back to a sha256 of the raw bytes so
2108
- * runs originating from non-RFC 2822 transports still receive a
2109
- * deterministic identifier.
2110
- *
2111
- * The parser walks the message until the headers/body separator
2112
- * (`CRLF CRLF` per RFC 2822 §2.1, with the lone-`LF` variant tolerated
2113
- * to match common in-memory senders). Header-field unfolding follows
2114
- * RFC 2822 §2.2.3: a continuation line begins with whitespace and
2115
- * appends to the prior line. Header-name comparison is
2116
- * case-insensitive per RFC 2822 §1.2.2.
2117
- */
2118
- async function deriveMessageId(rawMessage) {
2119
- const messageIdFromHeader = parseMessageIdHeader(rawMessage);
2120
- if (messageIdFromHeader !== null) {
2121
- return messageIdFromHeader;
2122
- }
2123
- const digest = await crypto.subtle.digest("SHA-256",
2124
- // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- ArrayBuffer-backed at the call site; Web Crypto's BufferSource type rejects Uint8Array<ArrayBufferLike> under TS 5.9 (microsoft/TypeScript#62240)
2125
- rawMessage);
2126
- return hexEncode(new Uint8Array(digest));
2127
- }
2128
- function parseMessageIdHeader(rawMessage) {
2129
- const text = new TextDecoder("utf-8", { fatal: false }).decode(rawMessage);
2130
- // Headers end at the first blank line. RFC 2822 mandates `CRLF CRLF`
2131
- // but tolerate `LF LF` for callers that normalize line endings.
2132
- let headerSection = text;
2133
- const crlfBoundary = text.indexOf("\r\n\r\n");
2134
- const lfBoundary = text.indexOf("\n\n");
2135
- if (crlfBoundary >= 0 && (lfBoundary < 0 || crlfBoundary < lfBoundary)) {
2136
- headerSection = text.slice(0, crlfBoundary);
2137
- }
2138
- else if (lfBoundary >= 0) {
2139
- headerSection = text.slice(0, lfBoundary);
2140
- }
2141
- // Unfold continuation lines (a line starting with WSP belongs to
2142
- // the prior header field).
2143
- const lines = headerSection.split(/\r?\n/);
2144
- const unfolded = [];
2145
- for (const line of lines) {
2146
- if (line.length > 0 && (line[0] === " " || line[0] === "\t")) {
2147
- if (unfolded.length === 0)
2148
- continue;
2149
- unfolded[unfolded.length - 1] += " " + line.trim();
2150
- continue;
2151
- }
2152
- unfolded.push(line);
2153
- }
2154
- for (const line of unfolded) {
2155
- const colon = line.indexOf(":");
2156
- if (colon < 0)
2157
- continue;
2158
- const name = line.slice(0, colon).trim().toLowerCase();
2159
- if (name !== "message-id")
2160
- continue;
2161
- return line.slice(colon + 1).trim();
2162
- }
2163
- return null;
2164
- }
2165
3068
  /**
2166
3069
  * Project the wire shape of a `terminal.event` upstream control frame
2167
3070
  * into the workflow-vocabulary `TerminalRunEvent` discriminated union