@intx/workflow-host 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +77 -14
  2. package/dist/adapters/mail-part-store.d.ts +46 -0
  3. package/dist/adapters/mail-part-store.js +251 -0
  4. package/dist/adapters/repo-store.d.ts +22 -1
  5. package/dist/adapters/repo-store.js +56 -65
  6. package/dist/adapters/spawn-child.d.ts +109 -44
  7. package/dist/adapters/spawn-child.js +77 -81
  8. package/dist/adapters/step-invoker.d.ts +52 -2
  9. package/dist/adapters/step-invoker.js +284 -37
  10. package/dist/adapters/substrate-mailbox-store.d.ts +80 -0
  11. package/dist/adapters/substrate-mailbox-store.js +404 -0
  12. package/dist/child/child-mailbox-reader.d.ts +10 -0
  13. package/dist/child/child-mailbox-reader.js +23 -0
  14. package/dist/child/credential-cell.d.ts +8 -0
  15. package/dist/child/credential-cell.js +66 -0
  16. package/dist/child/env-bootstrap.d.ts +20 -6
  17. package/dist/child/env-bootstrap.js +9 -1
  18. package/dist/child/from-process-env.d.ts +12 -0
  19. package/dist/child/from-process-env.js +6 -0
  20. package/dist/child/index.d.ts +6 -2
  21. package/dist/child/index.js +4 -1
  22. package/dist/child/mailbox-mutation-bridge.d.ts +61 -0
  23. package/dist/child/mailbox-mutation-bridge.js +101 -0
  24. package/dist/child/mailbox-watch-registry.d.ts +17 -0
  25. package/dist/child/mailbox-watch-registry.js +61 -0
  26. package/dist/child/outbound-mail-bridge.d.ts +3 -2
  27. package/dist/child/outbound-mail-bridge.js +20 -32
  28. package/dist/child/parked-correlations.d.ts +42 -0
  29. package/dist/child/parked-correlations.js +80 -0
  30. package/dist/child/pending-request.d.ts +89 -0
  31. package/dist/child/pending-request.js +80 -0
  32. package/dist/child/proxy-repo-store.d.ts +3 -2
  33. package/dist/child/proxy-repo-store.js +2 -0
  34. package/dist/child/run-child.d.ts +170 -14
  35. package/dist/child/run-child.js +569 -155
  36. package/dist/child/self-discovery.d.ts +10 -0
  37. package/dist/child/self-discovery.js +25 -1
  38. package/dist/child/substrate-write-bridge.d.ts +3 -2
  39. package/dist/child/substrate-write-bridge.js +21 -38
  40. package/dist/child/supervisor-backed-transport.d.ts +52 -6
  41. package/dist/child/supervisor-backed-transport.js +205 -62
  42. package/dist/child/verified-definition-loader.d.ts +33 -0
  43. package/dist/child/verified-definition-loader.js +43 -0
  44. package/dist/child/warm-agent-cache.d.ts +44 -4
  45. package/dist/child/warm-agent-cache.js +41 -10
  46. package/dist/index.d.ts +6 -4
  47. package/dist/index.js +6 -4
  48. package/dist/ipc/control-channel.d.ts +151 -2
  49. package/dist/ipc/control-channel.js +222 -29
  50. package/dist/ipc/event-channel.d.ts +32 -1
  51. package/dist/ipc/index.d.ts +1 -1
  52. package/dist/ipc/index.js +1 -1
  53. package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
  54. package/dist/mail-bus/hub-transport-adapter.js +9 -5
  55. package/dist/run-body-then-cleanup.d.ts +17 -0
  56. package/dist/run-body-then-cleanup.js +38 -0
  57. package/dist/seams/scheduler.d.ts +16 -6
  58. package/dist/seams/scheduler.js +87 -97
  59. package/dist/supervisor/cancel-signing.d.ts +2 -2
  60. package/dist/supervisor/cancel-signing.js +4 -8
  61. package/dist/supervisor/credentials.d.ts +28 -15
  62. package/dist/supervisor/credentials.js +7 -7
  63. package/dist/supervisor/dispatch-attribution.js +1 -1
  64. package/dist/supervisor/drain-timeout.d.ts +2 -2
  65. package/dist/supervisor/drain-timeout.js +1 -1
  66. package/dist/supervisor/index.d.ts +3 -3
  67. package/dist/supervisor/index.js +2 -2
  68. package/dist/supervisor/recycle.d.ts +10 -3
  69. package/dist/supervisor/recycle.js +18 -7
  70. package/dist/supervisor/run-event-compaction.d.ts +5 -5
  71. package/dist/supervisor/run-event-compaction.js +14 -19
  72. package/dist/supervisor/run-event-recovery.d.ts +34 -0
  73. package/dist/supervisor/run-event-recovery.js +45 -0
  74. package/dist/supervisor/spawn-env.d.ts +2 -2
  75. package/dist/supervisor/spawn-env.js +1 -1
  76. package/dist/supervisor/supervisor.d.ts +106 -26
  77. package/dist/supervisor/supervisor.js +1903 -414
  78. package/dist/supervisor/terminal-commit.d.ts +36 -0
  79. package/dist/supervisor/terminal-commit.js +130 -0
  80. package/dist/supervisor/types.d.ts +180 -23
  81. package/dist/testing/change-notifier.d.ts +12 -0
  82. package/dist/testing/change-notifier.js +63 -0
  83. package/dist/testing/index.d.ts +8 -0
  84. package/dist/testing/index.js +16 -0
  85. package/dist/testing/log-capture.d.ts +52 -0
  86. package/dist/testing/log-capture.js +124 -0
  87. package/dist/testing/mail-bus.d.ts +22 -0
  88. package/dist/testing/mail-bus.js +78 -0
  89. package/dist/testing/memory-streams.d.ts +43 -0
  90. package/dist/testing/memory-streams.js +211 -0
  91. package/dist/testing/spawn-observer.d.ts +12 -0
  92. package/dist/testing/spawn-observer.js +36 -0
  93. package/dist/testing/stub-repo-store.d.ts +10 -0
  94. package/dist/testing/stub-repo-store.js +39 -0
  95. package/dist/testing/supervisor-reaper.d.ts +24 -0
  96. package/dist/testing/supervisor-reaper.js +49 -0
  97. package/dist/testing/upstream-frames.d.ts +47 -0
  98. package/dist/testing/upstream-frames.js +94 -0
  99. package/dist/workflow-definition-loader.d.ts +187 -0
  100. package/dist/workflow-definition-loader.js +422 -0
  101. package/package.json +18 -11
@@ -43,49 +43,90 @@ import { type } from "arktype";
43
43
  import { getLogger } from "@intx/log";
44
44
  import { sampleStructuralCounters, forceRepack, } from "./dispatch-attribution.js";
45
45
  import { generateKeyPair } from "@intx/crypto";
46
- import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed, readOwnedMessageIds, replayProcessingToInbox as defaultReplayProcessingToInbox, DEFAULT_CONSUMED_RETENTION_MS, } from "@intx/hub-sessions/substrate";
47
- import { base64Decode, base64Encode, hexEncode } from "@intx/types";
46
+ import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed, scanRunsForBoot, readWorkflowRunLifecycle, replayProcessingToInbox as defaultReplayProcessingToInbox, StaleInboxEnqueueError, DEFAULT_CONSUMED_RETENTION_MS, } from "@intx/hub-sessions/substrate";
47
+ import { base64Decode, base64Encode, deriveMessageId, deriveWorkflowRunId, signalName, } from "@intx/types";
48
48
  import { RepoId } from "@intx/types/sidecar";
49
49
  import { createControlChannelSender, generateChannelId, generateHmacKey, receiveControlChannel, receiveEventChannel, } from "../ipc/index.js";
50
50
  import { assembleCredentialsSnapshot, } from "./credentials.js";
51
51
  import { commitCancelRequested } from "./cancel-signing.js";
52
+ import { commitRunFailed } from "./terminal-commit.js";
52
53
  import { buildChildSpawnEnv } from "./spawn-env.js";
53
54
  import { compactRunEvents } from "./run-event-compaction.js";
55
+ import { recoverInterruptedCompactions } from "./run-event-recovery.js";
56
+ import { decodeMail } from "@intx/mime";
57
+ import { commitMail, InvalidMailError } from "../adapters/mail-part-store.js";
58
+ import { mergeCredentialDelivery } from "../child/credential-cell.js";
59
+ import { createSubstrateMailboxStore, MAILBOX_INBOX_DIR, } from "../adapters/substrate-mailbox-store.js";
54
60
  import { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, } from "./drain-timeout.js";
55
61
  import { createRecyclePolicy, triggerRecycle, } from "./recycle.js";
56
62
  import { createTerminalBroadcaster, } from "./terminal-broadcaster.js";
57
63
  import { DEFAULT_KILL_TIMEOUT_MS, DEFAULT_READY_TIMEOUT_MS, defaultClearTimer, defaultSetTimer, killChildHandle, waitDeadline, } from "./child-termination.js";
58
64
  const logger = getLogger(["workflow-host", "supervisor"]);
65
+ /** IMAP system flag marking a dispatched mailbox entry as read. */
66
+ const MAILBOX_FLAG_SEEN = "\\Seen";
59
67
  /**
60
- * Default watchdog timeout for the supervisor's
61
- * `synchronouslyDispatchTerminalWrite`. The handler holds the
62
- * `substrate.write.response` back to the child until the dispatch
63
- * loop's `markConsumed` settles for the matching terminal event; an
64
- * unbounded wait would chain into a child / runtime / dispatch loop
65
- * deadlock if `markConsumed` never armed (bug in the dispatch loop, a
66
- * torn-down cohort, a stalled inbox primitive). 30s sits between the
67
- * recycle path's `DEFAULT_KILL_TIMEOUT_MS` (5s, a hard process-level
68
- * kill cap) and `DEFAULT_DRAIN_TIMEOUT_MS` (60s, the per-deployment
69
- * drain budget) -- generous enough to absorb a slow legitimate
70
- * markConsumed, tight enough to surface a real deadlock long before
71
- * the drainTimeout would otherwise mask it.
68
+ * Interchange keyword flag marking a mailbox entry the supervisor has dispatched
69
+ * as a workflow turn (a `trigger.fire` or `signal.deliver`).
72
70
  */
73
- export const DEFAULT_TERMINAL_WRITE_WATCHDOG_MS = 30_000;
71
+ const MAILBOX_FLAG_PROCESSED = "$Processed";
74
72
  /**
75
- * Raised when a `pendingMerges` entry or a
76
- * `markConsumedCompletionWaiters` waiter is rejected because the
77
- * cohort it was registered against has been aborted (cohort transition
78
- * during a recycle, or a supervisor shutdown). Callers awaiting the
79
- * resolved value receive an instance of this error so the failure mode
80
- * is recognisable from a generic substrate-merge or markConsumed
81
- * failure.
73
+ * IMAP system flag marking a mailbox entry for expunge. The warm agent sets it
74
+ * (via `mail_flag`) to consume a processed message; a subsequent `expunge`
75
+ * sweeps every entry carrying it out of the live INBOX.
82
76
  */
83
- export class MergeAbortedError extends Error {
84
- constructor(reason) {
85
- super(`supervisor cohort aborted before completion: ${reason}`);
86
- this.name = "MergeAbortedError";
87
- }
88
- }
77
+ const MAILBOX_FLAG_DELETED = "\\Deleted";
78
+ /**
79
+ * Default crash-loop bound: the supervisor stops respawning and latches
80
+ * the deployment once the workflow-process child exits unexpectedly this
81
+ * many times within `DEFAULT_CRASH_LOOP_WINDOW_MS`. Overridable via
82
+ * `WorkflowSupervisorBindings.crashLoopMaxCount`.
83
+ */
84
+ export const DEFAULT_CRASH_LOOP_MAX_COUNT = 3;
85
+ /**
86
+ * Default sliding window (ms) over which `DEFAULT_CRASH_LOOP_MAX_COUNT`
87
+ * unexpected exits latch the deployment. Overridable via
88
+ * `WorkflowSupervisorBindings.crashLoopWindowMs`.
89
+ */
90
+ export const DEFAULT_CRASH_LOOP_WINDOW_MS = 60_000;
91
+ /**
92
+ * Default stable-run duration (ms): once a respawned child stays up this
93
+ * long, the crash counter resets so flapping followed by stability does
94
+ * not permanently latch. Overridable via
95
+ * `WorkflowSupervisorBindings.crashLoopStableResetMs`.
96
+ */
97
+ export const DEFAULT_CRASH_LOOP_STABLE_RESET_MS = 60_000;
98
+ /**
99
+ * Default initial respawn backoff (ms): the wait before the first respawn
100
+ * after an unexpected exit. Overridable via
101
+ * `WorkflowSupervisorBindings.respawnBackoffInitialMs`.
102
+ */
103
+ export const DEFAULT_RESPAWN_BACKOFF_INITIAL_MS = 1_000;
104
+ /**
105
+ * Default cap (ms) on the exponential respawn backoff. Kept below
106
+ * `DEFAULT_CRASH_LOOP_WINDOW_MS` so a slow flapper's crashes still fall
107
+ * within the window and latch the guard. Overridable via
108
+ * `WorkflowSupervisorBindings.respawnBackoffMaxMs`.
109
+ */
110
+ export const DEFAULT_RESPAWN_BACKOFF_MAX_MS = 30_000;
111
+ /**
112
+ * Default watchdog for `reEmitParkedCorrelations`' wait on the child's
113
+ * `parked-correlations.response`. 30s is generous enough for a healthy child
114
+ * to enumerate its in-flight runs and load each parked snapshot, tight
115
+ * enough that a wedged-but-alive child does not hang the reconnect caller
116
+ * until some coarser timeout intervenes.
117
+ */
118
+ export const DEFAULT_PARKED_QUERY_WATCHDOG_MS = 30_000;
119
+ /**
120
+ * Backstop for `waitForRunTerminalOrPark`. A dispatch waits here for the child
121
+ * to park or terminate the run before releasing `markConsumed`; a lost park
122
+ * wake or a wedged child would otherwise hang the deployment's dispatch loop
123
+ * forever. Five minutes is far beyond any healthy per-message dispatch (which
124
+ * settles in well under a second), so this never fires on a legitimately long
125
+ * run without also being a genuine fault -- and when it does fire it is logged
126
+ * loudly and fails the dispatch (the mail is left reclaimable, never consumed
127
+ * on the assumption the run made progress), not silently swallowed.
128
+ */
129
+ export const TERMINAL_OR_PARK_BACKSTOP_MS = 300_000;
89
130
  /**
90
131
  * Construct a per-deployment supervisor. All host-specific
91
132
  * dependencies are pulled in via `bindings`; nothing in the
@@ -93,23 +134,74 @@ export class MergeAbortedError extends Error {
93
134
  */
94
135
  export function createWorkflowSupervisor(bindings) {
95
136
  let state = { phase: "idle" };
137
+ // The live credential delivery to seed the child on every spawn and every
138
+ // pre-trigger barrier. Initialized from the deploy-time delivery and MUTATED
139
+ // by `deliverCredentials` on every runtime update, so a mid-life revocation or
140
+ // rotation is durable: the re-assertion sites read THIS mirror, not the frozen
141
+ // `bindings.credentialDelivery`. Reading the frozen delivery would re-add an
142
+ // evicted credential on the next spawn/barrier, and the warm-recycle path
143
+ // (seeded only by the barrier) would lose every runtime update. Normalized on
144
+ // init via the same merge the child applies, so the mirror matches the child's
145
+ // deduped cell from step zero. Lives outside `state` because `state` is a
146
+ // phase union replaced on every transition; this must survive them.
147
+ let currentCredentialDelivery = bindings.credentialDelivery !== undefined
148
+ ? mergeCredentialDelivery(null, bindings.credentialDelivery, undefined)
149
+ : null;
96
150
  /**
97
- * In-flight runIds the supervisor knows about. A runId enters this
98
- * set when the supervisor forwards a `trigger.fire` for it on the
99
- * control channel; the runId leaves the set when the dispatch
100
- * loop's terminal-event watcher fires `markConsumed`. The drain
101
- * path arms one accumulator per entry here.
151
+ * ALL runIds the current child cohort is driving, regardless of
152
+ * who spawned them: supervisor-dispatched + self-discovered.
153
+ * Populated at `trigger.fire` time and when the child reports
154
+ * `resumed.runs`. Removed on terminal event or cohort teardown.
155
+ *
156
+ * Used by: `drain()` to arm one drainTimeout accumulator per run.
157
+ */
158
+ const cohortRunIds = new Set();
159
+ /**
160
+ * Runs observed terminal in this supervisor process. The terminal control
161
+ * frame follows the durable event commit, but retaining the observation
162
+ * closes the short visibility window before the working tree reflects that
163
+ * commit. Terminal membership is permanent for a deployment: the stable
164
+ * top-level run is never cleared and fired again.
165
+ */
166
+ const terminalRunIds = new Set();
167
+ /**
168
+ * Per-run input channel cache. When a long-lived run parks on an input
169
+ * signal, the child sends `park.notify` with `parkKind: "input"`. The
170
+ * supervisor stores the `correlationId` here so that subsequent mail
171
+ * deliveries can fire `signal.deliver` without re-reading the substrate.
172
+ * Cleared on terminal event or cohort abort.
173
+ */
174
+ const runInputChannels = new Map();
175
+ /**
176
+ * Waiters for dispatch loops blocked on a `park.notify` for a
177
+ * specific runId. When `park.notify` arrives, the handler resolves
178
+ * the waiter so the dispatch loop re-evaluates routing.
102
179
  */
103
- const inFlightRuns = new Set();
104
- // D2 attribution (measurement-only): the runId the dispatch loop is
180
+ const parkNotifyWaiters = new Map();
181
+ /**
182
+ * Monotonic per-run INPUT-park generation. Bumped on every
183
+ * `park.notify(input)` for a runId. `waitForRunTerminalOrPark` captures a
184
+ * `sinceGen` before its caller's pre-wait awaits and returns `"parked"` when
185
+ * the generation later exceeds it, so the wait keys on the park EDGE rather
186
+ * than `runInputChannels`' LEVEL state: a park that fired during the pre-wait
187
+ * awaits (before the waiter armed, so `resolveParkNotifyWaiter` no-op'd) is
188
+ * still observed, and a stale `runInputChannels` entry from a prior run or
189
+ * incarnation cannot false-positive because it did not bump the generation
190
+ * past `sinceGen`. Incarnation-scoped: cleared with `parkNotifyWaiters` on
191
+ * recycle/teardown so a reused runId cannot carry a stale captured generation
192
+ * across incarnations.
193
+ */
194
+ const parkGenerations = new Map();
195
+ // D2 attribution (measurement-only): the messageId the dispatch loop is
105
196
  // currently servicing. Set at `dispatch-start`, cleared after
106
197
  // `reply-produced`. The dispatch loop is strictly serial (one message
107
198
  // in flight at a time -- the sustained interactive case the bench
108
- // drives), so a child-proxied WAL `substrate.write.request` (whose
109
- // `agent-state/<key>/...` preservePrefix carries no runId) is
110
- // unambiguously attributable to this runId. Run-event writes carry the
111
- // runId in their `runs/<runId>/events/` prefix and do not need it.
112
- let currentDispatchRunId = null;
199
+ // drives), so a child-proxied `substrate.write.request` is unambiguously
200
+ // attributable to this message. Both the WAL leg (whose `agent-state/...`
201
+ // prefix names no run) and the run-event leg (whose `runs/<runId>/events/`
202
+ // prefix names only the stable per-deployment run id, not the message)
203
+ // take their per-message key from here.
204
+ let currentDispatchMessageId = null;
113
205
  /**
114
206
  * Per-run drainTimeout accumulators armed by `drain()`. Held so
115
207
  * `shutdown()` can stop every accumulator cleanly before tearing
@@ -134,21 +226,21 @@ export function createWorkflowSupervisor(bindings) {
134
226
  }
135
227
  });
136
228
  const drainTimeoutMs = bindings.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
137
- const terminalWriteWatchdogMs = bindings.terminalWriteWatchdogMs ?? DEFAULT_TERMINAL_WRITE_WATCHDOG_MS;
229
+ const parkedQueryWatchdogMs = bindings.parkedQueryWatchdogMs ?? DEFAULT_PARKED_QUERY_WATCHDOG_MS;
138
230
  // Pure observability: invoke the dispatch-timing hook (when wired) at
139
231
  // the two per-message boundaries the 4.7 latency gate brackets. A
140
232
  // throwing observer is swallowed and logged so a benchmark hook bug
141
233
  // cannot wedge the dispatch loop.
142
- function emitDispatchTiming(runId, marker, atMs) {
234
+ function emitDispatchTiming(messageId, marker, atMs) {
143
235
  const observer = bindings.onDispatchTiming;
144
236
  if (observer === undefined)
145
237
  return;
146
238
  try {
147
- observer({ kind: "roundtrip", runId, marker, atMs });
239
+ observer({ kind: "roundtrip", messageId, marker, atMs });
148
240
  }
149
241
  catch (cause) {
150
242
  const message = cause instanceof Error ? cause.message : String(cause);
151
- logger.warn `onDispatchTiming observer threw for ${runId} (${marker}): ${message}`;
243
+ logger.warn `onDispatchTiming observer threw for ${messageId} (${marker}): ${message}`;
152
244
  }
153
245
  }
154
246
  // D2 per-leg attribution (measurement-only). Emits a paired
@@ -160,14 +252,14 @@ export function createWorkflowSupervisor(bindings) {
160
252
  // observability: a throwing observer is swallowed + logged so a
161
253
  // benchmark hook bug cannot wedge dispatch, and no clock or directory
162
254
  // is sampled when the observer is unwired.
163
- function legMarkStart(runId, leg) {
255
+ function legMarkStart(messageId, leg) {
164
256
  if (bindings.onDispatchTiming === undefined)
165
257
  return 0;
166
258
  const atMs = performance.now();
167
259
  try {
168
260
  bindings.onDispatchTiming({
169
261
  kind: "leg",
170
- runId,
262
+ messageId,
171
263
  leg,
172
264
  phase: "start",
173
265
  atMs,
@@ -175,11 +267,11 @@ export function createWorkflowSupervisor(bindings) {
175
267
  }
176
268
  catch (cause) {
177
269
  const message = cause instanceof Error ? cause.message : String(cause);
178
- logger.warn `onDispatchTiming leg observer threw for ${runId} (${leg} start): ${message}`;
270
+ logger.warn `onDispatchTiming leg observer threw for ${messageId} (${leg} start): ${message}`;
179
271
  }
180
272
  return atMs;
181
273
  }
182
- function legMarkEnd(runId, leg) {
274
+ function legMarkEnd(messageId, leg) {
183
275
  const observer = bindings.onDispatchTiming;
184
276
  if (observer === undefined)
185
277
  return;
@@ -193,12 +285,12 @@ export function createWorkflowSupervisor(bindings) {
193
285
  // surface it on the log and emit the end mark without counters so
194
286
  // the timing slope is still recoverable.
195
287
  const message = cause instanceof Error ? cause.message : String(cause);
196
- logger.warn `structural-counter sample failed for ${runId} (${leg}): ${message}`;
288
+ logger.warn `structural-counter sample failed for ${messageId} (${leg}): ${message}`;
197
289
  }
198
290
  try {
199
291
  observer({
200
292
  kind: "leg",
201
- runId,
293
+ messageId,
202
294
  leg,
203
295
  phase: "end",
204
296
  atMs,
@@ -207,7 +299,7 @@ export function createWorkflowSupervisor(bindings) {
207
299
  }
208
300
  catch (cause) {
209
301
  const message = cause instanceof Error ? cause.message : String(cause);
210
- logger.warn `onDispatchTiming leg observer threw for ${runId} (${leg} end): ${message}`;
302
+ logger.warn `onDispatchTiming leg observer threw for ${messageId} (${leg} end): ${message}`;
211
303
  }
212
304
  }
213
305
  // §10c forced-repack A/B (measurement-only). Absent toggle => never
@@ -244,27 +336,26 @@ export function createWorkflowSupervisor(bindings) {
244
336
  }
245
337
  /**
246
338
  * Classify a child-proxied `substrate.write.request` into the D2 leg it
247
- * represents, plus the runId the per-message OLS fit groups on.
248
- * `runs/<runId>/events/` is the run-event bracket commit (runId from the
249
- * prefix); `agent-state/...` is the D1 conversation WAL append (no runId
250
- * in the prefix -- attributed to the dispatch loop's current serial
251
- * runId). Any other prefix is an unmarked proxied write. Returns `null`
252
- * when no observer is wired (so the supervisor samples nothing) or the
253
- * prefix is not an attributed leg.
339
+ * represents, plus the messageId the per-message OLS fit groups on.
340
+ * `runs/<runId>/events/` is the run-event bracket commit and `agent-state/...`
341
+ * is the D1 conversation WAL append; neither prefix carries the message
342
+ * identity (the run-event prefix names only the stable per-deployment run
343
+ * id), so both are attributed to the dispatch loop's current serial
344
+ * message. Any other prefix is an unmarked proxied write. Returns `null`
345
+ * when no observer is wired (so the supervisor samples nothing), when the
346
+ * prefix is not an attributed leg, or when no message is in flight to
347
+ * attribute it to.
254
348
  */
255
349
  function classifyProxiedWriteLeg(preservePrefix) {
256
350
  if (bindings.onDispatchTiming === undefined)
257
351
  return null;
258
- const runEventMatch = /^runs\/([^/]+)\/events\/$/.exec(preservePrefix);
259
- if (runEventMatch !== null) {
260
- const runId = runEventMatch[1];
261
- if (runId !== undefined)
262
- return { leg: "runevent", runId };
352
+ if (currentDispatchMessageId === null)
353
+ return null;
354
+ if (/^runs\/[^/]+\/events\/$/.test(preservePrefix)) {
355
+ return { leg: "runevent", messageId: currentDispatchMessageId };
263
356
  }
264
357
  if (preservePrefix.startsWith("agent-state/")) {
265
- if (currentDispatchRunId !== null) {
266
- return { leg: "wal", runId: currentDispatchRunId };
267
- }
358
+ return { leg: "wal", messageId: currentDispatchMessageId };
268
359
  }
269
360
  return null;
270
361
  }
@@ -286,7 +377,7 @@ export function createWorkflowSupervisor(bindings) {
286
377
  const deriveMailAuditRef = bindings.deriveMailAuditRef ?? defaultInProcessMailAuditRef;
287
378
  const defaultInboxWritePrincipal = {
288
379
  kind: "supervisor",
289
- deploymentId: bindings.deploymentId,
380
+ anchorRunId: bindings.anchorRunId,
290
381
  };
291
382
  const inboxWritePrincipal = bindings.inboxWritePrincipal ?? defaultInboxWritePrincipal;
292
383
  // Resolve the consumed-dedup retention horizon once at the bindings
@@ -302,6 +393,17 @@ export function createWorkflowSupervisor(bindings) {
302
393
  const readyTimeoutMs = bindings.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS;
303
394
  const readySetTimer = bindings.setTimer ?? defaultSetTimer;
304
395
  const readyClearTimer = bindings.clearTimer ?? defaultClearTimer;
396
+ // Resolve the crash-loop guard bounds once at the bindings edge (the
397
+ // layer that owns the operator config). The stable-run reset timer
398
+ // reuses the same injectable `setTimer`/`clearTimer` pair as the ready
399
+ // handshake, and the wall clock reuses the recycle-policy `now` seam, so
400
+ // tests drive the whole guard deterministically through existing knobs.
401
+ const crashLoopMaxCount = bindings.crashLoopMaxCount ?? DEFAULT_CRASH_LOOP_MAX_COUNT;
402
+ const crashLoopWindowMs = bindings.crashLoopWindowMs ?? DEFAULT_CRASH_LOOP_WINDOW_MS;
403
+ const crashLoopStableResetMs = bindings.crashLoopStableResetMs ?? DEFAULT_CRASH_LOOP_STABLE_RESET_MS;
404
+ const respawnBackoffInitialMs = bindings.respawnBackoffInitialMs ?? DEFAULT_RESPAWN_BACKOFF_INITIAL_MS;
405
+ const respawnBackoffMaxMs = bindings.respawnBackoffMaxMs ?? DEFAULT_RESPAWN_BACKOFF_MAX_MS;
406
+ const crashNow = bindings.recyclePolicyNow ?? defaultNow;
305
407
  /**
306
408
  * Resolved on every successful `enqueueInbox`; the dispatch loop
307
409
  * awaits this promise after a null dequeue so it returns to
@@ -332,12 +434,482 @@ export function createWorkflowSupervisor(bindings) {
332
434
  */
333
435
  let spawnContext = null;
334
436
  let recyclePolicy = null;
335
- let recycleInProgress = false;
437
+ // Mutual-exclusion latch shared by every respawn path (operator/policy/
438
+ // self recycle and crash-respawn). `runRespawn` owns the set/clear; each
439
+ // caller owns the contention read because the two paths diverge on
440
+ // contention (recycle throws, crash-respawn declines silently).
441
+ let respawnInProgress = false;
442
+ // Monotonic child-cohort generation, bumped atomically with each
443
+ // transition to `running` (initial spawn and every respawn's
444
+ // `installNewChild`). The exit-watcher captures the generation it was
445
+ // armed for; a watcher whose generation is no longer current is stale
446
+ // (a newer child already superseded it) and is ignored. Generation plus
447
+ // the phase guard plus `respawnInProgress` together classify every child
448
+ // exit as planned or unexpected without a separate per-handle marker.
449
+ let childGeneration = 0;
450
+ // An unexpected child exit that could not be handled the instant it was
451
+ // observed (a respawn was already in flight). Recorded generation-tagged
452
+ // and drained by `maybeHandleChildExit` once `runRespawn` clears the
453
+ // latch; a pending exit for a superseded generation is dropped as stale.
454
+ let pendingChildExit = null;
455
+ // Monotonic timestamps (ms, from `crashNow`) of recent unexpected exits,
456
+ // pruned to `crashLoopWindowMs`. The crash-loop guard latches when the
457
+ // count within the window reaches `crashLoopMaxCount`.
458
+ const crashTimestamps = [];
459
+ // Handle for the armed stable-run reset timer (or null). Cleared and
460
+ // re-armed on every transition to `running`; cleared on teardown.
461
+ let stableRunResetTimer = null;
462
+ // Current exponential respawn backoff (ms). Advances by doubling on each
463
+ // respawn (capped at `respawnBackoffMaxMs`) and resets to the initial
464
+ // value when a respawned child runs stably. See `waitRespawnBackoff`.
465
+ let respawnBackoffMs = respawnBackoffInitialMs;
466
+ // Every armed backoff wait: the injected timer handle plus the promise
467
+ // resolver, so a shutdown can cancel all of them and unblock the parked
468
+ // respawn coroutines (each then bails at its phase/generation re-check).
469
+ // A SET, not a single slot: more than one coroutine can be parked at once
470
+ // -- a recycle can install a fresh, live child DURING one crash's backoff
471
+ // wait, and that child crashing parks a second coroutine. A single slot
472
+ // would drop the earlier one's timer and leak it past shutdown.
473
+ const respawnBackoffWaits = new Set();
474
+ // =====================================================================
475
+ // CRASH-RESPAWN -- auto-recovery from an unexpected workflow-process exit
476
+ // =====================================================================
477
+ //
478
+ // An UNEXPECTED child exit (crash, OOM, panic, signal -- not a
479
+ // supervisor-initiated shutdown, drain, or recycle) is detected by
480
+ // watching `handle.exited`, the only universal death signal: a clean
481
+ // process death ends the IPC channel readers without a protocol-level
482
+ // crash callback (see `onChildCrash`). `armChildForRunning` arms that
483
+ // watcher and bumps `childGeneration` on every transition to `running`;
484
+ // an exit is classified as unexpected iff its generation is still current
485
+ // AND the phase is still `running` (a planned kill is observed in a
486
+ // non-running phase, or against a superseded generation).
487
+ //
488
+ // On an unexpected exit `handleUnexpectedChildExit` runs: it records the
489
+ // crash against the crash-loop guard and either
490
+ // - RESPAWNS -- after an exponential backoff (`waitRespawnBackoff`,
491
+ // 1s doubling to a 30s cap), it reuses the recycle path's
492
+ // `runRespawn` with a no-op drain (the child is already dead) so the
493
+ // stranded-mail replay and fresh-child spawn happen exactly as a
494
+ // recycle's do; or
495
+ // - LATCHES -- once the guard trips (`crashLoopMaxCount` exits within
496
+ // `crashLoopWindowMs`), it tears down to the terminal `crash-looping`
497
+ // phase and commits a `RunFailed` for the deployment's run so the
498
+ // crash-loop is durably observable as a failed run status.
499
+ //
500
+ // A respawned child that survives `crashLoopStableResetMs`
501
+ // (`armStableRunResetTimer`) resets both the crash counter and the
502
+ // backoff. The backoff wait sits OUTSIDE the `respawnInProgress` latch,
503
+ // so the generation re-check after the wait is what stops a recycle that
504
+ // installed a fresh cohort mid-wait from being respawned a second time.
505
+ // Full policy: `packages/workflow-host/README.md` "Respawn policy".
506
+ // A protocol violation on a live cohort's control or event channel. The
507
+ // channel receiver ends its iterator and invokes this; a clean process
508
+ // death does NOT (it just ends the reader with no crash callback), so
509
+ // the exit-watcher on `handle.exited` is the universal death signal and
510
+ // this path only covers the frame-level violations the receiver detects.
511
+ // On the live (`running`) cohort, force the child down so its `exited`
512
+ // fires the exit-watcher and the crash flows through the SAME
513
+ // respawn/crash-loop path a clean death takes -- a violation that emits a
514
+ // garbage frame just before dying must not divert to `stopped` while a
515
+ // clean SIGKILL respawns. In any other phase the owning lifecycle path
516
+ // (spawn handshake, recycle reap, shutdown) owns teardown.
336
517
  function onChildCrash(reason) {
337
- logger.error `workflow-process control channel crash: {reason}`;
338
- void shutdownInternal({ reason });
518
+ if (state.phase === "running") {
519
+ logger.error `workflow-process channel crash on live cohort; forcing child down to respawn: ${reason}`;
520
+ state.handle.kill();
521
+ return;
522
+ }
523
+ logger.error `workflow-process channel crash: ${reason}`;
524
+ // Only a live, registered supervisor driven down by a channel crash is a
525
+ // self-termination the host must reclaim, and that is `recycling`:
526
+ // `running` took the kill branch above (its exit reaches the crash-loop
527
+ // latch, which carries its own flag), `starting` is the pre-registration
528
+ // initial spawn handshake whose failure the deploy unwind owns, and
529
+ // `stopping` is a teardown already in flight (a host `shutdown()`, or a
530
+ // self-termination already firing). An allowlist, not a denylist, so a
531
+ // future phase defaults to no self-terminate rather than a spurious one.
532
+ const selfTerminated = state.phase === "recycling";
533
+ void shutdownInternal({ reason, selfTerminated });
534
+ }
535
+ // Prune crash timestamps older than the sliding window relative to `nowMs`.
536
+ function pruneCrashTimestamps(nowMs) {
537
+ const cutoff = nowMs - crashLoopWindowMs;
538
+ while (true) {
539
+ const oldest = crashTimestamps[0];
540
+ if (oldest === undefined || oldest > cutoff)
541
+ break;
542
+ crashTimestamps.shift();
543
+ }
544
+ }
545
+ function clearStableRunResetTimer() {
546
+ if (stableRunResetTimer !== null) {
547
+ readyClearTimer(stableRunResetTimer);
548
+ stableRunResetTimer = null;
549
+ }
550
+ }
551
+ // Arm (or re-arm) the stable-run reset for the cohort that just reached
552
+ // `running`. If the child is still that same live cohort when the timer
553
+ // fires, the run has been stable for `crashLoopStableResetMs` and the
554
+ // crash counter is cleared so a flap-then-stabilize sequence does not
555
+ // latch. A crash before the timer fires re-arms it for the next cohort;
556
+ // teardown clears it.
557
+ function armStableRunResetTimer(generation) {
558
+ clearStableRunResetTimer();
559
+ stableRunResetTimer = readySetTimer(() => {
560
+ stableRunResetTimer = null;
561
+ if (generation === childGeneration && state.phase === "running") {
562
+ // The cohort ran stably: reset both the crash counter and the
563
+ // exponential backoff so a flap followed by stability starts over.
564
+ crashTimestamps.length = 0;
565
+ respawnBackoffMs = respawnBackoffInitialMs;
566
+ }
567
+ }, crashLoopStableResetMs);
568
+ }
569
+ // Wait the current respawn backoff before a respawn. The wait uses the
570
+ // injected timer so tests drive it deterministically, and is cancellable:
571
+ // `cancelRespawnBackoffWaits` (called on shutdown) clears the timer and
572
+ // resolves the promise early so the parked respawn coroutine unblocks and
573
+ // bails at its phase/generation re-check rather than sleeping out a full
574
+ // 30s backoff against a torn-down supervisor.
575
+ function waitRespawnBackoff(ms) {
576
+ return new Promise((resolve) => {
577
+ const entry = {
578
+ timer: null,
579
+ resolve,
580
+ };
581
+ entry.timer = readySetTimer(() => {
582
+ respawnBackoffWaits.delete(entry);
583
+ resolve();
584
+ }, ms);
585
+ respawnBackoffWaits.add(entry);
586
+ });
587
+ }
588
+ // Cancel every armed backoff wait. Idempotent: an empty set is a no-op,
589
+ // and each entry's own fire path has already removed it.
590
+ function cancelRespawnBackoffWaits() {
591
+ for (const entry of respawnBackoffWaits) {
592
+ readyClearTimer(entry.timer);
593
+ entry.resolve();
594
+ }
595
+ respawnBackoffWaits.clear();
596
+ }
597
+ // Bump the generation and arm the `handle.exited` watcher for a child
598
+ // that just became the `running` cohort. Called atomically with the
599
+ // state swap to `running` (no await between the swap and this call).
600
+ // `exited` resolving OR rejecting both mean the process is gone. The
601
+ // stable-run reset timer is NOT armed here: on the pristine first spawn
602
+ // there is no crash counter to reset, so it is armed only after a
603
+ // respawn (see `handleUnexpectedChildExit`).
604
+ function armChildForRunning(handle) {
605
+ childGeneration += 1;
606
+ const generation = childGeneration;
607
+ void handle.exited
608
+ .then(() => {
609
+ onChildExited(generation, "workflow-process child exited");
610
+ })
611
+ .catch(() => {
612
+ onChildExited(generation, "workflow-process child exited (exit promise rejected)");
613
+ });
614
+ }
615
+ // Record a child exit and try to act on it. Stale exits (a newer cohort
616
+ // already installed) are dropped.
617
+ function onChildExited(generation, reason) {
618
+ if (generation !== childGeneration)
619
+ return;
620
+ pendingChildExit = { generation, reason };
621
+ maybeHandleChildExit();
339
622
  }
340
- function onMailMessage(rawMessage) {
623
+ // Drain a recorded child exit if the supervisor is in a state to act on
624
+ // it. Declines (leaving the exit pending) while a respawn is in flight;
625
+ // `runRespawn`'s `finally` re-invokes this after clearing the latch so
626
+ // an exit observed during the respawn is not lost. A pending exit for a
627
+ // superseded generation, or one observed after the deployment left the
628
+ // `running` phase (shutdown/recycle owns that teardown), is dropped.
629
+ function maybeHandleChildExit() {
630
+ if (respawnInProgress)
631
+ return;
632
+ const pending = pendingChildExit;
633
+ if (pending === null)
634
+ return;
635
+ if (pending.generation !== childGeneration) {
636
+ pendingChildExit = null;
637
+ return;
638
+ }
639
+ if (state.phase !== "running")
640
+ return;
641
+ pendingChildExit = null;
642
+ void handleUnexpectedChildExit(pending.reason).catch((cause) => {
643
+ // Fire-and-forget context (the exit-watcher has no caller to catch
644
+ // this). `runRespawn` already ran its own failure teardown to a
645
+ // terminal state before rethrowing, so the deployment is not wedged;
646
+ // surface the failure and stop.
647
+ const message = cause instanceof Error ? cause.message : String(cause);
648
+ logger.error `crash-respawn handling failed; deployment torn down: ${message}`;
649
+ });
650
+ }
651
+ // Handle one unexpected child exit: record it against the crash-loop
652
+ // guard and either latch the deployment (guard tripped) or respawn.
653
+ async function handleUnexpectedChildExit(reason) {
654
+ if (state.phase !== "running" || spawnContext === null) {
655
+ // Raced a shutdown/recycle between the drain check and here; the
656
+ // owning lifecycle path handles teardown.
657
+ return;
658
+ }
659
+ // A cohort that just crashed did not earn a stability reward: disarm its
660
+ // stable-run reset timer up front so it cannot fire during the backoff
661
+ // wait below (where the phase is still `running` and the generation is
662
+ // still this dead cohort's -- exactly the state the timer's own guard
663
+ // reads as "alive and stable") and wrongly clear the crash counter.
664
+ clearStableRunResetTimer();
665
+ // Capture the crashing cohort's generation. If a recycle or policy
666
+ // respawn installs a fresh cohort while the backoff wait below is
667
+ // parked, the generation advances and this handler must NOT respawn
668
+ // that healthy cohort -- the post-wait guard bails on the mismatch.
669
+ const armedGeneration = childGeneration;
670
+ const nowMs = crashNow();
671
+ crashTimestamps.push(nowMs);
672
+ pruneCrashTimestamps(nowMs);
673
+ if (crashTimestamps.length >= crashLoopMaxCount) {
674
+ // Crash-loop latch. The deployment stops respawning and tears down
675
+ // to the terminal `crash-looping` state so a rapidly-flapping child
676
+ // cannot saturate the host.
677
+ const crashCount = crashTimestamps.length;
678
+ logger.error `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms; stopping the deployment (${reason})`;
679
+ await shutdownInternal({
680
+ reason: `crash-loop: ${reason}`,
681
+ terminalPhase: "crash-looping",
682
+ selfTerminated: true,
683
+ });
684
+ // Commit the RunFailed tombstone AFTER teardown: shutdownInternal has
685
+ // quiesced the drain accumulators (stop + await disposed), so the
686
+ // run-event tree is settled and no escalation commit races this write.
687
+ // This RunFailed is the SOLE durable, externally-queryable signal of
688
+ // the crash-loop (the `crash-looping` phase is in-memory only), so a
689
+ // failure to write it is logged loudly rather than swallowed. Best-
690
+ // effort: the deployment is already terminal, so the write not landing
691
+ // costs observability, not correctness.
692
+ try {
693
+ // `anchorRunId` and the tombstone's `runId` are DISTINCT ids and must
694
+ // not be conflated. `bindings.anchorRunId` is the workflow-run repo
695
+ // slug (`deriveWorkflowRunRepoId`), which the supervisor principal's
696
+ // authz check keys on (`repoId.id === anchorRunId`). The RunFailed must
697
+ // land on the deployment's ONE top-level run, whose id is the local
698
+ // part of the deployment's mail address (`deriveWorkflowRunId`) -- the
699
+ // same id the dispatch loop writes every run event under. For a domain
700
+ // like `integration.interchange` the two ids differ (the repo slug
701
+ // carries a domain suffix), so writing the tombstone under the repo
702
+ // slug would strand it in a run subtree no reader consults.
703
+ await commitRunFailed({
704
+ substrate: bindings.repoStore,
705
+ repoId: bindings.workflowRunRepoId,
706
+ ref: bindings.workflowRunRef,
707
+ anchorRunId: bindings.anchorRunId,
708
+ runId: deriveWorkflowRunId(bindings.deploymentMailAddress),
709
+ at: new Date(nowMs).toISOString(),
710
+ message: `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms`,
711
+ });
712
+ }
713
+ catch (cause) {
714
+ const message = cause instanceof Error ? cause.message : String(cause);
715
+ logger.error `crash-loop RunFailed commit failed; deployment has no durable failure tombstone: ${message}`;
716
+ }
717
+ return;
718
+ }
719
+ const thisBackoffMs = respawnBackoffMs;
720
+ logger.warn `workflow-process exited unexpectedly; respawning after ${String(thisBackoffMs)}ms backoff (${reason})`;
721
+ await waitRespawnBackoff(thisBackoffMs);
722
+ // A recycle/policy respawn, or a shutdown, may have run during the
723
+ // backoff wait. Bail unless THIS dead cohort is still the current
724
+ // running one: the generation guard prevents respawning a healthy
725
+ // cohort a recycle just installed, and the phase guard prevents acting
726
+ // after teardown. No await separates this re-check from `runRespawn`'s
727
+ // synchronous `respawnInProgress` set, so nothing can interleave.
728
+ if (childGeneration !== armedGeneration ||
729
+ state.phase !== "running" ||
730
+ spawnContext === null) {
731
+ // The backoff above announced a respawn before parking. Report the bail
732
+ // too, so a deployment that stops respawning does not leave an operator
733
+ // with a promised respawn and no record of what happened to it.
734
+ logger.info `respawn backoff elapsed but the crashed cohort is no longer the running one; skipping respawn (phase=${state.phase}, armed generation ${String(armedGeneration)}, current generation ${String(childGeneration)}): ${reason}`;
735
+ return;
736
+ }
737
+ const priorRunning = state;
738
+ const priorContext = spawnContext;
739
+ // Advance the backoff only now that a respawn is actually happening, so
740
+ // a bail above does not inflate the wait for a respawn that never ran.
741
+ respawnBackoffMs = Math.min(respawnBackoffMs * 2, respawnBackoffMaxMs);
742
+ await runRespawn({
743
+ origin: "crash",
744
+ reason,
745
+ prior: priorRunning,
746
+ priorContext,
747
+ // The child is already dead: there is nothing to drain, and
748
+ // `runRespawn`'s kill step is a no-op on a dead handle. The replay
749
+ // step still runs, moving any mail stranded mid-flight back to the
750
+ // inbox tail before dispatch resumes.
751
+ drain: async () => undefined,
752
+ });
753
+ // The respawned child is now the running cohort. Arm the stable-run
754
+ // reset against its generation: if it survives `crashLoopStableResetMs`
755
+ // the crash counter and backoff reset, so a flap followed by stability
756
+ // does not latch on a later, unrelated crash.
757
+ armStableRunResetTimer(childGeneration);
758
+ }
759
+ // Eager per-run mailbox (§3b inbound). On arrival the supervisor commits each
760
+ // fresh inbound message into the deployment's substrate-backed INBOX and fires
761
+ // a one-way `mailbox.notify` to the child, so the warm agent's `watch` /
762
+ // `mail_wait` observes the arrival mid-turn -- decoupled from the FIFO claim-
763
+ // check dispatch that resolves a run's step input. The supervisor is the sole
764
+ // mailbox writer; the store is its long-lived in-memory mirror over the
765
+ // committed `mailbox/INBOX/` subtree, constructed lazily on the first arrival.
766
+ const mailboxWritePrincipal = {
767
+ kind: "supervisor",
768
+ anchorRunId: bindings.anchorRunId,
769
+ };
770
+ let mailboxStore = null;
771
+ // Claim-check messageId -> assigned mailbox uid, so a message dispatched as a
772
+ // turn can be flagged \Seen/$Processed by uid. In-memory only: a missing entry
773
+ // (a restart, an arrival whose eager commit failed, or an already-processed
774
+ // message whose entry was pruned) skips the flag mark, which is a cosmetic
775
+ // IMAP flag, never a delivery guarantee. `markMailboxProcessed` prunes an
776
+ // entry once its mark completes, so the map holds only messages awaiting the
777
+ // flag mark rather than growing for the deployment's life.
778
+ const mailboxUidByMessageId = new Map();
779
+ // Serializes every mailbox mutation (lazy construction, the arrival
780
+ // append+flush, the dispatch flag mark) so concurrent arrivals and a fire-and-
781
+ // forget flag mark never interleave against the shared in-memory mirror.
782
+ let mailboxTail = Promise.resolve();
783
+ function runMailboxExclusive(fn) {
784
+ const run = mailboxTail.then(fn, fn);
785
+ mailboxTail = run.then(() => undefined, () => undefined);
786
+ return run;
787
+ }
788
+ async function getMailboxStore() {
789
+ if (mailboxStore === null) {
790
+ mailboxStore = await createSubstrateMailboxStore({
791
+ substrate: bindings.repoStore,
792
+ repoId: bindings.workflowRunRepoId,
793
+ principal: mailboxWritePrincipal,
794
+ ref: bindings.workflowRunRef,
795
+ });
796
+ }
797
+ return mailboxStore;
798
+ }
799
+ function storedEnvelopeFromHeaders(headers, receivedAt) {
800
+ // The Date header is unvalidated external input; fall back to the arrival
801
+ // time when it is absent or unparseable so the store's `toISOString`
802
+ // serialization cannot throw on an Invalid Date.
803
+ const parsed = new Date(headers.date);
804
+ const date = Number.isNaN(parsed.getTime()) ? new Date(receivedAt) : parsed;
805
+ return {
806
+ messageId: headers.messageId,
807
+ from: headers.from,
808
+ to: headers.to,
809
+ subject: headers.subject ?? "",
810
+ date,
811
+ inReplyTo: headers.inReplyTo,
812
+ references: headers.references ?? [],
813
+ interchangeType: headers.interchangeType,
814
+ interchangeCorrelationId: headers.interchangeCorrelationId,
815
+ };
816
+ }
817
+ /**
818
+ * Eager-commit one freshly-arrived inbound message into the deployment's
819
+ * substrate mailbox, then notify the child. Runs on the mail-arrival path,
820
+ * before and independent of FIFO dispatch, so the warm agent's `mail_wait`
821
+ * observes the message mid-turn. Best-effort: the claim-check inbox is the
822
+ * durable delivery contract, so a decode or substrate fault here is logged
823
+ * loudly and never withholds the mail's ack -- the message still reaches the
824
+ * agent as its turn's step input via `trigger.fire`. The `mailbox.notify` is
825
+ * sent only AFTER the append is flushed, so the child reads committed state.
826
+ */
827
+ async function commitInboundToMailbox(messageId, rawMessage, receivedAt) {
828
+ try {
829
+ await runMailboxExclusive(async () => {
830
+ // The caller gates on a fresh `enqueued` outcome, so a redelivery never
831
+ // reaches here; this guard is belt-and-suspenders against a double
832
+ // append of the same messageId.
833
+ if (mailboxUidByMessageId.has(messageId))
834
+ return;
835
+ let decoded;
836
+ try {
837
+ decoded = decodeMail(rawMessage);
838
+ }
839
+ catch (cause) {
840
+ const message = cause instanceof Error ? cause.message : String(cause);
841
+ logger.error `eager mailbox commit: dropping undecodable inbound mail ${messageId}: ${message}`;
842
+ return;
843
+ }
844
+ const store = await getMailboxStore();
845
+ const uid = store.append(rawMessage, storedEnvelopeFromHeaders(decoded.headers, receivedAt), []);
846
+ mailboxUidByMessageId.set(messageId, uid);
847
+ await store.flush();
848
+ const commit = await bindings.repoStore.resolveRef(mailboxWritePrincipal, bindings.workflowRunRepoId, bindings.workflowRunRef);
849
+ if (commit === null) {
850
+ logger.error `eager mailbox commit: ${bindings.workflowRunRef} did not resolve after flush; skipping mailbox.notify for ${messageId}`;
851
+ return;
852
+ }
853
+ const sender = activeControlSender();
854
+ if (sender === null) {
855
+ logger.info `eager mailbox commit: no active control sender; committed ${messageId} as uid ${String(uid)} without mailbox.notify`;
856
+ return;
857
+ }
858
+ await sender.send({
859
+ type: "mailbox.notify",
860
+ data: {
861
+ runId: deriveWorkflowRunId(bindings.deploymentMailAddress),
862
+ mailbox: MAILBOX_INBOX_DIR,
863
+ uid,
864
+ headers: decoded.headers,
865
+ },
866
+ });
867
+ });
868
+ }
869
+ catch (cause) {
870
+ const message = cause instanceof Error ? cause.message : String(cause);
871
+ logger.error `eager mailbox commit failed for ${messageId}; mail still delivered via claim-check dispatch: ${message}`;
872
+ }
873
+ }
874
+ /**
875
+ * Flag a dispatched message's mailbox entry \Seen/$Processed. Fire-and-forget
876
+ * off the dispatch critical path: the flag is a cosmetic IMAP marker, so a
877
+ * missing uid (no eager mailbox entry) or a substrate fault is logged and
878
+ * dropped, never failing the turn.
879
+ */
880
+ function markMailboxProcessed(messageId) {
881
+ const uid = mailboxUidByMessageId.get(messageId);
882
+ if (uid === undefined)
883
+ return;
884
+ void runMailboxExclusive(async () => {
885
+ try {
886
+ const store = await getMailboxStore();
887
+ if (store.find(uid) === undefined)
888
+ return;
889
+ store.addFlags(uid, [MAILBOX_FLAG_SEEN, MAILBOX_FLAG_PROCESSED]);
890
+ await store.flush();
891
+ }
892
+ finally {
893
+ // The id->uid mapping exists only to flag this message once. After the
894
+ // mark runs (or the message is already gone), the entry is dead weight,
895
+ // so drop it to bound the map over a long-lived conversational mailbox.
896
+ // Redelivery dedup is owned by the durable inbox index, not this map.
897
+ // The delete runs inside the exclusive section so it never interleaves
898
+ // with the arrival path's `has(messageId)` check.
899
+ mailboxUidByMessageId.delete(messageId);
900
+ }
901
+ }).catch((cause) => {
902
+ const message = cause instanceof Error ? cause.message : String(cause);
903
+ logger.warn `mailbox flag mark failed for ${messageId} (uid ${String(uid)}): ${message}`;
904
+ });
905
+ }
906
+ // Resolves once the inbound mail is durably accepted (its inbox write landed
907
+ // or the message was already durably present); rejects when it was not (a
908
+ // phase where the deployment is not accepting mail, a transient enqueue
909
+ // failure, or a stale refusal). The host propagates that settlement to the
910
+ // wire, so resolution is the durable-receipt ACK signal and rejection is the
911
+ // WITHHOLD signal -- a withheld message is redelivered by the hub.
912
+ async function onMailMessage(rawMessage) {
341
913
  // Every inbound mail flows through the FIFO inbox claim-check
342
914
  // queue, regardless of the supervisor's current phase. The
343
915
  // dispatch loop (started by `spawn()` and restarted by the
@@ -349,16 +921,33 @@ export function createWorkflowSupervisor(bindings) {
349
921
  // the envelope's `receivedAt` prefix on the inbox filename.
350
922
  if (state.phase === "idle" ||
351
923
  state.phase === "stopping" ||
352
- state.phase === "stopped") {
353
- // The host's higher-level lifecycle is already tearing the
354
- // deployment down; the message drops on the floor rather than
355
- // landing in an inbox no live dispatch loop will service.
356
- return;
924
+ state.phase === "stopped" ||
925
+ state.phase === "crash-looping") {
926
+ // The host's higher-level lifecycle is already tearing the deployment
927
+ // down; nothing is enqueued. Reject rather than silently drop so the
928
+ // ack is WITHHELD and the hub redelivers -- a later generation (or a
929
+ // recycle-installed dispatch loop) may accept it, and a permanently
930
+ // torn-down address exhausts the hub's bounded retry budget instead of
931
+ // losing a message a transiently-idle deployment would have taken.
932
+ throw new Error(`inbound mail not accepted: supervisor phase is "${state.phase}"`);
933
+ }
934
+ try {
935
+ await enqueueInboundMail(rawMessage);
936
+ }
937
+ catch (cause) {
938
+ // Both branches WITHHOLD (rethrow); the split only sets log severity so
939
+ // a stale refusal surfaces as its own loud signal rather than blending
940
+ // into ordinary enqueue-failure noise. The ack/withhold decision is the
941
+ // rethrow itself, never this classification.
942
+ if (cause instanceof StaleInboxEnqueueError) {
943
+ logger.error `inbound mail refused as stale, withholding ack (hub will redeliver): ${cause.message}`;
944
+ }
945
+ else {
946
+ const message = cause instanceof Error ? cause.message : String(cause);
947
+ logger.error `enqueueInbox failed, withholding ack (hub will redeliver): ${message}`;
948
+ }
949
+ throw cause;
357
950
  }
358
- void enqueueInboundMail(rawMessage).catch((cause) => {
359
- const message = cause instanceof Error ? cause.message : String(cause);
360
- logger.error `enqueueInbox failed: ${message}`;
361
- });
362
951
  }
363
952
  async function enqueueInboundMail(rawMessage) {
364
953
  const messageId = await deriveMessageId(rawMessage);
@@ -373,11 +962,11 @@ export function createWorkflowSupervisor(bindings) {
373
962
  const rawMessageBase64 = base64Encode(rawMessage);
374
963
  // D2 leg: `enqueueInbox` runs in `onMailMessage` BEFORE dispatch, so
375
964
  // it is paid OUTSIDE the dispatch-start..reply-produced window -- its
376
- // growth is invisible to the 4.7 bracket. The leg mark, keyed by the
377
- // same messageId the dispatch loop later uses as the runId, makes the
378
- // out-of-window cost visible and joinable to the in-window legs.
965
+ // growth is invisible to the 4.7 bracket. This leg mark is keyed by the
966
+ // messageId, the same per-message key every in-window leg uses, so the
967
+ // D2 per-message OLS fit groups the enqueue leg with the rest.
379
968
  legMarkStart(messageId, "enqueue");
380
- await inboxPrimitives.enqueueInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
969
+ const outcome = await inboxPrimitives.enqueueInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
381
970
  address: bindings.deploymentMailAddress,
382
971
  messageId,
383
972
  receivedAt,
@@ -385,7 +974,25 @@ export function createWorkflowSupervisor(bindings) {
385
974
  rawMessage: rawMessageBase64,
386
975
  });
387
976
  legMarkEnd(messageId, "enqueue");
388
- wakeDispatch();
977
+ // Only a fresh enqueue added a new inbox entry; wake the dispatch loop for
978
+ // it alone. An `already-present` outcome landed nothing new -- returning
979
+ // (which acks) without waking is correct, since the earlier delivery of
980
+ // the same messageId already drives dispatch. This resolves for both
981
+ // outcomes: both mean the bytes are durably accounted for, so both ack.
982
+ if (outcome.outcome === "enqueued") {
983
+ // Eager-commit the fresh message into the per-run mailbox and notify the
984
+ // child BEFORE waking dispatch, so the warm agent's mail_wait can observe
985
+ // it committed. Non-fatal by contract: the enqueue above already secured
986
+ // the durable delivery, so this never withholds the ack.
987
+ await commitInboundToMailbox(messageId, rawMessage, receivedAt);
988
+ wakeDispatch();
989
+ }
990
+ else {
991
+ // A redelivery of a message already durably present: the ack still
992
+ // fires (it is on disk), but no new run is dispatched. Surface it so an
993
+ // at-least-once redelivery being made effectively-once is observable.
994
+ logger.info `inbound mail ${messageId} already durably present (${outcome.reason}); acknowledging without re-dispatch`;
995
+ }
389
996
  }
390
997
  /**
391
998
  * Pump child-initiated upstream control frames after `ready` has
@@ -448,6 +1055,18 @@ export function createWorkflowSupervisor(bindings) {
448
1055
  });
449
1056
  continue;
450
1057
  }
1058
+ if (payload.type === "mailbox.mutate.request") {
1059
+ // INBOUND half of mailbox ownership (§3b). The child asked the
1060
+ // supervisor -- the sole mailbox writer -- to apply a flag write or
1061
+ // expunge. Run it off the iterator's loop so the iterator keeps
1062
+ // draining while the store flushes; the handler owns the
1063
+ // `mailbox.mutate.response` reply that resolves the child's awaiter.
1064
+ void handleMailboxMutation(payload.data).catch((cause) => {
1065
+ const message = cause instanceof Error ? cause.message : String(cause);
1066
+ logger.error `mailbox.mutate.request handler crashed: ${message}`;
1067
+ });
1068
+ continue;
1069
+ }
451
1070
  if (payload.type === "terminal.event") {
452
1071
  // The workflow-process child mirrors every terminal-run commit
453
1072
  // over the control IPC. Fan it out to the COHORT'S broadcaster
@@ -456,16 +1075,98 @@ export function createWorkflowSupervisor(bindings) {
456
1075
  // to-one with its cohort's `controlIncoming` iterator: a
457
1076
  // buffered `terminal.event` the OLD child emitted before kill
458
1077
  // landed must NEVER route to the NEW cohort's broadcaster.
459
- // Without this binding, a stale OLD-cohort frame for a runId
460
- // the NEW cohort happens to be dispatching under the same id
461
- // (the normal recycle/replay case) would falsely settle the
462
- // NEW cohort's `waitForRunTerminal` and commit `markConsumed`
1078
+ // Every run shares the stable runId (the deployment mail
1079
+ // address), so an OLD-cohort frame and a NEW-cohort run collide
1080
+ // on that id by construction (the recycle/replay case); without
1081
+ // this binding the stale frame would falsely settle the NEW
1082
+ // cohort's `waitForRunTerminalOrPark` and commit `markConsumed`
463
1083
  // on a run still in flight. The broadcaster's own `dispose()`
464
1084
  // on cohort teardown turns post-dispose notify into a no-op,
465
1085
  // so a stale frame dequeued after the cohort was torn down
466
1086
  // drops cleanly without leaking into any successor cohort.
467
1087
  const event = terminalEventFromPayload(payload.data);
468
1088
  cohortBroadcaster.notify(payload.data.runId, event);
1089
+ terminalRunIds.add(payload.data.runId);
1090
+ // Clean up cohort tracking for the terminated run. Self-discovered
1091
+ // runs have no dispatch-loop entry, so their cleanup happens here.
1092
+ cohortRunIds.delete(payload.data.runId);
1093
+ runInputChannels.delete(payload.data.runId);
1094
+ continue;
1095
+ }
1096
+ if (payload.type === "park.notify") {
1097
+ // The workflow-process child reported a control-plane suspension: an
1098
+ // agent step parked on a reserved `signalName(correlationId)` channel.
1099
+ if (payload.data.parkKind === "input") {
1100
+ // Input parks are owned by the supervisor, not the hub. Register
1101
+ // cohort membership BEFORE caching the correlationId so a run is
1102
+ // never a channel-without-cohort entry (the routing-hygiene
1103
+ // invariant); a live run that parked is always in cohortRunIds
1104
+ // already, so this is idempotent belt-and-suspenders. Cache the
1105
+ // correlationId so the dispatch loop can fire signal.deliver on
1106
+ // subsequent mail without a substrate round-trip.
1107
+ cohortRunIds.add(payload.data.runId);
1108
+ runInputChannels.set(payload.data.runId, {
1109
+ correlationId: payload.data.correlationId,
1110
+ parkKind: "input",
1111
+ });
1112
+ // Stop any drain accumulator for a run that has parked. Input parks
1113
+ // own this because a drain arms no accumulator for a run that already
1114
+ // holds an input channel; an approval park never carries one, so its
1115
+ // drain interaction is a separate concern and stays out of this arm.
1116
+ const accumulator = drainAccumulators.get(payload.data.runId);
1117
+ if (accumulator !== undefined) {
1118
+ accumulator.stop();
1119
+ drainAccumulators.delete(payload.data.runId);
1120
+ }
1121
+ }
1122
+ else if (payload.data.parkKind === "approval") {
1123
+ // Approval parks are hub-registered through the shared
1124
+ // `registerSuspension` transform.
1125
+ registerSuspension({
1126
+ runId: payload.data.runId,
1127
+ correlationId: payload.data.correlationId,
1128
+ parkKind: "approval",
1129
+ ...(payload.data.snapshot !== undefined
1130
+ ? { snapshot: payload.data.snapshot }
1131
+ : {}),
1132
+ });
1133
+ }
1134
+ else {
1135
+ // A `signal-relay` park is relayed down into the body child by the
1136
+ // section runtime and is never hub-registered, so it does not ride
1137
+ // `park.notify`. One arriving here is a protocol violation; log and
1138
+ // drop rather than mis-registering it as an approval.
1139
+ logger.error `park.notify for run ${payload.data.runId} carried parkKind=${payload.data.parkKind}, which is not a hub-registered kind; dropping`;
1140
+ }
1141
+ // A park of ANY kind suspends the run, so a dispatch loop waiting on
1142
+ // `waitForRunTerminalOrPark` after firing the trigger (or delivering
1143
+ // the last signal) must be released here regardless of park kind. An
1144
+ // approval park that only registered its suspension would leave that
1145
+ // loop hanging to the terminal-or-park backstop. Bump the park
1146
+ // generation BEFORE resolving the waiter: a dispatch loop that captured
1147
+ // `sinceGen` for this run must see the newer generation both when its
1148
+ // armed waiter fires here and when it re-reads the generation after
1149
+ // arming (the check-after-register).
1150
+ parkGenerations.set(payload.data.runId, (parkGenerations.get(payload.data.runId) ?? 0) + 1);
1151
+ // Wake any dispatch loop waiting for this run to park.
1152
+ resolveParkNotifyWaiter(payload.data.runId);
1153
+ continue;
1154
+ }
1155
+ if (payload.type === "parked-correlations.response") {
1156
+ // The child answered a `reEmitParkedCorrelations` query. Resolve the
1157
+ // awaiting driver; a response with no pending entry (the query already
1158
+ // timed out and dropped it) is logged and dropped, never thrown, so it
1159
+ // cannot tear the pump down.
1160
+ resolveParkedResponse(payload.data);
1161
+ continue;
1162
+ }
1163
+ if (payload.type === "resumed.runs") {
1164
+ // The child self-discovered runs from the substrate after reconnect
1165
+ // or recycle. Seed cohort tracking so drain accumulators and dispatch
1166
+ // routing account for runs the supervisor did not personally fire.
1167
+ for (const runId of payload.data.runIds) {
1168
+ cohortRunIds.add(runId);
1169
+ }
469
1170
  continue;
470
1171
  }
471
1172
  logger.warn `workflow-process upstream control payload ignored: type=${payload.type}`;
@@ -473,22 +1174,26 @@ export function createWorkflowSupervisor(bindings) {
473
1174
  }
474
1175
  const pendingMerges = new Map();
475
1176
  /**
476
- * Reject every pending merge round-trip and every
477
- * `markConsumed` completion waiter. Invoked on cohort transitions
478
- * (shutdown, recycle's `installNewChild`) so closures awaiting these
479
- * promises do not outlive the cohort that armed them. Without this,
480
- * a `handleSubstrateWriteRequest` mid-merge or a dispatch-loop
481
- * caller awaiting `markConsumed` would sit on a resolver that the
482
- * dying control channel will never invoke.
1177
+ * Reject every pending merge round-trip and every park-notify
1178
+ * waiter. Invoked on cohort transitions (shutdown, recycle's
1179
+ * `installNewChild`) so closures awaiting these promises do not
1180
+ * outlive the cohort that armed them. Without this, a
1181
+ * `handleSubstrateWriteRequest` mid-merge or a dispatch loop
1182
+ * waiting for park would sit on a resolver that the dying control
1183
+ * channel will never invoke.
483
1184
  */
484
1185
  function rejectCohortAwaiters(reason) {
485
1186
  for (const [requestId, entry] of pendingMerges) {
486
1187
  pendingMerges.delete(requestId);
487
1188
  entry.resolve({ ok: false, reason: `cohort aborted: ${reason}` });
488
1189
  }
489
- for (const [runId, waiter] of markConsumedCompletionWaiters.entries()) {
490
- markConsumedCompletionWaiters.delete(runId);
491
- waiter.reject(new MergeAbortedError(`markConsumed waiter (${runId}): ${reason}`));
1190
+ for (const [requestId, entry] of pendingParkedQueries) {
1191
+ pendingParkedQueries.delete(requestId);
1192
+ entry.settle(null);
1193
+ }
1194
+ for (const [runId, resolve] of parkNotifyWaiters.entries()) {
1195
+ parkNotifyWaiters.delete(runId);
1196
+ resolve();
492
1197
  }
493
1198
  }
494
1199
  function resolveMergeResponse(data) {
@@ -525,6 +1230,125 @@ export function createWorkflowSupervisor(bindings) {
525
1230
  }
526
1231
  entry.resolve({ ok: false, reason: data.result.reason });
527
1232
  }
1233
+ // Stamp the deployment identity the supervisor owns onto a child-supplied
1234
+ // park and hand it to the host's suspension-register sink. Shared by the
1235
+ // `park.notify` arm (the happy-path emit) and `reEmitParkedCorrelations`
1236
+ // (the re-establishment re-emit). Best-effort: a throwing sink is logged,
1237
+ // not rethrown, so it cannot tear the upstream pump down or abort a re-emit
1238
+ // partway through the parked set. The sink (production: the sidecar) turns
1239
+ // the stamped registration into a `signal.correlation.register` frame the
1240
+ // hub co-writes the run's routing + approval rows from.
1241
+ function registerSuspension(park) {
1242
+ if (bindings.onSuspensionRegister === undefined) {
1243
+ logger.warn `suspension register for runId=${park.runId} but no onSuspensionRegister sink is wired; correlation ${park.correlationId} not registered`;
1244
+ return;
1245
+ }
1246
+ try {
1247
+ bindings.onSuspensionRegister({
1248
+ runId: park.runId,
1249
+ correlationId: park.correlationId,
1250
+ kind: park.parkKind,
1251
+ anchorRunId: bindings.anchorRunId,
1252
+ agentAddress: bindings.deploymentMailAddress,
1253
+ ...(park.snapshot !== undefined
1254
+ ? { approvalSnapshot: park.snapshot }
1255
+ : {}),
1256
+ });
1257
+ }
1258
+ catch (cause) {
1259
+ const message = cause instanceof Error ? cause.message : String(cause);
1260
+ logger.error `onSuspensionRegister sink threw for runId=${park.runId} correlationId=${park.correlationId}: ${message}`;
1261
+ }
1262
+ }
1263
+ const pendingParkedQueries = new Map();
1264
+ let parkedQuerySeq = 0;
1265
+ function resolveParkedResponse(data) {
1266
+ const entry = pendingParkedQueries.get(data.requestId);
1267
+ if (entry === undefined) {
1268
+ logger.warn `parked-correlations.response landed with no pending entry; requestId=${data.requestId} dropped`;
1269
+ return;
1270
+ }
1271
+ pendingParkedQueries.delete(data.requestId);
1272
+ entry.settle(data.parked);
1273
+ }
1274
+ async function reEmitParkedCorrelations() {
1275
+ // No-op (NOT throw) when the child is not addressable. `deliverSignal`
1276
+ // throws on non-running/starting (including recycling) because it points a
1277
+ // write at the dying child's closing pipe and wants the caller to retry;
1278
+ // this driver's contract is the opposite -- the next re-establishment
1279
+ // re-drives it -- so skipping recycling and letting the next spawn's
1280
+ // re-emit cover it is correct here, not a missed guard.
1281
+ if (state.phase !== "running" && state.phase !== "starting") {
1282
+ logger.info `reEmitParkedCorrelations: child not addressable (phase=${state.phase}); skipping`;
1283
+ return;
1284
+ }
1285
+ const controlSender = state.controlSender;
1286
+ const requestId = `pc-${String((parkedQuerySeq += 1))}`;
1287
+ const responded = new Promise((resolve) => {
1288
+ pendingParkedQueries.set(requestId, { settle: resolve });
1289
+ });
1290
+ // Watchdog: a wedged-but-alive child never tears its cohort down, so the
1291
+ // cohort-abort settle would never fire and this await would hang the
1292
+ // re-establishment caller. On expiry, drop the pending entry and return;
1293
+ // the next re-establishment re-drives (the hub co-write is idempotent).
1294
+ let timeoutHandle = null;
1295
+ const watchdog = new Promise((resolve) => {
1296
+ timeoutHandle = setTimeout(() => {
1297
+ timeoutHandle = null;
1298
+ if (pendingParkedQueries.delete(requestId)) {
1299
+ logger.warn `reEmitParkedCorrelations: requestId=${requestId} did not respond within ${String(parkedQueryWatchdogMs)}ms; re-registration retries on the next re-establishment`;
1300
+ }
1301
+ resolve("timeout");
1302
+ }, parkedQueryWatchdogMs);
1303
+ });
1304
+ let outcome;
1305
+ try {
1306
+ await controlSender.send({
1307
+ type: "parked-correlations.request",
1308
+ data: { requestId },
1309
+ });
1310
+ outcome = await Promise.race([responded, watchdog]);
1311
+ }
1312
+ catch (cause) {
1313
+ // The downstream send failed (a closing pipe). Best-effort: drop the
1314
+ // pending entry, log, and return; the next re-establishment re-drives.
1315
+ pendingParkedQueries.delete(requestId);
1316
+ if (timeoutHandle !== null)
1317
+ clearTimeout(timeoutHandle);
1318
+ const message = cause instanceof Error ? cause.message : String(cause);
1319
+ logger.warn `reEmitParkedCorrelations query failed: ${message}; re-registration retries on the next re-establishment`;
1320
+ return;
1321
+ }
1322
+ if (timeoutHandle !== null)
1323
+ clearTimeout(timeoutHandle);
1324
+ // `timeout` (watchdog fired) or `null` (cohort torn down before the child
1325
+ // answered): nothing to re-emit; the next re-establishment re-drives.
1326
+ if (outcome === "timeout" || outcome === null)
1327
+ return;
1328
+ for (const parked of outcome) {
1329
+ if (parked.parkKind === "input") {
1330
+ // Register cohort membership BEFORE the input channel so the run is
1331
+ // never a channel-without-cohort entry: the dispatch loop's routing
1332
+ // hygiene drops exactly such entries, and a live resumed run must not
1333
+ // be mistaken for a dead one and have its channel deleted.
1334
+ cohortRunIds.add(parked.runId);
1335
+ runInputChannels.set(parked.runId, {
1336
+ correlationId: parked.correlationId,
1337
+ parkKind: "input",
1338
+ });
1339
+ }
1340
+ else {
1341
+ registerSuspension({ ...parked, parkKind: "approval" });
1342
+ }
1343
+ }
1344
+ }
1345
+ function resolveParkNotifyWaiter(runId) {
1346
+ const resolve = parkNotifyWaiters.get(runId);
1347
+ if (resolve === undefined)
1348
+ return;
1349
+ parkNotifyWaiters.delete(runId);
1350
+ resolve();
1351
+ }
528
1352
  /**
529
1353
  * OUTBOUND half of mailbox ownership (§3a). The workflow-process child
530
1354
  * never holds the agent's signing key; it forwards the structured
@@ -577,6 +1401,111 @@ export function createWorkflowSupervisor(bindings) {
577
1401
  });
578
1402
  }
579
1403
  }
1404
+ /**
1405
+ * Apply a child-requested mailbox mutation to the owned store (INBOUND
1406
+ * half of mailbox ownership, §3b). The supervisor is the sole writer to
1407
+ * the workflow-run mailbox; the child never flushes it. A flag write
1408
+ * (`addFlags` / `removeFlags`) targets one uid; an `expunge` sweeps every
1409
+ * `\Deleted` message out of the live INBOX. The mutation is applied under
1410
+ * `runMailboxExclusive` and flushed before the reply, so the child's next
1411
+ * committed read observes it -- the flush-before-signal ordering
1412
+ * `commitInboundToMailbox` uses. A failure (unknown uid, wrong mailbox,
1413
+ * substrate fault) surfaces back as a structured `{ ok: false, reason }`
1414
+ * so the agent's mail-tool call fails loudly rather than dropping the
1415
+ * mutation silently.
1416
+ */
1417
+ async function handleMailboxMutation(data) {
1418
+ // Capture the sender once. Re-fetching after the flush could return a
1419
+ // successor cohort's sender and misroute the reply to the wrong child
1420
+ // (see the substrate-write handler's note). A null sender means the
1421
+ // supervisor is mid-recycle or tearing down: there is nothing to reply
1422
+ // on, so drop and warn -- the child's read end is closing alongside, so
1423
+ // its pending awaiter is rejected by the control loop's `cancelAll`.
1424
+ const controlSender = activeControlSender();
1425
+ if (controlSender === null) {
1426
+ logger.warn `mailbox.mutate.request received outside running phase; requestId=${data.requestId} dropped (child awaiter will fail on pipe close)`;
1427
+ return;
1428
+ }
1429
+ // The supervisor owns exactly one mailbox, the substrate INBOX. Reject a
1430
+ // request for any other name rather than silently mutate INBOX under it,
1431
+ // which would be a wrong-target durable write reported as success. The
1432
+ // frame carries an unconstrained mailbox string, so this is validated
1433
+ // here at the owning layer, not trusted from the child transport.
1434
+ if (data.mailbox !== MAILBOX_INBOX_DIR) {
1435
+ await controlSender.send({
1436
+ type: "mailbox.mutate.response",
1437
+ data: {
1438
+ requestId: data.requestId,
1439
+ result: {
1440
+ ok: false,
1441
+ reason: `unknown mailbox "${data.mailbox}"; only ${MAILBOX_INBOX_DIR} is writable`,
1442
+ },
1443
+ },
1444
+ });
1445
+ return;
1446
+ }
1447
+ try {
1448
+ const expungedUids = await runMailboxExclusive(async () => {
1449
+ const store = await getMailboxStore();
1450
+ if (data.op === "expunge") {
1451
+ // Snapshot the \Deleted uids before removing: `store.messages` is
1452
+ // the live array, so `.filter().map()` materializes the targets
1453
+ // before any `remove` splices it. The whole sweep runs
1454
+ // synchronously under the lock, so no snapshotted uid can vanish
1455
+ // before its `remove`.
1456
+ const uids = store.messages
1457
+ .filter((m) => m.flags.has(MAILBOX_FLAG_DELETED))
1458
+ .map((m) => m.uid);
1459
+ for (const uid of uids) {
1460
+ store.remove(uid);
1461
+ // Bound the id->uid map: drop any entry now pointing at a removed
1462
+ // uid. Not load-bearing -- `markMailboxProcessed` guards with
1463
+ // `find` -- but keeps the map from retaining dead uids.
1464
+ for (const [messageId, mappedUid] of mailboxUidByMessageId) {
1465
+ if (mappedUid === uid)
1466
+ mailboxUidByMessageId.delete(messageId);
1467
+ }
1468
+ }
1469
+ await store.flush();
1470
+ return uids;
1471
+ }
1472
+ if (data.op === "addFlags") {
1473
+ store.addFlags(data.uid, data.flags);
1474
+ }
1475
+ else {
1476
+ store.removeFlags(data.uid, data.flags);
1477
+ }
1478
+ await store.flush();
1479
+ return undefined;
1480
+ });
1481
+ await controlSender.send({
1482
+ type: "mailbox.mutate.response",
1483
+ data: {
1484
+ requestId: data.requestId,
1485
+ result: expungedUids === undefined
1486
+ ? { ok: true }
1487
+ : { ok: true, expungedUids },
1488
+ },
1489
+ });
1490
+ }
1491
+ catch (cause) {
1492
+ // Reply on the same captured sender. If this send itself throws (a
1493
+ // broken pipe), it propagates to the pump's `.catch`, and the child's
1494
+ // awaiter is rejected by the control loop's `cancelAll` -- the backstop
1495
+ // `handleOutboundMessage` also relies on. Accepted window: a mutation
1496
+ // can flush durably while its reply is undeliverable, so the agent tool
1497
+ // errors on a mutation that landed. This is inherent to apply-then-reply
1498
+ // across a teardown boundary and identical to `handleOutboundMessage`.
1499
+ const reason = cause instanceof Error ? cause.message : String(cause);
1500
+ await controlSender.send({
1501
+ type: "mailbox.mutate.response",
1502
+ data: {
1503
+ requestId: data.requestId,
1504
+ result: { ok: false, reason },
1505
+ },
1506
+ });
1507
+ }
1508
+ }
580
1509
  async function handleSubstrateWriteRequest(data) {
581
1510
  const controlSender = activeControlSender();
582
1511
  if (controlSender === null) {
@@ -628,7 +1557,7 @@ export function createWorkflowSupervisor(bindings) {
628
1557
  // is which process owns the substrate write contract.
629
1558
  const writePrincipal = {
630
1559
  kind: "workflow-process",
631
- deploymentId: bindings.deploymentId,
1560
+ anchorRunId: bindings.anchorRunId,
632
1561
  };
633
1562
  // The commit's terminal detection comes from the kind handler's
634
1563
  // typed `newlyTerminalRuns` signal (returned below), not a sniff of
@@ -656,7 +1585,7 @@ export function createWorkflowSupervisor(bindings) {
656
1585
  // benchmark's per-message OLS fit groups on.
657
1586
  const legClassification = classifyProxiedWriteLeg(data.preservePrefix);
658
1587
  if (legClassification !== null) {
659
- legMarkStart(legClassification.runId, legClassification.leg);
1588
+ legMarkStart(legClassification.messageId, legClassification.leg);
660
1589
  }
661
1590
  try {
662
1591
  const { commitSha, newlyTerminalRuns } = await bindings.repoStore.writeTreePreservingPrefix(writePrincipal, validatedRepoId, data.ref, {
@@ -701,22 +1630,11 @@ export function createWorkflowSupervisor(bindings) {
701
1630
  });
702
1631
  // D2 leg end: the substrate commit (hash objects, write tree,
703
1632
  // advance ref under the per-repo lock) just resolved. Stamped here,
704
- // before the terminal-write markConsumed-coupling wait below, so the
705
- // run-event/wal leg measures only its own commit and not the
706
- // dispatch loop's markConsumed (which the `markconsumed` leg owns).
1633
+ // before the response, so the run-event/wal leg measures only its own
1634
+ // commit and not the dispatch loop's markConsumed (which the
1635
+ // `markconsumed` leg owns).
707
1636
  if (legClassification !== null) {
708
- legMarkEnd(legClassification.runId, legClassification.leg);
709
- }
710
- const watchdog = await synchronouslyDispatchTerminalWrite(newlyTerminalRuns);
711
- if (!watchdog.ok) {
712
- await controlSender.send({
713
- type: "substrate.write.response",
714
- data: {
715
- requestId: data.requestId,
716
- result: { ok: false, reason: watchdog.reason },
717
- },
718
- });
719
- return;
1637
+ legMarkEnd(legClassification.messageId, legClassification.leg);
720
1638
  }
721
1639
  await controlSender.send({
722
1640
  type: "substrate.write.response",
@@ -740,7 +1658,7 @@ export function createWorkflowSupervisor(bindings) {
740
1658
  substrate: bindings.repoStore,
741
1659
  repoId: validatedRepoId,
742
1660
  ref: data.ref,
743
- deploymentId: bindings.deploymentId,
1661
+ anchorRunId: bindings.anchorRunId,
744
1662
  runId,
745
1663
  }).catch((cause) => {
746
1664
  logger.warn `compaction of run ${runId} failed: ${cause instanceof Error ? cause.message : String(cause)}`;
@@ -765,129 +1683,6 @@ export function createWorkflowSupervisor(bindings) {
765
1683
  });
766
1684
  }
767
1685
  }
768
- // Per-runId synchronization between the substrate-write handler
769
- // and the dispatch loop's `markConsumed`. The handler arms a
770
- // waiter when it commits a terminal-event blob and waits for the
771
- // dispatch loop to fire `resolveMarkConsumedWaiter(runId)` before
772
- // sending the substrate.write.response back to the child.
773
- const markConsumedCompletionWaiters = new Map();
774
- function resolveMarkConsumedWaiter(runId) {
775
- const waiter = markConsumedCompletionWaiters.get(runId);
776
- if (waiter === undefined)
777
- return;
778
- markConsumedCompletionWaiters.delete(runId);
779
- waiter.resolve();
780
- }
781
- /**
782
- * Hold the substrate.write.response until the dispatch loop's
783
- * markConsumed settles for each run the kind handler reports as newly
784
- * terminal in this commit. Terminal-ness comes from the handler's typed
785
- * `newlyTerminalRuns` signal -- determined authoritatively during
786
- * validation -- not re-derived from the committed path shape, so it
787
- * survives the run-event layout changing (e.g. compaction folding a
788
- * run's per-event files into one combined file). The wait is per-runId
789
- * so multiple runs can proceed concurrently if a future dispatch loop
790
- * ever processes more than one mail in parallel.
791
- *
792
- * A watchdog timeout (`terminalWriteWatchdogMs`) caps each wait so a
793
- * never-arming markConsumed (a bug in the dispatch loop, a torn-down
794
- * cohort, a stalled inbox primitive) does not deadlock the child's
795
- * write -- and therefore the runtime body, and therefore the dispatch
796
- * loop. On expiry the waiter is force-released and a structured failure
797
- * propagates back to the child as
798
- * `{ ok: false, reason: "terminal-write watchdog timeout: ..." }`.
799
- */
800
- async function synchronouslyDispatchTerminalWrite(newlyTerminalRuns) {
801
- const holds = [];
802
- for (const { runId, terminalEventJson } of newlyTerminalRuns) {
803
- if (!inFlightRuns.has(runId))
804
- continue;
805
- holds.push(holdResponseForMarkConsumed(runId, terminalEventJson));
806
- }
807
- if (holds.length === 0)
808
- return { ok: true };
809
- const results = await Promise.all(holds);
810
- return results.find((r) => !r.ok) ?? { ok: true };
811
- }
812
- async function holdResponseForMarkConsumed(runId, terminalEventJson) {
813
- const completed = new Promise((resolve, reject) => {
814
- markConsumedCompletionWaiters.set(runId, { resolve, reject });
815
- });
816
- const broadcaster = activeTerminalBroadcaster();
817
- if (broadcaster !== null) {
818
- const synthetic = synthesizeTerminalEvent(terminalEventJson);
819
- if (synthetic !== null) {
820
- broadcaster.notify(runId, synthetic);
821
- }
822
- }
823
- let timeoutHandle = null;
824
- const watchdog = new Promise((resolve) => {
825
- timeoutHandle = setTimeout(() => {
826
- timeoutHandle = null;
827
- // Force-release the waiter so the dispatch loop's eventual
828
- // resolve does not strand a dangling map entry, then surface
829
- // the structured failure to the caller. The reason text is
830
- // logged through the package logger so the watchdog is not
831
- // silent on the host side.
832
- const stillPending = markConsumedCompletionWaiters.get(runId) !== undefined;
833
- if (stillPending) {
834
- markConsumedCompletionWaiters.delete(runId);
835
- }
836
- const reason = `terminal-write watchdog timeout: markConsumed for runId=${runId} did not settle within ${String(terminalWriteWatchdogMs)}ms`;
837
- logger.error `${reason}`;
838
- resolve({ ok: false, reason });
839
- }, terminalWriteWatchdogMs);
840
- });
841
- const result = await Promise.race([
842
- completed.then(() => ({ ok: true })),
843
- watchdog,
844
- ]);
845
- if (timeoutHandle !== null) {
846
- clearTimeout(timeoutHandle);
847
- }
848
- return result;
849
- }
850
- function synthesizeTerminalEvent(terminalEventJson) {
851
- let parsed;
852
- try {
853
- parsed = JSON.parse(terminalEventJson);
854
- }
855
- catch {
856
- return null;
857
- }
858
- if (typeof parsed !== "object" ||
859
- parsed === null ||
860
- !("type" in parsed) ||
861
- !("seq" in parsed)) {
862
- return null;
863
- }
864
- const body = parsed;
865
- if (typeof body.seq !== "number")
866
- return null;
867
- const at = typeof body.at === "string" ? body.at : new Date().toISOString();
868
- if (body.type === "RunCompleted") {
869
- return { kind: "RunCompleted", seq: body.seq, at };
870
- }
871
- if (body.type === "RunCancelled") {
872
- return { kind: "RunCancelled", seq: body.seq, at };
873
- }
874
- if (body.type === "RunFailed") {
875
- // The wire schema makes `error.message` required when the event
876
- // type is `RunFailed`. An event that doesn't carry one is a
877
- // contract violation upstream of the supervisor; coercing it to an
878
- // empty string would silently hide the producer bug.
879
- if (typeof body.error?.message !== "string") {
880
- throw new Error(`synthesizeTerminalEvent: RunFailed event missing required error.message`);
881
- }
882
- return {
883
- kind: "RunFailed",
884
- seq: body.seq,
885
- at,
886
- error: { message: body.error.message },
887
- };
888
- }
889
- return null;
890
- }
891
1686
  function activeControlSender() {
892
1687
  if (state.phase === "starting" ||
893
1688
  state.phase === "running" ||
@@ -896,14 +1691,6 @@ export function createWorkflowSupervisor(bindings) {
896
1691
  }
897
1692
  return null;
898
1693
  }
899
- function activeTerminalBroadcaster() {
900
- if (state.phase === "starting" ||
901
- state.phase === "running" ||
902
- state.phase === "recycling") {
903
- return state.terminalBroadcaster;
904
- }
905
- return null;
906
- }
907
1694
  async function wireChild(args) {
908
1695
  const controlSender = createControlChannelSender({
909
1696
  privateKeySeed: args.ipcKeypair.privateKey,
@@ -921,10 +1708,11 @@ export function createWorkflowSupervisor(bindings) {
921
1708
  hmacKey: args.hmacKey,
922
1709
  channelId: args.channelId,
923
1710
  reader: args.handle.eventReader,
924
- onCrash: (reason) => {
925
- logger.error `workflow-process event channel crash: {reason}`;
926
- void shutdownInternal({ reason });
927
- },
1711
+ // Route event-channel crashes through the same funnel as
1712
+ // control-channel crashes so both drive the respawn/crash-loop path
1713
+ // uniformly on the live cohort (and defer to the owning lifecycle
1714
+ // path in every other phase).
1715
+ onCrash: onChildCrash,
928
1716
  });
929
1717
  const eventPump = pumpEvents(eventIter, args.onInferenceEvent);
930
1718
  return {
@@ -951,7 +1739,7 @@ export function createWorkflowSupervisor(bindings) {
951
1739
  channelId,
952
1740
  hmacKey,
953
1741
  hostPublicKey: ipcKeypair.publicKey,
954
- deploymentId: bindings.deploymentId,
1742
+ anchorRunId: bindings.anchorRunId,
955
1743
  deploymentMailAddress: bindings.deploymentMailAddress,
956
1744
  stepCount: bindings.stepCount,
957
1745
  definitionHash: opts.definitionHash,
@@ -1016,6 +1804,7 @@ export function createWorkflowSupervisor(bindings) {
1016
1804
  terminalBroadcaster: createTerminalBroadcaster(),
1017
1805
  dispatchLoop: null,
1018
1806
  replayDone: null,
1807
+ sweepDone: null,
1019
1808
  };
1020
1809
  // Everything from here to the successful `return` runs with the state
1021
1810
  // record in "starting" (then "running"). A throw at any of these
@@ -1029,7 +1818,7 @@ export function createWorkflowSupervisor(bindings) {
1029
1818
  repoStore: bindings.repoStore,
1030
1819
  principal: bindings.readPrincipal,
1031
1820
  stepOrder: opts.stepOrder,
1032
- deploymentId: bindings.deploymentId,
1821
+ anchorRunId: bindings.anchorRunId,
1033
1822
  deriveStepAddress: bindings.deriveStepAddress,
1034
1823
  ...(bindings.deriveStepRepoId !== undefined
1035
1824
  ? { deriveStepRepoId: bindings.deriveStepRepoId }
@@ -1048,8 +1837,13 @@ export function createWorkflowSupervisor(bindings) {
1048
1837
  // first `dequeueToProcessing` so a fresh inbound mail that lands
1049
1838
  // during the replay window cannot ship ahead of the orphan once
1050
1839
  // the replay completes.
1051
- const replayDone = readOwnedMessageIds(bindings.repoStore, bindings.workflowRunRepoId)
1052
- .then((ownedMessageIds) => inboxPrimitives.replayProcessingToInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress, { ownedMessageIds }))
1840
+ // One scan of `runs/` feeds both spawn-time recovery consumers: the
1841
+ // orphan replay (which gates dispatch) and the compaction sweep (which
1842
+ // does not). Sharing the walk keeps recovery off a second O(total-runs)
1843
+ // scan.
1844
+ const scanDone = scanRunsForBoot(bindings.repoStore, bindings.workflowRunRepoId);
1845
+ const replayDone = scanDone
1846
+ .then(({ ownedMessageIds }) => inboxPrimitives.replayProcessingToInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress, { ownedMessageIds }))
1053
1847
  .then(() => {
1054
1848
  wakeDispatch();
1055
1849
  })
@@ -1067,7 +1861,7 @@ export function createWorkflowSupervisor(bindings) {
1067
1861
  // sweep that picks up parked orphans. Left as logged
1068
1862
  // best-effort until that lands.
1069
1863
  const message = cause instanceof Error ? cause.message : String(cause);
1070
- logger.warn `replayProcessingToInbox on spawn failed: ${message}`;
1864
+ logger.warn `boot recovery scan or processing replay failed on spawn: ${message}`;
1071
1865
  });
1072
1866
  // Hold the replay promise on the active-state record so
1073
1867
  // `shutdownInternal` awaits its settlement before tearing the
@@ -1075,6 +1869,36 @@ export function createWorkflowSupervisor(bindings) {
1075
1869
  // flight would otherwise leave the substrate write pending past
1076
1870
  // the supervisor's exit.
1077
1871
  state.replayDone = replayDone;
1872
+ // Re-seal runs a crash left terminal-but-per-event when their
1873
+ // fire-and-forget fold never ran. Unlike the replay above, this must
1874
+ // NOT gate dispatch: reclaiming leaked per-event files is housekeeping
1875
+ // and cannot be allowed to delay the first dequeue. Best-effort, held
1876
+ // on the active-state record so shutdown awaits its settlement (see the
1877
+ // `sweepDone` field docstring for the teardown-latency tradeoff).
1878
+ const sweepDone = scanDone
1879
+ .then(({ pendingSealRunIds }) => recoverInterruptedCompactions({
1880
+ substrate: bindings.repoStore,
1881
+ repoId: bindings.workflowRunRepoId,
1882
+ ref: bindings.workflowRunRef,
1883
+ anchorRunId: bindings.anchorRunId,
1884
+ pendingSealRunIds,
1885
+ }))
1886
+ .then(({ sealed, failed }) => {
1887
+ if (sealed > 0) {
1888
+ logger.info `recovery sweep sealed ${String(sealed)} interrupted run(s)`;
1889
+ }
1890
+ if (failed.length > 0) {
1891
+ const detail = failed
1892
+ .map((f) => `${f.runId} (${f.message})`)
1893
+ .join("; ");
1894
+ logger.warn `recovery sweep left ${String(failed.length)} run(s) unsealed: ${detail}`;
1895
+ }
1896
+ })
1897
+ .catch((cause) => {
1898
+ const message = cause instanceof Error ? cause.message : String(cause);
1899
+ logger.warn `boot recovery scan or compaction sweep failed on spawn: ${message}`;
1900
+ });
1901
+ state.sweepDone = sweepDone;
1078
1902
  bindings.mailBus.registerAddress(bindings.deploymentMailAddress);
1079
1903
  const mailUnsubscribe = bindings.mailBus.subscribeMailForAddress(bindings.deploymentMailAddress, onMailMessage);
1080
1904
  state.mailUnsubscribe = mailUnsubscribe;
@@ -1124,19 +1948,47 @@ export function createWorkflowSupervisor(bindings) {
1124
1948
  // same control channel `trigger.fire` uses, so the ordering
1125
1949
  // guarantee (`grants-updated` lands before `trigger.fire`) holds
1126
1950
  // for buffered and post-ready inbound mail alike.
1127
- await wired.wiring.controlSender.send({
1128
- type: "grants-updated",
1129
- data: {
1130
- snapshot: {
1131
- steps: credentialsSnapshot.steps.map((s) => ({
1132
- stepId: s.stepId,
1133
- address: s.address,
1134
- grants: [...s.grants],
1135
- contentHash: s.contentHash,
1136
- })),
1951
+ //
1952
+ // Suppressed when `onRunStart` is wired: that binding makes the
1953
+ // dispatch loop push a per-run snapshot before each `trigger.fire`,
1954
+ // so the spawn-time push would only mask a broken per-run barrier
1955
+ // (the child would already hold grants and never hit its throw-on-
1956
+ // null guard). The per-run push is then the sole grants source.
1957
+ if (bindings.onRunStart === undefined) {
1958
+ await wired.wiring.controlSender.send({
1959
+ type: "grants-updated",
1960
+ data: {
1961
+ snapshot: {
1962
+ steps: credentialsSnapshot.steps.map((s) => ({
1963
+ stepId: s.stepId,
1964
+ address: s.address,
1965
+ grants: [...s.grants],
1966
+ contentHash: s.contentHash,
1967
+ })),
1968
+ },
1137
1969
  },
1138
- },
1139
- });
1970
+ });
1971
+ }
1972
+ // Deliver the run's credential material to the child on EVERY spawn, not
1973
+ // only through the per-trigger `onRunStart` barrier. A restored run resumes
1974
+ // from its parked state without a fresh `trigger.fire` (a signal wakes it),
1975
+ // so the barrier would never re-deliver the cell and the resumed run's
1976
+ // inference would fail closed. Seeding the live cell here at spawn lets an
1977
+ // offline restart resolve each source's credential from the persisted
1978
+ // (unsealed) delivery without waiting on a hub reconnect. Unlike the grants
1979
+ // push above this is NOT suppressed when `onRunStart` is wired: the barrier
1980
+ // fires per trigger, but a resume has no trigger, so the spawn push is the
1981
+ // only credential source on the resume path. Idempotent with the barrier's
1982
+ // own push on a fresh run: the child merges both, and both carry the same
1983
+ // mirror. Absent when the deployment binds no credentials. Reads the live
1984
+ // mirror, not the frozen deploy delivery, so a credential revoked earlier
1985
+ // this process stays evicted.
1986
+ if (currentCredentialDelivery !== null) {
1987
+ await wired.wiring.controlSender.send({
1988
+ type: "credentials-updated",
1989
+ data: { delivery: currentCredentialDelivery },
1990
+ });
1991
+ }
1140
1992
  // Transition to running. The dispatch loop (started below)
1141
1993
  // picks up any pre-ready buffered mail through the FIFO inbox
1142
1994
  // queue rather than through an in-memory buffer; arrival order
@@ -1169,7 +2021,13 @@ export function createWorkflowSupervisor(bindings) {
1169
2021
  terminalBroadcaster: startingPhaseBroadcaster,
1170
2022
  dispatchLoop,
1171
2023
  replayDone,
2024
+ sweepDone,
1172
2025
  };
2026
+ // Bump the generation and arm the exit-watcher atomically with the
2027
+ // running transition (no await between the swap above and this call)
2028
+ // so an unexpected exit of this child is classified against the
2029
+ // right generation.
2030
+ armChildForRunning(handle);
1173
2031
  // Kick the dispatch loop in case mail landed in the inbox
1174
2032
  // before the loop's first `await dispatchWake`. A wake against a
1175
2033
  // freshly-minted promise is a no-op; the dispatch loop's first
@@ -1200,6 +2058,18 @@ export function createWorkflowSupervisor(bindings) {
1200
2058
  const message = cause instanceof Error ? cause.message : String(cause);
1201
2059
  logger.error `upstream control pump failed: ${message}`;
1202
2060
  });
2061
+ // Trigger A: re-register every correlation the freshly-ready child is
2062
+ // parked on. A `park.notify` register can be lost while the hub is down
2063
+ // at the original suspend; a child that resumes such a parked run (a
2064
+ // sidecar restart re-spawning this deployment, or a recycle -- see the
2065
+ // matching call in `installNewChild`) re-parks without re-emitting, so
2066
+ // the supervisor re-drives it from every re-establishment. Fire-and-
2067
+ // forget after the pump is armed to route the response: best-effort,
2068
+ // watchdog-bounded, and an empty round-trip when nothing is parked.
2069
+ void reEmitParkedCorrelations().catch((cause) => {
2070
+ const message = cause instanceof Error ? cause.message : String(cause);
2071
+ logger.warn `re-emit of parked correlations on re-establishment failed: ${message}`;
2072
+ });
1203
2073
  // Arm the recycle policy. The policy is a no-op when all bounds
1204
2074
  // are `undefined`; bounds resolution lives inside `createRecyclePolicy`.
1205
2075
  if (bindings.recyclePolicy !== undefined) {
@@ -1318,60 +2188,200 @@ export function createWorkflowSupervisor(bindings) {
1318
2188
  }
1319
2189
  /**
1320
2190
  * Forward one dequeued inbox entry to the child as `trigger.fire`
1321
- * and record its runId as in-flight. The runId is the messageId
1322
- * the envelope carries (one run per trigger fire per discovery
1323
- * Q3.1); the same value is what the dispatch loop waits on via
1324
- * `terminalEventSource`.
2191
+ * and record its runId as in-flight. The runId is the local part of the
2192
+ * deployment's mail address (see `deriveWorkflowRunId`), identifying its one
2193
+ * top-level run. The resolved `Mail` (headers plus committed part references)
2194
+ * rides in the frame as the run's trigger payload; the `messageId`
2195
+ * accompanies it for correlation and audit. The runId is the same value the
2196
+ * dispatch loop waits on via `terminalEventSource`.
1325
2197
  */
1326
- async function forwardDispatchedEntry(sender, messageId, receivedAt) {
2198
+ async function forwardDispatchedEntry(sender, messageId, receivedAt, runId, payload) {
1327
2199
  await sender.send({
1328
2200
  type: "trigger.fire",
1329
2201
  data: {
1330
- runId: messageId,
2202
+ runId,
1331
2203
  messageId,
1332
2204
  receivedAt,
2205
+ payload,
1333
2206
  },
1334
2207
  });
1335
- inFlightRuns.add(messageId);
1336
- return messageId;
2208
+ cohortRunIds.add(runId);
2209
+ return runId;
2210
+ }
2211
+ /**
2212
+ * Resolve a dequeued inbound mail to the run's input: a decoded `Mail`
2213
+ * (headers plus part descriptors that reference the part bytes committed to
2214
+ * the workflow-run substrate). The supervisor is the sole mail owner and
2215
+ * commits the parts here (a direct workflow-run write; the workflow child's
2216
+ * control loop cannot do a synchronous proxied write without deadlock), so
2217
+ * both turns share this one preparation site.
2218
+ *
2219
+ * The two failure modes are deliberately distinct:
2220
+ * - A DETERMINISTIC input rejection -- missing bytes, unparseable MIME, or
2221
+ * a messageId that cannot form a path segment -- returns `{ ok: false }`
2222
+ * so the caller drops the mail. Replaying it would fail identically.
2223
+ * - A TRANSIENT substrate write failure propagates (thrown), so the caller
2224
+ * treats it as a dispatch fault and leaves the mail reclaimable rather
2225
+ * than silently discarding it on an infrastructure hiccup.
2226
+ */
2227
+ async function prepareMail(envelope, runId) {
2228
+ if (envelope.rawMessage === undefined) {
2229
+ return {
2230
+ ok: false,
2231
+ rejection: {
2232
+ code: "malformed_mail",
2233
+ message: `inbound mail ${envelope.messageId} carries no rawMessage bytes`,
2234
+ },
2235
+ };
2236
+ }
2237
+ let decoded;
2238
+ try {
2239
+ decoded = decodeMail(base64Decode(envelope.rawMessage));
2240
+ }
2241
+ catch (cause) {
2242
+ const message = cause instanceof Error ? cause.message : String(cause);
2243
+ return {
2244
+ ok: false,
2245
+ rejection: {
2246
+ code: "malformed_mail",
2247
+ message: `inbound mail ${envelope.messageId} could not be decoded: ${message}`,
2248
+ },
2249
+ };
2250
+ }
2251
+ const writePrincipal = {
2252
+ kind: "supervisor",
2253
+ anchorRunId: bindings.anchorRunId,
2254
+ };
2255
+ try {
2256
+ const mail = await commitMail({
2257
+ substrate: bindings.repoStore,
2258
+ repoId: bindings.workflowRunRepoId,
2259
+ principal: writePrincipal,
2260
+ runId,
2261
+ ref: bindings.workflowRunRef,
2262
+ }, envelope.messageId, decoded);
2263
+ return { ok: true, mail };
2264
+ }
2265
+ catch (cause) {
2266
+ if (cause instanceof InvalidMailError) {
2267
+ return {
2268
+ ok: false,
2269
+ rejection: { code: "malformed_mail", message: cause.message },
2270
+ };
2271
+ }
2272
+ throw cause;
2273
+ }
2274
+ }
2275
+ /**
2276
+ * Push the run's grants snapshot to the child ahead of its
2277
+ * `trigger.fire`. Returns `true` if the barrier FAILED (the caller must
2278
+ * skip the fire; the run has already been settled as `RunFailed`) and
2279
+ * `false` if the barrier passed or is not armed (`onRunStart` unwired,
2280
+ * where `spawn` supplied the snapshot instead).
2281
+ *
2282
+ * The sink is a request/response contract: the supervisor awaits the
2283
+ * returned snapshot and awaits the `grants-updated` send so both land on
2284
+ * the child's control channel before the trigger. A throw from either --
2285
+ * the sink itself or the control send -- is surfaced as a synthesized
2286
+ * `RunFailed` fanned out to this run's broadcaster watcher, never
2287
+ * swallowed, so the run fails deterministically instead of the child
2288
+ * authorizing against a stale or absent snapshot.
2289
+ */
2290
+ async function pushRunGrants(sender, runId, broadcaster) {
2291
+ if (bindings.onRunStart === undefined)
2292
+ return false;
2293
+ try {
2294
+ const snapshot = await bindings.onRunStart({
2295
+ runId,
2296
+ anchorRunId: bindings.anchorRunId,
2297
+ });
2298
+ await sender.send({
2299
+ type: "grants-updated",
2300
+ data: {
2301
+ snapshot: {
2302
+ steps: snapshot.steps.map((s) => ({
2303
+ stepId: s.stepId,
2304
+ address: s.address,
2305
+ grants: [...s.grants],
2306
+ contentHash: s.contentHash,
2307
+ })),
2308
+ },
2309
+ },
2310
+ });
2311
+ // Deliver the deployment's credential material on the same pre-trigger
2312
+ // barrier, so a tool that resolves a credential on the first step already
2313
+ // has it in the child's cell. Reads the live mirror, not the frozen deploy
2314
+ // delivery: a rotation or revocation delivered earlier via
2315
+ // `deliverCredentials` is reflected here, and a recycled child (seeded only
2316
+ // by this barrier) inherits the current set instead of the deploy-time one.
2317
+ // Absent when the deployment binds none.
2318
+ if (currentCredentialDelivery !== null) {
2319
+ await sender.send({
2320
+ type: "credentials-updated",
2321
+ data: { delivery: currentCredentialDelivery },
2322
+ });
2323
+ }
2324
+ return false;
2325
+ }
2326
+ catch (cause) {
2327
+ const message = cause instanceof Error ? cause.message : String(cause);
2328
+ logger.error `onRunStart grants barrier failed for run ${runId}; failing the run: ${message}`;
2329
+ broadcaster.notify(runId, {
2330
+ kind: "RunFailed",
2331
+ seq: 0,
2332
+ at: new Date().toISOString(),
2333
+ error: {
2334
+ message: `workflow-host supervisor: run ${runId} not authorized; grants barrier failed before trigger.fire: ${message}`,
2335
+ },
2336
+ });
2337
+ return true;
2338
+ }
1337
2339
  }
1338
2340
  /**
1339
2341
  * One iteration of the dispatch loop: dequeue the FIFO-first inbox
1340
- * entry, forward it as a `trigger.fire`, wait for the corresponding
1341
- * run's terminal event (or for the cohort to abort), then
1342
- * `markConsumed`. Returns `true` if a dispatch landed (caller should
1343
- * loop immediately) and `false` if the inbox was empty (caller
1344
- * should await the next wake).
2342
+ * entry, decide whether to `signal.deliver` or `trigger.fire` (or wait
2343
+ * if the run is in-flight but not yet parked), then `markConsumed`
2344
+ * once the child has taken up the message -- for a `trigger.fire` that
2345
+ * means after the run reaches a terminal event or parks, so the
2346
+ * claim-check entry the child still needs to read is not deleted out
2347
+ * from under it. Returns `true` if a dispatch landed (caller should
2348
+ * loop immediately) and `false` if the inbox was empty (caller should
2349
+ * await the next wake).
1345
2350
  */
1346
2351
  async function dispatchOne(sender, cohortAbort, broadcaster) {
1347
2352
  if (cohortAbort.signal.aborted)
1348
2353
  return false;
1349
- // Subscribe to the terminal broadcaster BEFORE forwarding the
1350
- // trigger.fire so a terminal event the child notifies between
1351
- // forward and subscribe cannot be missed. The broadcaster fires
1352
- // its listeners synchronously inside `notify`; with the subscribe
1353
- // ordered first the listener buffers the event until the
1354
- // dispatch loop's `iter.next()` consumes it.
1355
2354
  const beforeDequeueMs = dispatchTimingEnabled() ? performance.now() : 0;
1356
2355
  const dequeued = await inboxPrimitives.dequeueToProcessing(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress);
1357
2356
  if (dequeued === null)
1358
2357
  return false;
1359
2358
  const envelope = dequeued.envelope;
1360
- const runId = envelope.messageId;
1361
- currentDispatchRunId = runId;
1362
- emitDispatchTiming(runId, "dispatch-start", beforeDequeueMs);
2359
+ const runId = deriveWorkflowRunId(bindings.deploymentMailAddress);
2360
+ const messageId = envelope.messageId;
2361
+ let rejection;
2362
+ const rejectTerminalRun = () => {
2363
+ if (rejection !== undefined)
2364
+ return;
2365
+ rejection = {
2366
+ code: "workflow_run_terminal",
2367
+ message: `Workflow run ${runId} is terminal and cannot be fired again`,
2368
+ };
2369
+ logger.warn `rejecting inbound mail ${messageId}: workflow run ${runId} is terminal`;
2370
+ };
2371
+ currentDispatchMessageId = messageId;
2372
+ emitDispatchTiming(messageId, "dispatch-start", beforeDequeueMs);
1363
2373
  // D2 leg: the claim-check dequeue READ. `dispatch-start` is sampled
1364
2374
  // BEFORE the dequeue (so the roundtrip bracket includes the read);
1365
2375
  // the dequeue leg's own start mark is that same pre-dequeue sample
1366
2376
  // re-stamped under the leg channel, and its end is now (the read just
1367
2377
  // completed). Emitting the start retroactively here -- rather than
1368
- // before the await -- keeps the leg keyed by the runId, which is only
1369
- // known after the dequeue resolves.
2378
+ // before the await -- keeps the leg keyed by the messageId, which is
2379
+ // only known after the dequeue resolves.
1370
2380
  if (bindings.onDispatchTiming !== undefined) {
1371
2381
  try {
1372
2382
  bindings.onDispatchTiming({
1373
2383
  kind: "leg",
1374
- runId,
2384
+ messageId,
1375
2385
  leg: "dequeue",
1376
2386
  phase: "start",
1377
2387
  atMs: beforeDequeueMs,
@@ -1379,28 +2389,233 @@ export function createWorkflowSupervisor(bindings) {
1379
2389
  }
1380
2390
  catch (cause) {
1381
2391
  const message = cause instanceof Error ? cause.message : String(cause);
1382
- logger.warn `onDispatchTiming leg observer threw for ${runId} (dequeue start): ${message}`;
2392
+ logger.warn `onDispatchTiming leg observer threw for ${messageId} (dequeue start): ${message}`;
2393
+ }
2394
+ }
2395
+ legMarkEnd(messageId, "dequeue");
2396
+ // In-memory cohort membership is deliberately not the lifecycle
2397
+ // authority: it is empty for a new run, after a terminal frame, and while
2398
+ // a child is rediscovering a live run after restart. Consult the durable
2399
+ // log before deciding that "no cohort" means "fire". A live log is added
2400
+ // back to cohort tracking so the existing terminal/park wait handles the
2401
+ // recovery window; a terminal log is rejected permanently.
2402
+ if (!cohortRunIds.has(runId)) {
2403
+ const lifecycle = terminalRunIds.has(runId)
2404
+ ? "terminal"
2405
+ : await readWorkflowRunLifecycle(bindings.repoStore, bindings.workflowRunRepoId, runId);
2406
+ if (lifecycle === "terminal") {
2407
+ rejectTerminalRun();
2408
+ }
2409
+ else if (lifecycle === "live") {
2410
+ cohortRunIds.add(runId);
2411
+ }
2412
+ }
2413
+ if (rejection === undefined) {
2414
+ // Subscribe to the terminal broadcaster BEFORE the grants barrier so
2415
+ // a synthetic `RunFailed` from a barrier failure can be captured.
2416
+ const preIter = broadcaster.source(runId)[Symbol.asyncIterator]();
2417
+ // Per-run grants barrier. When `onRunStart` is wired, push this run's
2418
+ // grants snapshot BEFORE the trigger/signal so the child's authorize
2419
+ // closure binds to it rather than throwing on a null snapshot. The push
2420
+ // and the fire share the child's control channel, so a `grants-updated`
2421
+ // awaited here is observed by the child ahead of the trigger. A barrier
2422
+ // failure (the sink throws, or the push fails) fails the run loudly --
2423
+ // a synthesized `RunFailed` fanned out to this run's watcher -- and the
2424
+ // trigger is NOT fired, so no step ever runs against absent grants.
2425
+ const barrierFailed = await pushRunGrants(sender, runId, broadcaster);
2426
+ if (barrierFailed) {
2427
+ // Wait for the synthetic RunFailed before consuming the message.
2428
+ await waitForRunTerminal(preIter, cohortAbort.signal);
2429
+ // Clean up for synthetic barrier failure (real terminal events are
2430
+ // cleaned up by pumpUpstreamControl, but synthetic ones are not).
2431
+ cohortRunIds.delete(runId);
2432
+ runInputChannels.delete(runId);
2433
+ }
2434
+ else {
2435
+ // Dispose the pre-created iterator; the normal path creates fresh
2436
+ // iterators inside the dispatch-decision loop when waiting.
2437
+ if (typeof preIter.return === "function") {
2438
+ await preIter.return();
2439
+ }
2440
+ // Unified dispatch: park → signal.deliver; no live run → trigger.fire;
2441
+ // in-flight but undecided → wait for terminal or park, then re-evaluate.
2442
+ while (!cohortAbort.signal.aborted) {
2443
+ if (terminalRunIds.has(runId)) {
2444
+ rejectTerminalRun();
2445
+ break;
2446
+ }
2447
+ // Capture the park generation BEFORE this iteration's pre-wait awaits
2448
+ // so `waitForRunTerminalOrPark` can accept a strictly-newer park that
2449
+ // fires during them (see the latch there). Re-captured each iteration
2450
+ // so a wait that returned "parked" is not re-counted next time around.
2451
+ const sinceGen = parkGenerations.get(runId) ?? 0;
2452
+ // Routing hygiene (defense-in-depth the latch never depends on): a
2453
+ // runInputChannels entry with no live run in this cohort is a stale
2454
+ // routing hazard left by a dead incarnation. Drop it BEFORE the signal
2455
+ // branch so a fresh mail cannot be routed onto a dead run's
2456
+ // correlation. The resumed.runs invariant (cohortRunIds registered
2457
+ // before its input channel) keeps a LIVE resumed run out of this
2458
+ // branch, so this only ever drops genuinely-dead entries.
2459
+ if (!cohortRunIds.has(runId) && runInputChannels.has(runId)) {
2460
+ runInputChannels.delete(runId);
2461
+ }
2462
+ const inputChannel = runInputChannels.get(runId);
2463
+ if (inputChannel !== undefined) {
2464
+ // Resolve the inbound mail to the run's input HERE, the single site
2465
+ // that knows this payload's provenance is mail, applying the SAME
2466
+ // preparation the turn-1 trigger does. The signal.deliver frame's
2467
+ // payload is the resume decision in FINAL form -- a Mail (headers plus committed part references); deliverSignal's structured signals ship their own
2468
+ // payload unchanged. Done BEFORE minting the terminal watcher so a
2469
+ // failure here cannot leak an un-finalized iterator.
2470
+ const prepared = await prepareMail(envelope, runId);
2471
+ if (!prepared.ok) {
2472
+ // A DETERMINISTICALLY malformed turn-2 mail cannot resume the
2473
+ // parked agent. DROP it: log loudly and consume it (break to the
2474
+ // post-loop markConsumed) rather than throwing -- replay would
2475
+ // re-deliver the same poison mail forever. The run stays parked
2476
+ // on its current correlation, ready for the next valid mail; one
2477
+ // bad mail must not tear down a long-lived conversation. A
2478
+ // TRANSIENT write failure is NOT caught here: `prepareMail`
2479
+ // throws it, so it propagates as a dispatch fault and the mail
2480
+ // stays reclaimable for retry.
2481
+ logger.error `signal.deliver for run ${runId}: dropping malformed inbound mail ${envelope.messageId}: ${prepared.rejection.message}`;
2482
+ break;
2483
+ }
2484
+ // Mint the terminal watcher only now, after the payload resolved, so
2485
+ // a terminal the resumed run reaches right after applying the signal
2486
+ // is not missed; the park watcher is armed inside
2487
+ // waitForRunTerminalOrPark.
2488
+ const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
2489
+ let waitEntered = false;
2490
+ try {
2491
+ await sender.send({
2492
+ type: "signal.deliver",
2493
+ data: {
2494
+ runId,
2495
+ signalName: signalName(inputChannel.correlationId),
2496
+ signalId: envelope.messageId,
2497
+ payload: prepared.mail,
2498
+ },
2499
+ });
2500
+ // Invalidate the cached input channel: its correlation is now
2501
+ // consumed by this delivery, so the NEXT mail must not reuse it.
2502
+ // The resumed run re-parks on a FRESH correlation (a new
2503
+ // park.notify repopulates runInputChannels); a mail arriving before
2504
+ // that re-park waits via the in-flight branch rather than
2505
+ // delivering onto the stale channel. Routing hygiene only -- the
2506
+ // wait keys on the park-generation edge, not this level state.
2507
+ runInputChannels.delete(runId);
2508
+ // The message was dispatched as a turn: mark its eager mailbox
2509
+ // entry \Seen/$Processed. Fire-and-forget off the dispatch path.
2510
+ markMailboxProcessed(envelope.messageId);
2511
+ // Durable-consume contract, mirroring the trigger.fire path: hold
2512
+ // markConsumed until the child has durably taken up the signal --
2513
+ // the resumed run re-parks or reaches a terminal event. That gate
2514
+ // is downstream of durability DESPITE the child's fire-and-forget
2515
+ // SignalReceived writer: the runtime reaches re-park/terminal only
2516
+ // by resuming from the COMMITTED SignalReceived, which its per-run
2517
+ // subscribeKind substrate subscription surfaces only after the
2518
+ // commit lands -- so the substrate subscription IS the ack, and a
2519
+ // failed deliver commit is observed by nothing, never re-parks, and
2520
+ // never releases markConsumed (the mail stays reclaimable). A crash
2521
+ // before the re-park/terminal leaves the claim-check entry in
2522
+ // processing/, so replayProcessingToInbox re-delivers the signal on
2523
+ // restart. On cohort abort the wait returns and the post-loop guard
2524
+ // skips markConsumed.
2525
+ waitEntered = true;
2526
+ await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
2527
+ }
2528
+ finally {
2529
+ // waitForRunTerminalOrPark finalizes the iterator it consumes; the
2530
+ // only leak is when `sender.send` throws before the wait is
2531
+ // entered, so finalize only in that case.
2532
+ if (!waitEntered && typeof iter.return === "function") {
2533
+ await iter.return(undefined).catch(() => {
2534
+ /* best-effort finalisation of the watcher iterator. */
2535
+ });
2536
+ }
2537
+ }
2538
+ break;
2539
+ }
2540
+ if (!cohortRunIds.has(runId)) {
2541
+ // Resolve the inbound mail to the run's input before firing. A
2542
+ // DETERMINISTICALLY malformed first trigger cannot start the run:
2543
+ // record the rejection on the consumed entry and drop it (break to
2544
+ // the post-loop markConsumed), since replay would fail identically.
2545
+ // A TRANSIENT write failure instead propagates from
2546
+ // `prepareMail` as a dispatch fault, leaving the mail
2547
+ // reclaimable. Unlike a turn-2 parse failure (which leaves a live
2548
+ // run parked), a malformed first trigger produces no run at all --
2549
+ // the rejection surfaces on the consumed entry, not as a RunFailed
2550
+ // terminal event.
2551
+ const prepared = await prepareMail(envelope, runId);
2552
+ if (!prepared.ok) {
2553
+ if (rejection === undefined)
2554
+ rejection = prepared.rejection;
2555
+ logger.error `trigger.fire for run ${runId}: rejecting malformed inbound mail ${envelope.messageId}: ${prepared.rejection.message}`;
2556
+ break;
2557
+ }
2558
+ // Subscribe the terminal watcher BEFORE the trigger fires. The
2559
+ // broadcaster drops a notify that has no listener (its subscribe-
2560
+ // before-fire contract), so a terminal that lands while
2561
+ // forwardDispatchedEntry is in flight would be lost and the wait
2562
+ // would hang to the backstop.
2563
+ const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
2564
+ let waitEntered = false;
2565
+ try {
2566
+ await forwardDispatchedEntry(sender, envelope.messageId, envelope.receivedAt, runId, prepared.mail);
2567
+ // The message was dispatched as a turn: mark its eager mailbox
2568
+ // entry \Seen/$Processed. Fire-and-forget off the dispatch path.
2569
+ markMailboxProcessed(envelope.messageId);
2570
+ // Wait for the child to durably take up this trigger (RunStarted
2571
+ // committed, then the run parks or terminates) before allowing
2572
+ // `markConsumed` to move the claim-check entry out of
2573
+ // `processing/`. The payload now rides the frame, so the child no
2574
+ // longer reads it from the entry -- but the durable-consume
2575
+ // contract still holds markConsumed until the run's uptake is
2576
+ // committed, so a crash before RunStarted leaves the entry in
2577
+ // processing/ for replayProcessingToInbox to re-deliver. On cohort
2578
+ // abort the wait returns and the post-loop guard skips
2579
+ // markConsumed.
2580
+ waitEntered = true;
2581
+ await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
2582
+ }
2583
+ finally {
2584
+ // waitForRunTerminalOrPark finalizes the iterator it consumes; the
2585
+ // only leak is when forward throws before the wait is
2586
+ // entered, so finalize only in that case.
2587
+ if (!waitEntered && typeof iter.return === "function") {
2588
+ await iter.return(undefined).catch(() => {
2589
+ /* best-effort finalisation of the watcher iterator. */
2590
+ });
2591
+ }
2592
+ }
2593
+ break;
2594
+ }
2595
+ const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
2596
+ const outcome = await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
2597
+ if (outcome === "aborted")
2598
+ break;
2599
+ if (outcome === "terminal") {
2600
+ // This message was waiting for an already-live run to expose its
2601
+ // next input correlation. The run terminated first, so the mail
2602
+ // was never delivered and must not fall through to trigger.fire.
2603
+ rejectTerminalRun();
2604
+ break;
2605
+ }
2606
+ // Continue loop: re-evaluate runInputChannels / cohortRunIds
2607
+ }
1383
2608
  }
1384
2609
  }
1385
- legMarkEnd(runId, "dequeue");
1386
- const iterable = broadcaster.source(runId);
1387
- const iter = iterable[Symbol.asyncIterator]();
1388
- await forwardDispatchedEntry(sender, envelope.messageId, envelope.receivedAt);
1389
- await waitForRunTerminal(iter, cohortAbort.signal);
1390
- emitDispatchTiming(runId, "reply-produced", performance.now());
1391
- inFlightRuns.delete(runId);
1392
2610
  if (cohortAbort.signal.aborted) {
1393
- // The cohort tore down before the terminal event arrived (or
1394
- // alongside it). Skip `markConsumed` so the recycle path's
1395
- // drain-side replay can reclaim the processing entry.
1396
- currentDispatchRunId = null;
1397
- resolveMarkConsumedWaiter(runId);
2611
+ currentDispatchMessageId = null;
1398
2612
  return false;
1399
2613
  }
1400
- // D2 leg: `markConsumed` is paid AFTER `reply-produced` (stamped
1401
- // above), so its growth is invisible to the 4.7 round-trip bracket --
1402
- // the leg mark makes the out-of-window cost visible.
1403
- legMarkStart(runId, "markconsumed");
2614
+ emitDispatchTiming(messageId, "reply-produced", performance.now());
2615
+ // D2 leg: `markConsumed` is paid AFTER `reply-produced`, so its growth
2616
+ // is invisible to the 4.7 round-trip bracket -- the leg mark makes the
2617
+ // out-of-window cost visible.
2618
+ legMarkStart(messageId, "markconsumed");
1404
2619
  try {
1405
2620
  await inboxPrimitives.markConsumed(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
1406
2621
  address: bindings.deploymentMailAddress,
@@ -1408,17 +2623,21 @@ export function createWorkflowSupervisor(bindings) {
1408
2623
  runId,
1409
2624
  consumedAt: Date.now(),
1410
2625
  retentionHorizonMs: consumedRetentionMs,
2626
+ ...(rejection !== undefined ? { rejection } : {}),
1411
2627
  });
1412
2628
  }
1413
2629
  catch (cause) {
1414
- const message = cause instanceof Error ? cause.message : String(cause);
1415
- logger.error `markConsumed failed for run ${runId}: ${message}`;
1416
- }
1417
- legMarkEnd(runId, "markconsumed");
1418
- resolveMarkConsumedWaiter(runId);
1419
- // §10c forced-repack A/B (measurement-only; no-op when unwired).
2630
+ // A markConsumed failure is fatal:
2631
+ // swallowing it treats the dispatch as complete while the mail is NOT
2632
+ // durably recorded consumed, hiding the failure and leaving a mail that
2633
+ // is neither cleanly consumed nor visibly failed. Propagate into the
2634
+ // dispatch fault handler so the failure surfaces and the mail stays
2635
+ // reclaimable.
2636
+ throw new Error(`failed to markConsumed for run ${runId}`, { cause });
2637
+ }
2638
+ legMarkEnd(messageId, "markconsumed");
1420
2639
  maybeRepack(runId);
1421
- currentDispatchRunId = null;
2640
+ currentDispatchMessageId = null;
1422
2641
  return true;
1423
2642
  }
1424
2643
  /**
@@ -1460,6 +2679,94 @@ export function createWorkflowSupervisor(bindings) {
1460
2679
  }
1461
2680
  }
1462
2681
  }
2682
+ /**
2683
+ * Wait until the run's terminal event lands on the cohort
2684
+ * broadcaster's iterator, the child parks the run (bumping the park
2685
+ * generation past `sinceGen`), or the cohort aborts. Returns `"terminal"`
2686
+ * when a terminal event arrived, `"parked"` when the run parked,
2687
+ * and `"aborted"` when the cohort tore down. Throws when the backstop
2688
+ * fires (see `TERMINAL_OR_PARK_BACKSTOP_MS`).
2689
+ *
2690
+ * `sinceGen` is the park generation the CALLER captured before its pre-wait
2691
+ * awaits; the wait accepts only a STRICTLY NEWER park (`generation >
2692
+ * sinceGen`). Keying on that edge -- not `runInputChannels`' level state --
2693
+ * makes the wait's correctness local: a park during the pre-wait awaits is
2694
+ * observed even though its `resolveParkNotifyWaiter` no-op'd, and a stale
2695
+ * channel entry from a prior run or incarnation cannot false-positive.
2696
+ */
2697
+ async function waitForRunTerminalOrPark(iter, abortSignal, runId, sinceGen) {
2698
+ let onAbort = null;
2699
+ const abortPromise = new Promise((resolve) => {
2700
+ if (abortSignal.aborted) {
2701
+ resolve({ source: "abort" });
2702
+ return;
2703
+ }
2704
+ onAbort = () => resolve({ source: "abort" });
2705
+ abortSignal.addEventListener("abort", onAbort, { once: true });
2706
+ });
2707
+ let parkResolve = null;
2708
+ const parkPromise = new Promise((resolve) => {
2709
+ parkResolve = () => resolve({ source: "park" });
2710
+ parkNotifyWaiters.set(runId, parkResolve);
2711
+ });
2712
+ let timeoutHandle;
2713
+ const timeoutPromise = new Promise((resolve) => {
2714
+ timeoutHandle = setTimeout(() => resolve({ source: "timeout" }), TERMINAL_OR_PARK_BACKSTOP_MS);
2715
+ });
2716
+ try {
2717
+ if (abortSignal.aborted)
2718
+ return "aborted";
2719
+ // Check-after-register: read the generation now that the waiter above is
2720
+ // armed, SYNCHRONOUSLY (no await between arming and this read, so no
2721
+ // `park.notify` can interleave). A generation past `sinceGen` means the
2722
+ // run already parked -- during the caller's pre-wait awaits, before the
2723
+ // waiter armed, so `resolveParkNotifyWaiter` no-op'd and the armed
2724
+ // parkPromise would never fire -- and this catches it rather than hanging
2725
+ // to the backstop.
2726
+ if ((parkGenerations.get(runId) ?? 0) > sinceGen)
2727
+ return "parked";
2728
+ const result = await Promise.race([
2729
+ iter.next().then((r) => ({ source: "iter", r })),
2730
+ abortPromise,
2731
+ parkPromise,
2732
+ timeoutPromise,
2733
+ ]);
2734
+ if (result.source === "abort")
2735
+ return "aborted";
2736
+ if (result.source === "park")
2737
+ return "parked";
2738
+ if (result.source === "timeout") {
2739
+ // Backstop against a lost wake or a wedged child: the run neither
2740
+ // parked, terminated, nor aborted within a generous window. Surface it
2741
+ // LOUDLY and throw so the dispatch fails -- the caller does not
2742
+ // markConsumed on a throw, so the mail stays reclaimable in
2743
+ // processing/ and is never consumed on the assumption the run
2744
+ // progressed.
2745
+ logger.error `waitForRunTerminalOrPark backstop fired for run ${runId} after ${TERMINAL_OR_PARK_BACKSTOP_MS}ms; failing the dispatch so the mail stays reclaimable`;
2746
+ throw new Error(`waitForRunTerminalOrPark backstop: run ${runId} did not park or terminate within ${TERMINAL_OR_PARK_BACKSTOP_MS}ms`);
2747
+ }
2748
+ if (result.r.done === true)
2749
+ return "aborted";
2750
+ // A terminal event for this runId arrived; stop waiting.
2751
+ return "terminal";
2752
+ }
2753
+ finally {
2754
+ if (timeoutHandle !== undefined) {
2755
+ clearTimeout(timeoutHandle);
2756
+ }
2757
+ if (parkResolve !== null) {
2758
+ parkNotifyWaiters.delete(runId);
2759
+ }
2760
+ if (onAbort !== null) {
2761
+ abortSignal.removeEventListener("abort", onAbort);
2762
+ }
2763
+ if (typeof iter.return === "function") {
2764
+ await iter.return(undefined).catch(() => {
2765
+ /* swallowed: best-effort finalisation of the watcher iterator. */
2766
+ });
2767
+ }
2768
+ }
2769
+ }
1463
2770
  /**
1464
2771
  * The dispatch loop body. Runs until the cohort aborts; each
1465
2772
  * iteration drains one inbox entry through the FIFO claim-check
@@ -1482,6 +2789,13 @@ export function createWorkflowSupervisor(bindings) {
1482
2789
  return;
1483
2790
  }
1484
2791
  while (!cohortAbort.signal.aborted) {
2792
+ // Capture the wake BEFORE the dispatch iteration (capture-before-check,
2793
+ // same discipline as the park-generation latch). `wakeDispatch` resolves
2794
+ // the CURRENT promise and swaps in a fresh one, so a mail that enqueues
2795
+ // DURING dispatchOne resolves THIS captured promise; capturing it after
2796
+ // dispatchOne would await the fresh, unresolved promise and strand that
2797
+ // mail until some later wake.
2798
+ const wake = dispatchWake.promise;
1485
2799
  let dispatched;
1486
2800
  try {
1487
2801
  dispatched = await dispatchOne(sender, cohortAbort, broadcaster);
@@ -1499,7 +2813,6 @@ export function createWorkflowSupervisor(bindings) {
1499
2813
  continue;
1500
2814
  if (cohortAbort.signal.aborted)
1501
2815
  return;
1502
- const wake = dispatchWake.promise;
1503
2816
  const abortPromise = new Promise((resolve) => {
1504
2817
  if (cohortAbort.signal.aborted) {
1505
2818
  resolve();
@@ -1517,7 +2830,7 @@ export function createWorkflowSupervisor(bindings) {
1517
2830
  substrate: bindings.repoStore,
1518
2831
  repoId: bindings.workflowRunRepoId,
1519
2832
  ref: bindings.workflowRunRef,
1520
- deploymentId: bindings.deploymentId,
2833
+ anchorRunId: bindings.anchorRunId,
1521
2834
  runId: opts.runId,
1522
2835
  origin: opts.origin,
1523
2836
  reason: opts.reason,
@@ -1530,7 +2843,9 @@ export function createWorkflowSupervisor(bindings) {
1530
2843
  await shutdownInternal({ reason: "shutdown requested" });
1531
2844
  }
1532
2845
  async function shutdownInternal(opts) {
1533
- if (state.phase === "idle" || state.phase === "stopped")
2846
+ if (state.phase === "idle" ||
2847
+ state.phase === "stopped" ||
2848
+ state.phase === "crash-looping")
1534
2849
  return;
1535
2850
  const prior = state;
1536
2851
  state = { phase: "stopping" };
@@ -1566,11 +2881,15 @@ export function createWorkflowSupervisor(bindings) {
1566
2881
  }
1567
2882
  }
1568
2883
  drainAccumulators.clear();
2884
+ cohortRunIds.clear();
2885
+ runInputChannels.clear();
2886
+ parkNotifyWaiters.clear();
2887
+ parkGenerations.clear();
1569
2888
  if (prior.phase === "starting" ||
1570
2889
  prior.phase === "running" ||
1571
2890
  prior.phase === "recycling") {
1572
2891
  prior.terminalCohortAbort.abort();
1573
- // Reject every pending merge round-trip and markConsumed waiter
2892
+ // Reject every pending merge round-trip and park-notify waiter
1574
2893
  // so handler closures awaiting them (including fire-and-forget
1575
2894
  // `handleSubstrateWriteRequest` instances) cannot outlive the
1576
2895
  // dying cohort. Without this, the `await new Promise` inside
@@ -1618,6 +2937,21 @@ export function createWorkflowSupervisor(bindings) {
1618
2937
  path only waits for the substrate write to settle. */
1619
2938
  });
1620
2939
  }
2940
+ if ((prior.phase === "starting" ||
2941
+ prior.phase === "running" ||
2942
+ prior.phase === "recycling") &&
2943
+ prior.sweepDone !== null) {
2944
+ // Await the spawn-time compaction sweep before teardown so an
2945
+ // in-flight fold's substrate commit does not outlive the supervisor
2946
+ // and interleave with the next incarnation's boot. Teardown latency
2947
+ // is bounded by the recovery backlog (see the `sweepDone` field
2948
+ // docstring); a normal boot has zero or one pending fold.
2949
+ await prior.sweepDone.catch(() => {
2950
+ /* swallowed: the sweep's own catch already surfaces failures to
2951
+ the supervisor's warn channel; the shutdown path only waits for
2952
+ the in-flight fold's substrate commit to settle. */
2953
+ });
2954
+ }
1621
2955
  if (recyclePolicy !== null) {
1622
2956
  try {
1623
2957
  recyclePolicy.stop();
@@ -1628,6 +2962,24 @@ export function createWorkflowSupervisor(bindings) {
1628
2962
  }
1629
2963
  recyclePolicy = null;
1630
2964
  }
2965
+ // Disarm the crash-loop stable-run reset timer so it cannot fire
2966
+ // against a torn-down supervisor. Also drop any pending child exit
2967
+ // that `maybeHandleChildExit` recorded but declined to act on -- it
2968
+ // leaves one pending when a respawn was in flight, or when the phase
2969
+ // had already left `running`. (The crash-loop latch path does NOT
2970
+ // leave one pending: `maybeHandleChildExit` nulls `pendingChildExit`
2971
+ // before invoking the handler that latches.) A shutdown-initiated
2972
+ // kill of a live child in the `finally` below resolves its
2973
+ // `handle.exited`, so the watcher may re-record a pending exit AFTER
2974
+ // this clear -- harmless: the phase is terminal, so
2975
+ // `maybeHandleChildExit` no-ops on it, and `spawn()` requires `idle`,
2976
+ // so the stale slot is never re-examined.
2977
+ clearStableRunResetTimer();
2978
+ // Cancel every armed respawn backoff wait. The phase was flipped to
2979
+ // `stopping` synchronously above, so each parked respawn coroutine
2980
+ // this unblocks re-checks the phase and bails without respawning.
2981
+ cancelRespawnBackoffWaits();
2982
+ pendingChildExit = null;
1631
2983
  spawnContext = null;
1632
2984
  if (prior.phase === "starting" ||
1633
2985
  prior.phase === "running" ||
@@ -1675,7 +3027,27 @@ export function createWorkflowSupervisor(bindings) {
1675
3027
  /* swallowed for the same reason as above. */
1676
3028
  });
1677
3029
  }
1678
- state = { phase: "stopped" };
3030
+ state = { phase: opts.terminalPhase ?? "stopped" };
3031
+ }
3032
+ // Surface a self-termination to the host after the terminal transition is
3033
+ // committed. The already-terminal early-return at the top dedups the common
3034
+ // case, but it does NOT cover the `stopping` window, so two self-terminating
3035
+ // callers interleaving through teardown can each fire (e.g. an onChildCrash
3036
+ // during `recycling` plus the recycle-failure catch). The sink is therefore
3037
+ // idempotent-required, not exactly-once; the reclaim it drives absorbs a
3038
+ // repeat by design. Wrapped so a throwing sink cannot re-escape here and
3039
+ // break the documented shutdown totality.
3040
+ if (opts.selfTerminated === true) {
3041
+ try {
3042
+ bindings.onSelfTerminate?.({
3043
+ phase: opts.terminalPhase ?? "stopped",
3044
+ reason: opts.reason,
3045
+ });
3046
+ }
3047
+ catch (cause) {
3048
+ const message = cause instanceof Error ? cause.message : String(cause);
3049
+ logger.warn `onSelfTerminate sink threw: ${message}`;
3050
+ }
1679
3051
  }
1680
3052
  logger.info `supervisor shutdown complete (${opts.reason})`;
1681
3053
  }
@@ -1702,11 +3074,11 @@ export function createWorkflowSupervisor(bindings) {
1702
3074
  */
1703
3075
  async function drainImpl(opts, ctx) {
1704
3076
  // Drain is meaningful only when a workflow-process child is up;
1705
- // calling it from `idle`/`stopping`/`stopped` is a no-op so the
1706
- // higher-level host shutdown sequence can call drain
1707
- // unconditionally without sniffing the phase. The recycle path
1708
- // calls drain via `drainImpl({}, { fromRecycle: true })` and
1709
- // admits `recycling` because the drain step runs against a
3077
+ // calling it from any non-active phase (`idle`/`stopping`/`stopped`/
3078
+ // `crash-looping`) is a no-op so the higher-level host shutdown
3079
+ // sequence can call drain unconditionally without sniffing the phase.
3080
+ // The recycle path calls drain via `drainImpl({}, { fromRecycle: true })`
3081
+ // and admits `recycling` because the drain step runs against a
1710
3082
  // still-live controlSender before the kill lands.
1711
3083
  if (state.phase !== "running" &&
1712
3084
  state.phase !== "starting" &&
@@ -1730,15 +3102,22 @@ export function createWorkflowSupervisor(bindings) {
1730
3102
  // body's cancellation cascade tears the run down without the
1731
3103
  // supervisor having to thread any per-run wiring beyond what the
1732
3104
  // accumulator already encapsulates.
3105
+ //
3106
+ // Runs that are already parked do not need drain escalation; the
3107
+ // runtime parks the run and the supervisor simply stops delivering
3108
+ // new mail. The cohort abort on shutdown/recycle will eventually
3109
+ // tear the run down.
1733
3110
  const cohortSource = perCohortTerminalSource(state.terminalCohortAbort, state.terminalBroadcaster);
1734
- for (const runId of inFlightRuns) {
3111
+ for (const runId of cohortRunIds) {
1735
3112
  if (drainAccumulators.has(runId))
1736
3113
  continue;
3114
+ if (runInputChannels.has(runId))
3115
+ continue;
1737
3116
  const accumulator = accumulatorFactory({
1738
3117
  substrate: bindings.repoStore,
1739
3118
  repoId: bindings.workflowRunRepoId,
1740
3119
  ref: bindings.workflowRunRef,
1741
- deploymentId: bindings.deploymentId,
3120
+ anchorRunId: bindings.anchorRunId,
1742
3121
  runId,
1743
3122
  signAsPrincipal: bindings.signAsPrincipal,
1744
3123
  drainTimeoutMs,
@@ -1752,7 +3131,7 @@ export function createWorkflowSupervisor(bindings) {
1752
3131
  }
1753
3132
  }
1754
3133
  async function recycle(opts) {
1755
- if (recycleInProgress) {
3134
+ if (respawnInProgress) {
1756
3135
  throw new Error("supervisor: recycle already in progress");
1757
3136
  }
1758
3137
  if (state.phase !== "running") {
@@ -1761,10 +3140,43 @@ export function createWorkflowSupervisor(bindings) {
1761
3140
  if (spawnContext === null) {
1762
3141
  throw new Error("supervisor: recycle called without a spawn context; spawn() must complete first");
1763
3142
  }
1764
- recycleInProgress = true;
1765
- const origin = opts.origin ?? "operator";
1766
- const prior = state;
1767
- const priorContext = spawnContext;
3143
+ // The contention read (`respawnInProgress`) stays here in the caller's
3144
+ // precondition zone rather than inside `runRespawn`: an operator double-
3145
+ // recycle is an error and must throw. `runRespawn` sets the latch
3146
+ // synchronously at entry, so this read and that set are never separated
3147
+ // by an await, and no second respawn can interleave between them.
3148
+ return runRespawn({
3149
+ origin: opts.origin ?? "operator",
3150
+ reason: opts.reason,
3151
+ prior: state,
3152
+ priorContext: spawnContext,
3153
+ drain: async (deadlineMs) => {
3154
+ // The recycle path's drain step shares the drain primitive but
3155
+ // bypasses the public surface's `recycling` silent-no-op so the
3156
+ // still-live controlSender (this step runs BEFORE abortPriorCohort
3157
+ // + kill) receives the frame. The public `drain()` silently no-ops
3158
+ // on `recycling` for external callers because the kill/respawn gap
3159
+ // can leave the controlSender dying.
3160
+ await drainImpl({ deadlineMs }, { fromRecycle: true });
3161
+ },
3162
+ });
3163
+ }
3164
+ /**
3165
+ * Shared kill/replay/respawn/install driver. The caller has already
3166
+ * verified the supervisor is `running` with a live spawn context and
3167
+ * snapshotted both as `prior`/`priorContext`; this function transitions
3168
+ * to `recycling`, runs the six-step `triggerRecycle` sequence with the
3169
+ * caller-supplied `drain` step, and swaps in the new cohort via the
3170
+ * inline `installNewChild` callback. The operator/policy/self recycle
3171
+ * path is the sole caller today; the crash-respawn path will call it with
3172
+ * a no-op drain (its child is already dead).
3173
+ */
3174
+ async function runRespawn(args) {
3175
+ // Set synchronously at entry (before any await) so the caller's
3176
+ // contention read and this set cannot be separated by an event-loop
3177
+ // turn; two respawns can never interleave.
3178
+ respawnInProgress = true;
3179
+ const { origin, reason, prior, priorContext, drain } = args;
1768
3180
  // The cohort abort no longer fires up-front. triggerRecycle drives
1769
3181
  // the drain and replay steps against a LIVE cohort first, then
1770
3182
  // invokes `abortPriorCohort` (the callback below) between replay
@@ -1793,6 +3205,7 @@ export function createWorkflowSupervisor(bindings) {
1793
3205
  terminalBroadcaster: prior.terminalBroadcaster,
1794
3206
  dispatchLoop: null,
1795
3207
  replayDone: null,
3208
+ sweepDone: prior.sweepDone,
1796
3209
  };
1797
3210
  let attempt;
1798
3211
  try {
@@ -1808,16 +3221,7 @@ export function createWorkflowSupervisor(bindings) {
1808
3221
  channelId: prior.channelId,
1809
3222
  eventPump: prior.eventPump,
1810
3223
  },
1811
- drain: async (deadlineMs) => {
1812
- // The recycle path's drain step shares the drain
1813
- // primitive but bypasses the public surface's `recycling`
1814
- // silent-no-op so the still-live controlSender (this
1815
- // step runs BEFORE abortPriorCohort + kill) receives the
1816
- // frame. The public `drain()` silently no-ops on
1817
- // `recycling` for external callers because the
1818
- // kill/respawn gap can leave the controlSender dying.
1819
- await drainImpl({ deadlineMs }, { fromRecycle: true });
1820
- },
3224
+ drain,
1821
3225
  replayProcessingToInbox: async () => {
1822
3226
  await inboxPrimitives.replayProcessingToInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress);
1823
3227
  },
@@ -1828,6 +3232,13 @@ export function createWorkflowSupervisor(bindings) {
1828
3232
  prior.terminalCohortAbort.abort();
1829
3233
  wakeDispatch();
1830
3234
  },
3235
+ // Kept inline rather than extracted: this cohort-swap closes over
3236
+ // the supervisor's full mutable cohort state (drainAccumulators,
3237
+ // cohortRunIds, runInputChannels, parkNotifyWaiters, parkGenerations,
3238
+ // rejectCohortAwaiters, spawnContext, plus `prior`/`priorContext`).
3239
+ // A standalone helper would take all of it as parameters for zero
3240
+ // reuse -- the crash-respawn path reaches this callback transitively
3241
+ // through `runRespawn`, so it needs no separate extraction.
1831
3242
  installNewChild: ({ wiring, credentialsSnapshot, controlIncoming, }) => {
1832
3243
  // Phase guard: a `shutdown()` that landed during the
1833
3244
  // kill/respawn gap (between `subprocessSpawner` and this
@@ -1871,7 +3282,11 @@ export function createWorkflowSupervisor(bindings) {
1871
3282
  accumulator.stop();
1872
3283
  }
1873
3284
  drainAccumulators.clear();
1874
- // Reject every pending merge round-trip and markConsumed
3285
+ cohortRunIds.clear();
3286
+ runInputChannels.clear();
3287
+ parkNotifyWaiters.clear();
3288
+ parkGenerations.clear();
3289
+ // Reject every pending merge round-trip and park-notify
1875
3290
  // waiter registered against the dying cohort so handler
1876
3291
  // closures cannot survive the kill/respawn gap. The new
1877
3292
  // child will re-issue substrate writes through fresh
@@ -1906,7 +3321,14 @@ export function createWorkflowSupervisor(bindings) {
1906
3321
  terminalBroadcaster: newBroadcaster,
1907
3322
  dispatchLoop: newDispatchLoop,
1908
3323
  replayDone: null,
3324
+ sweepDone: prior.sweepDone,
1909
3325
  };
3326
+ // Bump the generation and arm the exit-watcher for the
3327
+ // respawned child atomically with this running transition, so
3328
+ // the predecessor's watcher (already stale by generation) never
3329
+ // drives a spurious respawn and a crash of THIS child is
3330
+ // classified against the new generation.
3331
+ armChildForRunning(wiring.handle);
1910
3332
  // Cache fresh spawn context with the updated spawnedAt
1911
3333
  // so the policy timer's uptime check resets on recycle.
1912
3334
  const now = bindings.recyclePolicyNow ?? defaultNow;
@@ -1934,6 +3356,16 @@ export function createWorkflowSupervisor(bindings) {
1934
3356
  // entries the previous cohort's replayProcessingToInbox
1935
3357
  // just moved back.
1936
3358
  wakeDispatch();
3359
+ // Trigger A on the recycle seam: the respawned child re-parks any
3360
+ // surviving parked run without re-emitting, and a recycle leaves
3361
+ // the hub link untouched so the reconnect trigger never fires --
3362
+ // so re-drive the re-registration here too. Same fire-and-forget
3363
+ // contract as the spawn seam; the fresh cohort's controlSender is
3364
+ // in `state` now, and its pump (armed above) routes the response.
3365
+ void reEmitParkedCorrelations().catch((cause) => {
3366
+ const message = cause instanceof Error ? cause.message : String(cause);
3367
+ logger.warn `re-emit of parked correlations on re-establishment failed: ${message}`;
3368
+ });
1937
3369
  },
1938
3370
  onCrash: onChildCrash,
1939
3371
  // Edge-resolved once at the supervisor factory; recycle bounds
@@ -1945,7 +3377,7 @@ export function createWorkflowSupervisor(bindings) {
1945
3377
  ...(bindings.recyclePolicyClearTimer !== undefined
1946
3378
  ? { clearTimer: bindings.recyclePolicyClearTimer }
1947
3379
  : {}),
1948
- }, { origin, reason: opts.reason });
3380
+ }, { origin, reason });
1949
3381
  // After the recycle, await the previous cohort's dispatch
1950
3382
  // loop so a teardown coroutine cannot survive past the
1951
3383
  // recycle's return point.
@@ -1963,11 +3395,23 @@ export function createWorkflowSupervisor(bindings) {
1963
3395
  // level shutdown. Tear the prior cohort down through the same
1964
3396
  // path a real shutdown uses so the supervisor reaches a clean
1965
3397
  // `stopped` state, then re-throw so the operator sees the
1966
- // recycle failure and can redeploy.
3398
+ // failure and can redeploy.
3399
+ //
3400
+ // This teardown-to-`stopped` is shared by both callers, and that is
3401
+ // deliberate. A crash-origin respawn whose spawn/wire/handshake fails
3402
+ // is a broken deploy (a bad binary, unreadable credentials), NOT a
3403
+ // flapping child, so it does NOT feed the crash-loop exit counter and
3404
+ // does NOT reach `crash-looping`; conflating a mechanism failure with
3405
+ // a flap would muddy what that counter means. It reaches `stopped`,
3406
+ // the same terminal a failed operator recycle reaches. (Consequence:
3407
+ // this path leaves no `RunFailed` tombstone, unlike the exit-count
3408
+ // latch; the crash-respawn happy path -- a clean child death whose
3409
+ // respawn succeeds -- is what the crash-loop guard bounds.)
1967
3410
  const message = cause instanceof Error ? cause.message : String(cause);
1968
3411
  logger.error `recycle failed; tearing supervisor down: ${message}`;
1969
3412
  await shutdownInternal({
1970
3413
  reason: `recycle failed: ${message}`,
3414
+ selfTerminated: true,
1971
3415
  }).catch((shutdownCause) => {
1972
3416
  const inner = shutdownCause instanceof Error
1973
3417
  ? shutdownCause.message
@@ -1977,7 +3421,12 @@ export function createWorkflowSupervisor(bindings) {
1977
3421
  throw cause;
1978
3422
  }
1979
3423
  finally {
1980
- recycleInProgress = false;
3424
+ respawnInProgress = false;
3425
+ // Drain any child exit observed during the respawn. A crash of the
3426
+ // freshly-installed child that raced this respawn's completion was
3427
+ // deferred (respawnInProgress was set); handle it now that the latch
3428
+ // is clear. A pending exit for a superseded generation drops as stale.
3429
+ maybeHandleChildExit();
1981
3430
  }
1982
3431
  return attempt;
1983
3432
  }
@@ -1998,6 +3447,27 @@ export function createWorkflowSupervisor(bindings) {
1998
3447
  if (state.phase !== "running" && state.phase !== "starting") {
1999
3448
  throw new Error(`supervisor: deliverSignal called in phase ${state.phase}; expected starting/running`);
2000
3449
  }
3450
+ // Refresh the run's grant floor on the SAME control channel immediately
3451
+ // before the signal, so a standing ("always") approval resolved for a
3452
+ // parked run lowers the floor for the resumed run's later calls. Ordering
3453
+ // is structural: both frames ride this single seq-ordered FIFO, so the
3454
+ // `grants-updated` is observed by the child ahead of the `signal.deliver`
3455
+ // -- no dependence on hub-side dispatch timing. Best-effort by design; a
3456
+ // failed refresh is non-fatal (the durable file still governs the next
3457
+ // barrier), and it only re-reads that file, so a signal with no standing
3458
+ // approval just re-pushes the unchanged floor.
3459
+ await deliverGrants(opts.runId);
3460
+ // `deliverGrants` awaits a substrate read, yielding the event loop. A
3461
+ // crash/recycle can land in that window and swap `state` (its
3462
+ // `controlSender` then points at the dying child). Re-assert the phase the
3463
+ // pre-await guard checked, so the signal is never written into a recycling
3464
+ // child's closing pipe; the caller retries once the recycle completes. The
3465
+ // phase is read through the full union type because the pre-await guard
3466
+ // control-flow-narrowed `state`, which the yield may have invalidated.
3467
+ const phaseAfterRefresh = state.phase;
3468
+ if (phaseAfterRefresh !== "running" && phaseAfterRefresh !== "starting") {
3469
+ throw new Error(`supervisor: deliverSignal raced a recycle in phase ${phaseAfterRefresh}; expected starting/running`);
3470
+ }
2001
3471
  await state.controlSender.send({
2002
3472
  type: "signal.deliver",
2003
3473
  data: {
@@ -2026,6 +3496,85 @@ export function createWorkflowSupervisor(bindings) {
2026
3496
  },
2027
3497
  });
2028
3498
  }
3499
+ async function deliverCredentials(opts) {
3500
+ // Compute the next mirror first. It must advance regardless of phase so the
3501
+ // NEXT spawn/barrier re-asserts this update: a credential revoked while the
3502
+ // supervisor holds no live child (a crash-loop retry, a recycle transient)
3503
+ // must still be gone when the child (re)starts, not resurrected from the
3504
+ // frozen deploy delivery. This is what makes an offline-revoke reconcile
3505
+ // durable even when it arrives at a supervisor without a running child.
3506
+ const next = mergeCredentialDelivery(currentCredentialDelivery, opts.delivery, opts.revoke);
3507
+ // Send to the live child ONLY in starting/running. Outside those the control
3508
+ // sender points at a dying/absent child, so a frame would buffer behind a
3509
+ // SIGTERM or write into a closed pipe; the mirror advance below is the sole
3510
+ // effect, and the child that eventually spawns is seeded from it. The
3511
+ // supervisor is the single producer of `credentials-updated` frames.
3512
+ if (state.phase === "running" || state.phase === "starting") {
3513
+ await state.controlSender.send({
3514
+ type: "credentials-updated",
3515
+ data: {
3516
+ delivery: opts.delivery,
3517
+ ...(opts.revoke !== undefined ? { revoke: opts.revoke } : {}),
3518
+ },
3519
+ });
3520
+ }
3521
+ // Advance the mirror. A throw from `send` above skips this, leaving the
3522
+ // mirror matching the child that never received the frame. A whole-object
3523
+ // swap via the same merge the child applies, so a concurrent reader sees a
3524
+ // coherent object.
3525
+ currentCredentialDelivery = next;
3526
+ }
3527
+ /**
3528
+ * Refresh a live run's grant floor mid-run: re-read this run's durable
3529
+ * `runs/<runId>/grants.json` (via `onRunStart`, the same read the pre-trigger
3530
+ * barrier uses) and push it to the child as a `grants-updated` frame. The
3531
+ * enforcement path for a standing (`scope: "always"`) approval, which lowers
3532
+ * a tool's `ask` to `allow` in that file: the barrier only runs before a
3533
+ * trigger/signal dispatch, so a run already executing (or being resumed
3534
+ * without a fresh barrier) needs this to observe the change now.
3535
+ *
3536
+ * Distinct from `pushRunGrants` on two axes, both deliberate:
3537
+ * - It NEVER synthesizes a `RunFailed`. A refresh for a run whose child is
3538
+ * not live is normal (the durable file already carries the change and the
3539
+ * next barrier or respawn re-reads it), so it no-ops (`skipped`) rather
3540
+ * than failing the run, and a send failure to a live child is logged
3541
+ * loudly but stays non-fatal (the file still wins at the next barrier).
3542
+ * - It only ever pushes the durable file's contents through `onRunStart`; it
3543
+ * accepts no caller-supplied grants, so it can only tighten or refresh a
3544
+ * floor, never inject one a deploy did not approve.
3545
+ */
3546
+ async function deliverGrants(runId) {
3547
+ if (bindings.onRunStart === undefined)
3548
+ return "skipped";
3549
+ if (state.phase !== "running" && state.phase !== "starting") {
3550
+ return "skipped";
3551
+ }
3552
+ try {
3553
+ const snapshot = await bindings.onRunStart({
3554
+ runId,
3555
+ anchorRunId: bindings.anchorRunId,
3556
+ });
3557
+ await state.controlSender.send({
3558
+ type: "grants-updated",
3559
+ data: {
3560
+ snapshot: {
3561
+ steps: snapshot.steps.map((s) => ({
3562
+ stepId: s.stepId,
3563
+ address: s.address,
3564
+ grants: [...s.grants],
3565
+ contentHash: s.contentHash,
3566
+ })),
3567
+ },
3568
+ },
3569
+ });
3570
+ return "pushed";
3571
+ }
3572
+ catch (cause) {
3573
+ const message = cause instanceof Error ? cause.message : String(cause);
3574
+ logger.error `deliverGrants refresh failed for run ${runId}; the durable grants file still governs the next barrier/respawn: ${message}`;
3575
+ return "skipped";
3576
+ }
3577
+ }
2029
3578
  function getCredentialsSnapshot() {
2030
3579
  if (state.phase === "starting" || state.phase === "running") {
2031
3580
  return state.credentialsSnapshot;
@@ -2040,6 +3589,9 @@ export function createWorkflowSupervisor(bindings) {
2040
3589
  recycle,
2041
3590
  deliverSignal,
2042
3591
  deliverSources,
3592
+ deliverCredentials,
3593
+ deliverGrants,
3594
+ reEmitParkedCorrelations,
2043
3595
  getCredentialsSnapshot,
2044
3596
  };
2045
3597
  }
@@ -2097,71 +3649,6 @@ async function pumpEvents(iter, onInferenceEvent) {
2097
3649
  function defaultInProcessMailAuditRef(messageId, _rawMessage) {
2098
3650
  return { store: "in-process", path: messageId };
2099
3651
  }
2100
- /**
2101
- * Derive a stable message identifier from the raw bytes the bus
2102
- * delivered. The RFC 2822 `Message-ID` header (if present) is the
2103
- * canonical identifier the audit log surfaces as
2104
- * `RunStarted.consumedMessageId`; downstream consumers join inbound
2105
- * mail to workflow-run events on this value, so the header parse must
2106
- * win when the sender emitted one. A message that lacks a
2107
- * `Message-ID` header falls back to a sha256 of the raw bytes so
2108
- * runs originating from non-RFC 2822 transports still receive a
2109
- * deterministic identifier.
2110
- *
2111
- * The parser walks the message until the headers/body separator
2112
- * (`CRLF CRLF` per RFC 2822 §2.1, with the lone-`LF` variant tolerated
2113
- * to match common in-memory senders). Header-field unfolding follows
2114
- * RFC 2822 §2.2.3: a continuation line begins with whitespace and
2115
- * appends to the prior line. Header-name comparison is
2116
- * case-insensitive per RFC 2822 §1.2.2.
2117
- */
2118
- async function deriveMessageId(rawMessage) {
2119
- const messageIdFromHeader = parseMessageIdHeader(rawMessage);
2120
- if (messageIdFromHeader !== null) {
2121
- return messageIdFromHeader;
2122
- }
2123
- const digest = await crypto.subtle.digest("SHA-256",
2124
- // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- ArrayBuffer-backed at the call site; Web Crypto's BufferSource type rejects Uint8Array<ArrayBufferLike> under TS 5.9 (microsoft/TypeScript#62240)
2125
- rawMessage);
2126
- return hexEncode(new Uint8Array(digest));
2127
- }
2128
- function parseMessageIdHeader(rawMessage) {
2129
- const text = new TextDecoder("utf-8", { fatal: false }).decode(rawMessage);
2130
- // Headers end at the first blank line. RFC 2822 mandates `CRLF CRLF`
2131
- // but tolerate `LF LF` for callers that normalize line endings.
2132
- let headerSection = text;
2133
- const crlfBoundary = text.indexOf("\r\n\r\n");
2134
- const lfBoundary = text.indexOf("\n\n");
2135
- if (crlfBoundary >= 0 && (lfBoundary < 0 || crlfBoundary < lfBoundary)) {
2136
- headerSection = text.slice(0, crlfBoundary);
2137
- }
2138
- else if (lfBoundary >= 0) {
2139
- headerSection = text.slice(0, lfBoundary);
2140
- }
2141
- // Unfold continuation lines (a line starting with WSP belongs to
2142
- // the prior header field).
2143
- const lines = headerSection.split(/\r?\n/);
2144
- const unfolded = [];
2145
- for (const line of lines) {
2146
- if (line.length > 0 && (line[0] === " " || line[0] === "\t")) {
2147
- if (unfolded.length === 0)
2148
- continue;
2149
- unfolded[unfolded.length - 1] += " " + line.trim();
2150
- continue;
2151
- }
2152
- unfolded.push(line);
2153
- }
2154
- for (const line of unfolded) {
2155
- const colon = line.indexOf(":");
2156
- if (colon < 0)
2157
- continue;
2158
- const name = line.slice(0, colon).trim().toLowerCase();
2159
- if (name !== "message-id")
2160
- continue;
2161
- return line.slice(colon + 1).trim();
2162
- }
2163
- return null;
2164
- }
2165
3652
  /**
2166
3653
  * Project the wire shape of a `terminal.event` upstream control frame
2167
3654
  * into the workflow-vocabulary `TerminalRunEvent` discriminated union
@@ -2220,6 +3707,8 @@ function outboundMessageFromPayload(payload) {
2220
3707
  message.summary = payload.summary;
2221
3708
  if (payload.inReplyTo !== undefined)
2222
3709
  message.inReplyTo = payload.inReplyTo;
3710
+ if (payload.references !== undefined)
3711
+ message.references = payload.references;
2223
3712
  if (payload.correlationId !== undefined) {
2224
3713
  message.correlationId = payload.correlationId;
2225
3714
  }