@intx/workflow-host 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -10
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +53 -53
- package/dist/adapters/spawn-child.d.ts +71 -42
- package/dist/adapters/spawn-child.js +83 -77
- package/dist/adapters/step-invoker.js +84 -7
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/index.d.ts +2 -1
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +107 -13
- package/dist/child/run-child.js +290 -108
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/conversation-text.d.ts +23 -0
- package/dist/conversation-text.js +56 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3 -2
- package/dist/ipc/control-channel.d.ts +58 -0
- package/dist/ipc/control-channel.js +94 -1
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/seams/scheduler.d.ts +4 -6
- package/dist/seams/scheduler.js +74 -93
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +1 -1
- package/dist/supervisor/credentials.d.ts +11 -10
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +5 -2
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +5 -5
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +82 -25
- package/dist/supervisor/supervisor.js +1313 -410
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +134 -0
- package/dist/supervisor/types.d.ts +150 -23
- package/dist/workflow-definition-loader.d.ts +131 -0
- package/dist/workflow-definition-loader.js +316 -0
- package/package.json +12 -11
|
@@ -43,49 +43,73 @@ import { type } from "arktype";
|
|
|
43
43
|
import { getLogger } from "@intx/log";
|
|
44
44
|
import { sampleStructuralCounters, forceRepack, } from "./dispatch-attribution.js";
|
|
45
45
|
import { generateKeyPair } from "@intx/crypto";
|
|
46
|
-
import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed, readOwnedMessageIds, replayProcessingToInbox as defaultReplayProcessingToInbox, DEFAULT_CONSUMED_RETENTION_MS, } from "@intx/hub-sessions/substrate";
|
|
47
|
-
import { base64Decode, base64Encode,
|
|
46
|
+
import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed, readOwnedMessageIds, readWorkflowRunLifecycle, replayProcessingToInbox as defaultReplayProcessingToInbox, StaleInboxEnqueueError, DEFAULT_CONSUMED_RETENTION_MS, } from "@intx/hub-sessions/substrate";
|
|
47
|
+
import { base64Decode, base64Encode, deriveMessageId, deriveWorkflowRunId, signalName, } from "@intx/types";
|
|
48
48
|
import { RepoId } from "@intx/types/sidecar";
|
|
49
49
|
import { createControlChannelSender, generateChannelId, generateHmacKey, receiveControlChannel, receiveEventChannel, } from "../ipc/index.js";
|
|
50
50
|
import { assembleCredentialsSnapshot, } from "./credentials.js";
|
|
51
51
|
import { commitCancelRequested } from "./cancel-signing.js";
|
|
52
|
+
import { commitRunFailed } from "./terminal-commit.js";
|
|
52
53
|
import { buildChildSpawnEnv } from "./spawn-env.js";
|
|
53
54
|
import { compactRunEvents } from "./run-event-compaction.js";
|
|
55
|
+
import { extractConversationText } from "../conversation-text.js";
|
|
54
56
|
import { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, } from "./drain-timeout.js";
|
|
55
57
|
import { createRecyclePolicy, triggerRecycle, } from "./recycle.js";
|
|
56
58
|
import { createTerminalBroadcaster, } from "./terminal-broadcaster.js";
|
|
57
59
|
import { DEFAULT_KILL_TIMEOUT_MS, DEFAULT_READY_TIMEOUT_MS, defaultClearTimer, defaultSetTimer, killChildHandle, waitDeadline, } from "./child-termination.js";
|
|
58
60
|
const logger = getLogger(["workflow-host", "supervisor"]);
|
|
59
61
|
/**
|
|
60
|
-
* Default
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
* unbounded wait would chain into a child / runtime / dispatch loop
|
|
65
|
-
* deadlock if `markConsumed` never armed (bug in the dispatch loop, a
|
|
66
|
-
* torn-down cohort, a stalled inbox primitive). 30s sits between the
|
|
67
|
-
* recycle path's `DEFAULT_KILL_TIMEOUT_MS` (5s, a hard process-level
|
|
68
|
-
* kill cap) and `DEFAULT_DRAIN_TIMEOUT_MS` (60s, the per-deployment
|
|
69
|
-
* drain budget) -- generous enough to absorb a slow legitimate
|
|
70
|
-
* markConsumed, tight enough to surface a real deadlock long before
|
|
71
|
-
* the drainTimeout would otherwise mask it.
|
|
62
|
+
* Default crash-loop bound: the supervisor stops respawning and latches
|
|
63
|
+
* the deployment once the workflow-process child exits unexpectedly this
|
|
64
|
+
* many times within `DEFAULT_CRASH_LOOP_WINDOW_MS`. Overridable via
|
|
65
|
+
* `WorkflowSupervisorBindings.crashLoopMaxCount`.
|
|
72
66
|
*/
|
|
73
|
-
export const
|
|
67
|
+
export const DEFAULT_CRASH_LOOP_MAX_COUNT = 3;
|
|
74
68
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
* during a recycle, or a supervisor shutdown). Callers awaiting the
|
|
79
|
-
* resolved value receive an instance of this error so the failure mode
|
|
80
|
-
* is recognisable from a generic substrate-merge or markConsumed
|
|
81
|
-
* failure.
|
|
69
|
+
* Default sliding window (ms) over which `DEFAULT_CRASH_LOOP_MAX_COUNT`
|
|
70
|
+
* unexpected exits latch the deployment. Overridable via
|
|
71
|
+
* `WorkflowSupervisorBindings.crashLoopWindowMs`.
|
|
82
72
|
*/
|
|
83
|
-
export
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
73
|
+
export const DEFAULT_CRASH_LOOP_WINDOW_MS = 60_000;
|
|
74
|
+
/**
|
|
75
|
+
* Default stable-run duration (ms): once a respawned child stays up this
|
|
76
|
+
* long, the crash counter resets so flapping followed by stability does
|
|
77
|
+
* not permanently latch. Overridable via
|
|
78
|
+
* `WorkflowSupervisorBindings.crashLoopStableResetMs`.
|
|
79
|
+
*/
|
|
80
|
+
export const DEFAULT_CRASH_LOOP_STABLE_RESET_MS = 60_000;
|
|
81
|
+
/**
|
|
82
|
+
* Default initial respawn backoff (ms): the wait before the first respawn
|
|
83
|
+
* after an unexpected exit. Overridable via
|
|
84
|
+
* `WorkflowSupervisorBindings.respawnBackoffInitialMs`.
|
|
85
|
+
*/
|
|
86
|
+
export const DEFAULT_RESPAWN_BACKOFF_INITIAL_MS = 1_000;
|
|
87
|
+
/**
|
|
88
|
+
* Default cap (ms) on the exponential respawn backoff. Kept below
|
|
89
|
+
* `DEFAULT_CRASH_LOOP_WINDOW_MS` so a slow flapper's crashes still fall
|
|
90
|
+
* within the window and latch the guard. Overridable via
|
|
91
|
+
* `WorkflowSupervisorBindings.respawnBackoffMaxMs`.
|
|
92
|
+
*/
|
|
93
|
+
export const DEFAULT_RESPAWN_BACKOFF_MAX_MS = 30_000;
|
|
94
|
+
/**
|
|
95
|
+
* Default watchdog for `reEmitParkedCorrelations`' wait on the child's
|
|
96
|
+
* `parked-correlations.response`. 30s is generous enough for a healthy child
|
|
97
|
+
* to enumerate its in-flight runs and load each parked snapshot, tight
|
|
98
|
+
* enough that a wedged-but-alive child does not hang the reconnect caller
|
|
99
|
+
* until some coarser timeout intervenes.
|
|
100
|
+
*/
|
|
101
|
+
export const DEFAULT_PARKED_QUERY_WATCHDOG_MS = 30_000;
|
|
102
|
+
/**
|
|
103
|
+
* Backstop for `waitForRunTerminalOrPark`. A dispatch waits here for the child
|
|
104
|
+
* to park or terminate the run before releasing `markConsumed`; a lost park
|
|
105
|
+
* wake or a wedged child would otherwise hang the deployment's dispatch loop
|
|
106
|
+
* forever. Five minutes is far beyond any healthy per-message dispatch (which
|
|
107
|
+
* settles in well under a second), so this never fires on a legitimately long
|
|
108
|
+
* run without also being a genuine fault -- and when it does fire it is logged
|
|
109
|
+
* loudly and fails the dispatch (the mail is left reclaimable, never consumed
|
|
110
|
+
* on the assumption the run made progress), not silently swallowed.
|
|
111
|
+
*/
|
|
112
|
+
export const TERMINAL_OR_PARK_BACKSTOP_MS = 300_000;
|
|
89
113
|
/**
|
|
90
114
|
* Construct a per-deployment supervisor. All host-specific
|
|
91
115
|
* dependencies are pulled in via `bindings`; nothing in the
|
|
@@ -94,22 +118,60 @@ export class MergeAbortedError extends Error {
|
|
|
94
118
|
export function createWorkflowSupervisor(bindings) {
|
|
95
119
|
let state = { phase: "idle" };
|
|
96
120
|
/**
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
121
|
+
* ALL runIds the current child cohort is driving, regardless of
|
|
122
|
+
* who spawned them: supervisor-dispatched + self-discovered.
|
|
123
|
+
* Populated at `trigger.fire` time and when the child reports
|
|
124
|
+
* `resumed.runs`. Removed on terminal event or cohort teardown.
|
|
125
|
+
*
|
|
126
|
+
* Used by: `drain()` to arm one drainTimeout accumulator per run.
|
|
127
|
+
*/
|
|
128
|
+
const cohortRunIds = new Set();
|
|
129
|
+
/**
|
|
130
|
+
* Runs observed terminal in this supervisor process. The terminal control
|
|
131
|
+
* frame follows the durable event commit, but retaining the observation
|
|
132
|
+
* closes the short visibility window before the working tree reflects that
|
|
133
|
+
* commit. Terminal membership is permanent for a deployment: the stable
|
|
134
|
+
* top-level run is never cleared and fired again.
|
|
135
|
+
*/
|
|
136
|
+
const terminalRunIds = new Set();
|
|
137
|
+
/**
|
|
138
|
+
* Per-run input channel cache. When a long-lived run parks on an input
|
|
139
|
+
* signal, the child sends `park.notify` with `parkKind: "input"`. The
|
|
140
|
+
* supervisor stores the `correlationId` here so that subsequent mail
|
|
141
|
+
* deliveries can fire `signal.deliver` without re-reading the substrate.
|
|
142
|
+
* Cleared on terminal event or cohort abort.
|
|
143
|
+
*/
|
|
144
|
+
const runInputChannels = new Map();
|
|
145
|
+
/**
|
|
146
|
+
* Waiters for dispatch loops blocked on a `park.notify` for a
|
|
147
|
+
* specific runId. When `park.notify` arrives, the handler resolves
|
|
148
|
+
* the waiter so the dispatch loop re-evaluates routing.
|
|
149
|
+
*/
|
|
150
|
+
const parkNotifyWaiters = new Map();
|
|
151
|
+
/**
|
|
152
|
+
* Monotonic per-run INPUT-park generation. Bumped on every
|
|
153
|
+
* `park.notify(input)` for a runId. `waitForRunTerminalOrPark` captures a
|
|
154
|
+
* `sinceGen` before its caller's pre-wait awaits and returns `"parked"` when
|
|
155
|
+
* the generation later exceeds it, so the wait keys on the park EDGE rather
|
|
156
|
+
* than `runInputChannels`' LEVEL state: a park that fired during the pre-wait
|
|
157
|
+
* awaits (before the waiter armed, so `resolveParkNotifyWaiter` no-op'd) is
|
|
158
|
+
* still observed, and a stale `runInputChannels` entry from a prior run or
|
|
159
|
+
* incarnation cannot false-positive because it did not bump the generation
|
|
160
|
+
* past `sinceGen`. Incarnation-scoped: cleared with `parkNotifyWaiters` on
|
|
161
|
+
* recycle/teardown so a reused runId cannot carry a stale captured generation
|
|
162
|
+
* across incarnations.
|
|
102
163
|
*/
|
|
103
|
-
const
|
|
104
|
-
// D2 attribution (measurement-only): the
|
|
164
|
+
const parkGenerations = new Map();
|
|
165
|
+
// D2 attribution (measurement-only): the messageId the dispatch loop is
|
|
105
166
|
// currently servicing. Set at `dispatch-start`, cleared after
|
|
106
167
|
// `reply-produced`. The dispatch loop is strictly serial (one message
|
|
107
168
|
// in flight at a time -- the sustained interactive case the bench
|
|
108
|
-
// drives), so a child-proxied
|
|
109
|
-
//
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
|
|
169
|
+
// drives), so a child-proxied `substrate.write.request` is unambiguously
|
|
170
|
+
// attributable to this message. Both the WAL leg (whose `agent-state/...`
|
|
171
|
+
// prefix names no run) and the run-event leg (whose `runs/<runId>/events/`
|
|
172
|
+
// prefix names only the stable per-deployment run id, not the message)
|
|
173
|
+
// take their per-message key from here.
|
|
174
|
+
let currentDispatchMessageId = null;
|
|
113
175
|
/**
|
|
114
176
|
* Per-run drainTimeout accumulators armed by `drain()`. Held so
|
|
115
177
|
* `shutdown()` can stop every accumulator cleanly before tearing
|
|
@@ -134,21 +196,21 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
134
196
|
}
|
|
135
197
|
});
|
|
136
198
|
const drainTimeoutMs = bindings.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
|
|
137
|
-
const
|
|
199
|
+
const parkedQueryWatchdogMs = bindings.parkedQueryWatchdogMs ?? DEFAULT_PARKED_QUERY_WATCHDOG_MS;
|
|
138
200
|
// Pure observability: invoke the dispatch-timing hook (when wired) at
|
|
139
201
|
// the two per-message boundaries the 4.7 latency gate brackets. A
|
|
140
202
|
// throwing observer is swallowed and logged so a benchmark hook bug
|
|
141
203
|
// cannot wedge the dispatch loop.
|
|
142
|
-
function emitDispatchTiming(
|
|
204
|
+
function emitDispatchTiming(messageId, marker, atMs) {
|
|
143
205
|
const observer = bindings.onDispatchTiming;
|
|
144
206
|
if (observer === undefined)
|
|
145
207
|
return;
|
|
146
208
|
try {
|
|
147
|
-
observer({ kind: "roundtrip",
|
|
209
|
+
observer({ kind: "roundtrip", messageId, marker, atMs });
|
|
148
210
|
}
|
|
149
211
|
catch (cause) {
|
|
150
212
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
151
|
-
logger.warn `onDispatchTiming observer threw for ${
|
|
213
|
+
logger.warn `onDispatchTiming observer threw for ${messageId} (${marker}): ${message}`;
|
|
152
214
|
}
|
|
153
215
|
}
|
|
154
216
|
// D2 per-leg attribution (measurement-only). Emits a paired
|
|
@@ -160,14 +222,14 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
160
222
|
// observability: a throwing observer is swallowed + logged so a
|
|
161
223
|
// benchmark hook bug cannot wedge dispatch, and no clock or directory
|
|
162
224
|
// is sampled when the observer is unwired.
|
|
163
|
-
function legMarkStart(
|
|
225
|
+
function legMarkStart(messageId, leg) {
|
|
164
226
|
if (bindings.onDispatchTiming === undefined)
|
|
165
227
|
return 0;
|
|
166
228
|
const atMs = performance.now();
|
|
167
229
|
try {
|
|
168
230
|
bindings.onDispatchTiming({
|
|
169
231
|
kind: "leg",
|
|
170
|
-
|
|
232
|
+
messageId,
|
|
171
233
|
leg,
|
|
172
234
|
phase: "start",
|
|
173
235
|
atMs,
|
|
@@ -175,11 +237,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
175
237
|
}
|
|
176
238
|
catch (cause) {
|
|
177
239
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
178
|
-
logger.warn `onDispatchTiming leg observer threw for ${
|
|
240
|
+
logger.warn `onDispatchTiming leg observer threw for ${messageId} (${leg} start): ${message}`;
|
|
179
241
|
}
|
|
180
242
|
return atMs;
|
|
181
243
|
}
|
|
182
|
-
function legMarkEnd(
|
|
244
|
+
function legMarkEnd(messageId, leg) {
|
|
183
245
|
const observer = bindings.onDispatchTiming;
|
|
184
246
|
if (observer === undefined)
|
|
185
247
|
return;
|
|
@@ -193,12 +255,12 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
193
255
|
// surface it on the log and emit the end mark without counters so
|
|
194
256
|
// the timing slope is still recoverable.
|
|
195
257
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
196
|
-
logger.warn `structural-counter sample failed for ${
|
|
258
|
+
logger.warn `structural-counter sample failed for ${messageId} (${leg}): ${message}`;
|
|
197
259
|
}
|
|
198
260
|
try {
|
|
199
261
|
observer({
|
|
200
262
|
kind: "leg",
|
|
201
|
-
|
|
263
|
+
messageId,
|
|
202
264
|
leg,
|
|
203
265
|
phase: "end",
|
|
204
266
|
atMs,
|
|
@@ -207,7 +269,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
207
269
|
}
|
|
208
270
|
catch (cause) {
|
|
209
271
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
210
|
-
logger.warn `onDispatchTiming leg observer threw for ${
|
|
272
|
+
logger.warn `onDispatchTiming leg observer threw for ${messageId} (${leg} end): ${message}`;
|
|
211
273
|
}
|
|
212
274
|
}
|
|
213
275
|
// §10c forced-repack A/B (measurement-only). Absent toggle => never
|
|
@@ -244,27 +306,26 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
244
306
|
}
|
|
245
307
|
/**
|
|
246
308
|
* Classify a child-proxied `substrate.write.request` into the D2 leg it
|
|
247
|
-
* represents, plus the
|
|
248
|
-
* `runs/<runId>/events/` is the run-event bracket commit
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
*
|
|
252
|
-
*
|
|
253
|
-
*
|
|
309
|
+
* represents, plus the messageId the per-message OLS fit groups on.
|
|
310
|
+
* `runs/<runId>/events/` is the run-event bracket commit and `agent-state/...`
|
|
311
|
+
* is the D1 conversation WAL append; neither prefix carries the message
|
|
312
|
+
* identity (the run-event prefix names only the stable per-deployment run
|
|
313
|
+
* id), so both are attributed to the dispatch loop's current serial
|
|
314
|
+
* message. Any other prefix is an unmarked proxied write. Returns `null`
|
|
315
|
+
* when no observer is wired (so the supervisor samples nothing), when the
|
|
316
|
+
* prefix is not an attributed leg, or when no message is in flight to
|
|
317
|
+
* attribute it to.
|
|
254
318
|
*/
|
|
255
319
|
function classifyProxiedWriteLeg(preservePrefix) {
|
|
256
320
|
if (bindings.onDispatchTiming === undefined)
|
|
257
321
|
return null;
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
return { leg: "runevent", runId };
|
|
322
|
+
if (currentDispatchMessageId === null)
|
|
323
|
+
return null;
|
|
324
|
+
if (/^runs\/[^/]+\/events\/$/.test(preservePrefix)) {
|
|
325
|
+
return { leg: "runevent", messageId: currentDispatchMessageId };
|
|
263
326
|
}
|
|
264
327
|
if (preservePrefix.startsWith("agent-state/")) {
|
|
265
|
-
|
|
266
|
-
return { leg: "wal", runId: currentDispatchRunId };
|
|
267
|
-
}
|
|
328
|
+
return { leg: "wal", messageId: currentDispatchMessageId };
|
|
268
329
|
}
|
|
269
330
|
return null;
|
|
270
331
|
}
|
|
@@ -286,7 +347,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
286
347
|
const deriveMailAuditRef = bindings.deriveMailAuditRef ?? defaultInProcessMailAuditRef;
|
|
287
348
|
const defaultInboxWritePrincipal = {
|
|
288
349
|
kind: "supervisor",
|
|
289
|
-
|
|
350
|
+
anchorRunId: bindings.anchorRunId,
|
|
290
351
|
};
|
|
291
352
|
const inboxWritePrincipal = bindings.inboxWritePrincipal ?? defaultInboxWritePrincipal;
|
|
292
353
|
// Resolve the consumed-dedup retention horizon once at the bindings
|
|
@@ -302,6 +363,17 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
302
363
|
const readyTimeoutMs = bindings.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS;
|
|
303
364
|
const readySetTimer = bindings.setTimer ?? defaultSetTimer;
|
|
304
365
|
const readyClearTimer = bindings.clearTimer ?? defaultClearTimer;
|
|
366
|
+
// Resolve the crash-loop guard bounds once at the bindings edge (the
|
|
367
|
+
// layer that owns the operator config). The stable-run reset timer
|
|
368
|
+
// reuses the same injectable `setTimer`/`clearTimer` pair as the ready
|
|
369
|
+
// handshake, and the wall clock reuses the recycle-policy `now` seam, so
|
|
370
|
+
// tests drive the whole guard deterministically through existing knobs.
|
|
371
|
+
const crashLoopMaxCount = bindings.crashLoopMaxCount ?? DEFAULT_CRASH_LOOP_MAX_COUNT;
|
|
372
|
+
const crashLoopWindowMs = bindings.crashLoopWindowMs ?? DEFAULT_CRASH_LOOP_WINDOW_MS;
|
|
373
|
+
const crashLoopStableResetMs = bindings.crashLoopStableResetMs ?? DEFAULT_CRASH_LOOP_STABLE_RESET_MS;
|
|
374
|
+
const respawnBackoffInitialMs = bindings.respawnBackoffInitialMs ?? DEFAULT_RESPAWN_BACKOFF_INITIAL_MS;
|
|
375
|
+
const respawnBackoffMaxMs = bindings.respawnBackoffMaxMs ?? DEFAULT_RESPAWN_BACKOFF_MAX_MS;
|
|
376
|
+
const crashNow = bindings.recyclePolicyNow ?? defaultNow;
|
|
305
377
|
/**
|
|
306
378
|
* Resolved on every successful `enqueueInbox`; the dispatch loop
|
|
307
379
|
* awaits this promise after a null dequeue so it returns to
|
|
@@ -332,12 +404,321 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
332
404
|
*/
|
|
333
405
|
let spawnContext = null;
|
|
334
406
|
let recyclePolicy = null;
|
|
335
|
-
|
|
407
|
+
// Mutual-exclusion latch shared by every respawn path (operator/policy/
|
|
408
|
+
// self recycle and crash-respawn). `runRespawn` owns the set/clear; each
|
|
409
|
+
// caller owns the contention read because the two paths diverge on
|
|
410
|
+
// contention (recycle throws, crash-respawn declines silently).
|
|
411
|
+
let respawnInProgress = false;
|
|
412
|
+
// Monotonic child-cohort generation, bumped atomically with each
|
|
413
|
+
// transition to `running` (initial spawn and every respawn's
|
|
414
|
+
// `installNewChild`). The exit-watcher captures the generation it was
|
|
415
|
+
// armed for; a watcher whose generation is no longer current is stale
|
|
416
|
+
// (a newer child already superseded it) and is ignored. Generation plus
|
|
417
|
+
// the phase guard plus `respawnInProgress` together classify every child
|
|
418
|
+
// exit as planned or unexpected without a separate per-handle marker.
|
|
419
|
+
let childGeneration = 0;
|
|
420
|
+
// An unexpected child exit that could not be handled the instant it was
|
|
421
|
+
// observed (a respawn was already in flight). Recorded generation-tagged
|
|
422
|
+
// and drained by `maybeHandleChildExit` once `runRespawn` clears the
|
|
423
|
+
// latch; a pending exit for a superseded generation is dropped as stale.
|
|
424
|
+
let pendingChildExit = null;
|
|
425
|
+
// Monotonic timestamps (ms, from `crashNow`) of recent unexpected exits,
|
|
426
|
+
// pruned to `crashLoopWindowMs`. The crash-loop guard latches when the
|
|
427
|
+
// count within the window reaches `crashLoopMaxCount`.
|
|
428
|
+
const crashTimestamps = [];
|
|
429
|
+
// Handle for the armed stable-run reset timer (or null). Cleared and
|
|
430
|
+
// re-armed on every transition to `running`; cleared on teardown.
|
|
431
|
+
let stableRunResetTimer = null;
|
|
432
|
+
// Current exponential respawn backoff (ms). Advances by doubling on each
|
|
433
|
+
// respawn (capped at `respawnBackoffMaxMs`) and resets to the initial
|
|
434
|
+
// value when a respawned child runs stably. See `waitRespawnBackoff`.
|
|
435
|
+
let respawnBackoffMs = respawnBackoffInitialMs;
|
|
436
|
+
// Every armed backoff wait: the injected timer handle plus the promise
|
|
437
|
+
// resolver, so a shutdown can cancel all of them and unblock the parked
|
|
438
|
+
// respawn coroutines (each then bails at its phase/generation re-check).
|
|
439
|
+
// A SET, not a single slot: more than one coroutine can be parked at once
|
|
440
|
+
// -- a recycle can install a fresh, live child DURING one crash's backoff
|
|
441
|
+
// wait, and that child crashing parks a second coroutine. A single slot
|
|
442
|
+
// would drop the earlier one's timer and leak it past shutdown.
|
|
443
|
+
const respawnBackoffWaits = new Set();
|
|
444
|
+
// =====================================================================
|
|
445
|
+
// CRASH-RESPAWN -- auto-recovery from an unexpected workflow-process exit
|
|
446
|
+
// =====================================================================
|
|
447
|
+
//
|
|
448
|
+
// An UNEXPECTED child exit (crash, OOM, panic, signal -- not a
|
|
449
|
+
// supervisor-initiated shutdown, drain, or recycle) is detected by
|
|
450
|
+
// watching `handle.exited`, the only universal death signal: a clean
|
|
451
|
+
// process death ends the IPC channel readers without a protocol-level
|
|
452
|
+
// crash callback (see `onChildCrash`). `armChildForRunning` arms that
|
|
453
|
+
// watcher and bumps `childGeneration` on every transition to `running`;
|
|
454
|
+
// an exit is classified as unexpected iff its generation is still current
|
|
455
|
+
// AND the phase is still `running` (a planned kill is observed in a
|
|
456
|
+
// non-running phase, or against a superseded generation).
|
|
457
|
+
//
|
|
458
|
+
// On an unexpected exit `handleUnexpectedChildExit` runs: it records the
|
|
459
|
+
// crash against the crash-loop guard and either
|
|
460
|
+
// - RESPAWNS -- after an exponential backoff (`waitRespawnBackoff`,
|
|
461
|
+
// 1s doubling to a 30s cap), it reuses the recycle path's
|
|
462
|
+
// `runRespawn` with a no-op drain (the child is already dead) so the
|
|
463
|
+
// stranded-mail replay and fresh-child spawn happen exactly as a
|
|
464
|
+
// recycle's do; or
|
|
465
|
+
// - LATCHES -- once the guard trips (`crashLoopMaxCount` exits within
|
|
466
|
+
// `crashLoopWindowMs`), it tears down to the terminal `crash-looping`
|
|
467
|
+
// phase and commits a `RunFailed` for the deployment's run so the
|
|
468
|
+
// crash-loop is durably observable as a failed run status.
|
|
469
|
+
//
|
|
470
|
+
// A respawned child that survives `crashLoopStableResetMs`
|
|
471
|
+
// (`armStableRunResetTimer`) resets both the crash counter and the
|
|
472
|
+
// backoff. The backoff wait sits OUTSIDE the `respawnInProgress` latch,
|
|
473
|
+
// so the generation re-check after the wait is what stops a recycle that
|
|
474
|
+
// installed a fresh cohort mid-wait from being respawned a second time.
|
|
475
|
+
// Full policy: `packages/workflow-host/README.md` "Respawn policy".
|
|
476
|
+
// A protocol violation on a live cohort's control or event channel. The
|
|
477
|
+
// channel receiver ends its iterator and invokes this; a clean process
|
|
478
|
+
// death does NOT (it just ends the reader with no crash callback), so
|
|
479
|
+
// the exit-watcher on `handle.exited` is the universal death signal and
|
|
480
|
+
// this path only covers the frame-level violations the receiver detects.
|
|
481
|
+
// On the live (`running`) cohort, force the child down so its `exited`
|
|
482
|
+
// fires the exit-watcher and the crash flows through the SAME
|
|
483
|
+
// respawn/crash-loop path a clean death takes -- a violation that emits a
|
|
484
|
+
// garbage frame just before dying must not divert to `stopped` while a
|
|
485
|
+
// clean SIGKILL respawns. In any other phase the owning lifecycle path
|
|
486
|
+
// (spawn handshake, recycle reap, shutdown) owns teardown.
|
|
336
487
|
function onChildCrash(reason) {
|
|
337
|
-
|
|
488
|
+
if (state.phase === "running") {
|
|
489
|
+
logger.error `workflow-process channel crash on live cohort; forcing child down to respawn: {reason}`;
|
|
490
|
+
state.handle.kill();
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
logger.error `workflow-process channel crash: {reason}`;
|
|
338
494
|
void shutdownInternal({ reason });
|
|
339
495
|
}
|
|
340
|
-
|
|
496
|
+
// Prune crash timestamps older than the sliding window relative to `nowMs`.
|
|
497
|
+
function pruneCrashTimestamps(nowMs) {
|
|
498
|
+
const cutoff = nowMs - crashLoopWindowMs;
|
|
499
|
+
while (true) {
|
|
500
|
+
const oldest = crashTimestamps[0];
|
|
501
|
+
if (oldest === undefined || oldest > cutoff)
|
|
502
|
+
break;
|
|
503
|
+
crashTimestamps.shift();
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
function clearStableRunResetTimer() {
|
|
507
|
+
if (stableRunResetTimer !== null) {
|
|
508
|
+
readyClearTimer(stableRunResetTimer);
|
|
509
|
+
stableRunResetTimer = null;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
// Arm (or re-arm) the stable-run reset for the cohort that just reached
|
|
513
|
+
// `running`. If the child is still that same live cohort when the timer
|
|
514
|
+
// fires, the run has been stable for `crashLoopStableResetMs` and the
|
|
515
|
+
// crash counter is cleared so a flap-then-stabilize sequence does not
|
|
516
|
+
// latch. A crash before the timer fires re-arms it for the next cohort;
|
|
517
|
+
// teardown clears it.
|
|
518
|
+
function armStableRunResetTimer(generation) {
|
|
519
|
+
clearStableRunResetTimer();
|
|
520
|
+
stableRunResetTimer = readySetTimer(() => {
|
|
521
|
+
stableRunResetTimer = null;
|
|
522
|
+
if (generation === childGeneration && state.phase === "running") {
|
|
523
|
+
// The cohort ran stably: reset both the crash counter and the
|
|
524
|
+
// exponential backoff so a flap followed by stability starts over.
|
|
525
|
+
crashTimestamps.length = 0;
|
|
526
|
+
respawnBackoffMs = respawnBackoffInitialMs;
|
|
527
|
+
}
|
|
528
|
+
}, crashLoopStableResetMs);
|
|
529
|
+
}
|
|
530
|
+
// Wait the current respawn backoff before a respawn. The wait uses the
|
|
531
|
+
// injected timer so tests drive it deterministically, and is cancellable:
|
|
532
|
+
// `cancelRespawnBackoffWaits` (called on shutdown) clears the timer and
|
|
533
|
+
// resolves the promise early so the parked respawn coroutine unblocks and
|
|
534
|
+
// bails at its phase/generation re-check rather than sleeping out a full
|
|
535
|
+
// 30s backoff against a torn-down supervisor.
|
|
536
|
+
function waitRespawnBackoff(ms) {
|
|
537
|
+
return new Promise((resolve) => {
|
|
538
|
+
const entry = {
|
|
539
|
+
timer: null,
|
|
540
|
+
resolve,
|
|
541
|
+
};
|
|
542
|
+
entry.timer = readySetTimer(() => {
|
|
543
|
+
respawnBackoffWaits.delete(entry);
|
|
544
|
+
resolve();
|
|
545
|
+
}, ms);
|
|
546
|
+
respawnBackoffWaits.add(entry);
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
// Cancel every armed backoff wait. Idempotent: an empty set is a no-op,
|
|
550
|
+
// and each entry's own fire path has already removed it.
|
|
551
|
+
function cancelRespawnBackoffWaits() {
|
|
552
|
+
for (const entry of respawnBackoffWaits) {
|
|
553
|
+
readyClearTimer(entry.timer);
|
|
554
|
+
entry.resolve();
|
|
555
|
+
}
|
|
556
|
+
respawnBackoffWaits.clear();
|
|
557
|
+
}
|
|
558
|
+
// Bump the generation and arm the `handle.exited` watcher for a child
|
|
559
|
+
// that just became the `running` cohort. Called atomically with the
|
|
560
|
+
// state swap to `running` (no await between the swap and this call).
|
|
561
|
+
// `exited` resolving OR rejecting both mean the process is gone. The
|
|
562
|
+
// stable-run reset timer is NOT armed here: on the pristine first spawn
|
|
563
|
+
// there is no crash counter to reset, so it is armed only after a
|
|
564
|
+
// respawn (see `handleUnexpectedChildExit`).
|
|
565
|
+
function armChildForRunning(handle) {
|
|
566
|
+
childGeneration += 1;
|
|
567
|
+
const generation = childGeneration;
|
|
568
|
+
void handle.exited
|
|
569
|
+
.then(() => {
|
|
570
|
+
onChildExited(generation, "workflow-process child exited");
|
|
571
|
+
})
|
|
572
|
+
.catch(() => {
|
|
573
|
+
onChildExited(generation, "workflow-process child exited (exit promise rejected)");
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
// Record a child exit and try to act on it. Stale exits (a newer cohort
|
|
577
|
+
// already installed) are dropped.
|
|
578
|
+
function onChildExited(generation, reason) {
|
|
579
|
+
if (generation !== childGeneration)
|
|
580
|
+
return;
|
|
581
|
+
pendingChildExit = { generation, reason };
|
|
582
|
+
maybeHandleChildExit();
|
|
583
|
+
}
|
|
584
|
+
// Drain a recorded child exit if the supervisor is in a state to act on
|
|
585
|
+
// it. Declines (leaving the exit pending) while a respawn is in flight;
|
|
586
|
+
// `runRespawn`'s `finally` re-invokes this after clearing the latch so
|
|
587
|
+
// an exit observed during the respawn is not lost. A pending exit for a
|
|
588
|
+
// superseded generation, or one observed after the deployment left the
|
|
589
|
+
// `running` phase (shutdown/recycle owns that teardown), is dropped.
|
|
590
|
+
function maybeHandleChildExit() {
|
|
591
|
+
if (respawnInProgress)
|
|
592
|
+
return;
|
|
593
|
+
const pending = pendingChildExit;
|
|
594
|
+
if (pending === null)
|
|
595
|
+
return;
|
|
596
|
+
if (pending.generation !== childGeneration) {
|
|
597
|
+
pendingChildExit = null;
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
if (state.phase !== "running")
|
|
601
|
+
return;
|
|
602
|
+
pendingChildExit = null;
|
|
603
|
+
void handleUnexpectedChildExit(pending.reason).catch((cause) => {
|
|
604
|
+
// Fire-and-forget context (the exit-watcher has no caller to catch
|
|
605
|
+
// this). `runRespawn` already ran its own failure teardown to a
|
|
606
|
+
// terminal state before rethrowing, so the deployment is not wedged;
|
|
607
|
+
// surface the failure and stop.
|
|
608
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
609
|
+
logger.error `crash-respawn handling failed; deployment torn down: ${message}`;
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
// Handle one unexpected child exit: record it against the crash-loop
|
|
613
|
+
// guard and either latch the deployment (guard tripped) or respawn.
|
|
614
|
+
async function handleUnexpectedChildExit(reason) {
|
|
615
|
+
if (state.phase !== "running" || spawnContext === null) {
|
|
616
|
+
// Raced a shutdown/recycle between the drain check and here; the
|
|
617
|
+
// owning lifecycle path handles teardown.
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
620
|
+
// A cohort that just crashed did not earn a stability reward: disarm its
|
|
621
|
+
// stable-run reset timer up front so it cannot fire during the backoff
|
|
622
|
+
// wait below (where the phase is still `running` and the generation is
|
|
623
|
+
// still this dead cohort's -- exactly the state the timer's own guard
|
|
624
|
+
// reads as "alive and stable") and wrongly clear the crash counter.
|
|
625
|
+
clearStableRunResetTimer();
|
|
626
|
+
// Capture the crashing cohort's generation. If a recycle or policy
|
|
627
|
+
// respawn installs a fresh cohort while the backoff wait below is
|
|
628
|
+
// parked, the generation advances and this handler must NOT respawn
|
|
629
|
+
// that healthy cohort -- the post-wait guard bails on the mismatch.
|
|
630
|
+
const armedGeneration = childGeneration;
|
|
631
|
+
const nowMs = crashNow();
|
|
632
|
+
crashTimestamps.push(nowMs);
|
|
633
|
+
pruneCrashTimestamps(nowMs);
|
|
634
|
+
if (crashTimestamps.length >= crashLoopMaxCount) {
|
|
635
|
+
// Crash-loop latch. The deployment stops respawning and tears down
|
|
636
|
+
// to the terminal `crash-looping` state so a rapidly-flapping child
|
|
637
|
+
// cannot saturate the host.
|
|
638
|
+
const crashCount = crashTimestamps.length;
|
|
639
|
+
logger.error `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms; stopping the deployment (${reason})`;
|
|
640
|
+
await shutdownInternal({
|
|
641
|
+
reason: `crash-loop: ${reason}`,
|
|
642
|
+
terminalPhase: "crash-looping",
|
|
643
|
+
});
|
|
644
|
+
// Commit the RunFailed tombstone AFTER teardown: shutdownInternal has
|
|
645
|
+
// quiesced the drain accumulators (stop + await disposed), so the
|
|
646
|
+
// run-event tree is settled and no escalation commit races this write.
|
|
647
|
+
// This RunFailed is the SOLE durable, externally-queryable signal of
|
|
648
|
+
// the crash-loop (the `crash-looping` phase is in-memory only), so a
|
|
649
|
+
// failure to write it is logged loudly rather than swallowed. Best-
|
|
650
|
+
// effort: the deployment is already terminal, so the write not landing
|
|
651
|
+
// costs observability, not correctness.
|
|
652
|
+
try {
|
|
653
|
+
// `anchorRunId` and the tombstone's `runId` are DISTINCT ids and must
|
|
654
|
+
// not be conflated. `bindings.anchorRunId` is the workflow-run repo
|
|
655
|
+
// slug (`deriveWorkflowRunRepoId`), which the supervisor principal's
|
|
656
|
+
// authz check keys on (`repoId.id === anchorRunId`). The RunFailed must
|
|
657
|
+
// land on the deployment's ONE top-level run, whose id is the local
|
|
658
|
+
// part of the deployment's mail address (`deriveWorkflowRunId`) -- the
|
|
659
|
+
// same id the dispatch loop writes every run event under. For a domain
|
|
660
|
+
// like `integration.interchange` the two ids differ (the repo slug
|
|
661
|
+
// carries a domain suffix), so writing the tombstone under the repo
|
|
662
|
+
// slug would strand it in a run subtree no reader consults.
|
|
663
|
+
await commitRunFailed({
|
|
664
|
+
substrate: bindings.repoStore,
|
|
665
|
+
repoId: bindings.workflowRunRepoId,
|
|
666
|
+
ref: bindings.workflowRunRef,
|
|
667
|
+
anchorRunId: bindings.anchorRunId,
|
|
668
|
+
runId: deriveWorkflowRunId(bindings.deploymentMailAddress),
|
|
669
|
+
at: new Date(nowMs).toISOString(),
|
|
670
|
+
message: `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms`,
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
catch (cause) {
|
|
674
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
675
|
+
logger.error `crash-loop RunFailed commit failed; deployment has no durable failure tombstone: ${message}`;
|
|
676
|
+
}
|
|
677
|
+
return;
|
|
678
|
+
}
|
|
679
|
+
const thisBackoffMs = respawnBackoffMs;
|
|
680
|
+
logger.warn `workflow-process exited unexpectedly; respawning after ${String(thisBackoffMs)}ms backoff (${reason})`;
|
|
681
|
+
await waitRespawnBackoff(thisBackoffMs);
|
|
682
|
+
// A recycle/policy respawn, or a shutdown, may have run during the
|
|
683
|
+
// backoff wait. Bail unless THIS dead cohort is still the current
|
|
684
|
+
// running one: the generation guard prevents respawning a healthy
|
|
685
|
+
// cohort a recycle just installed, and the phase guard prevents acting
|
|
686
|
+
// after teardown. No await separates this re-check from `runRespawn`'s
|
|
687
|
+
// synchronous `respawnInProgress` set, so nothing can interleave.
|
|
688
|
+
if (childGeneration !== armedGeneration ||
|
|
689
|
+
state.phase !== "running" ||
|
|
690
|
+
spawnContext === null) {
|
|
691
|
+
return;
|
|
692
|
+
}
|
|
693
|
+
const priorRunning = state;
|
|
694
|
+
const priorContext = spawnContext;
|
|
695
|
+
// Advance the backoff only now that a respawn is actually happening, so
|
|
696
|
+
// a bail above does not inflate the wait for a respawn that never ran.
|
|
697
|
+
respawnBackoffMs = Math.min(respawnBackoffMs * 2, respawnBackoffMaxMs);
|
|
698
|
+
await runRespawn({
|
|
699
|
+
origin: "crash",
|
|
700
|
+
reason,
|
|
701
|
+
prior: priorRunning,
|
|
702
|
+
priorContext,
|
|
703
|
+
// The child is already dead: there is nothing to drain, and
|
|
704
|
+
// `runRespawn`'s kill step is a no-op on a dead handle. The replay
|
|
705
|
+
// step still runs, moving any mail stranded mid-flight back to the
|
|
706
|
+
// inbox tail before dispatch resumes.
|
|
707
|
+
drain: async () => undefined,
|
|
708
|
+
});
|
|
709
|
+
// The respawned child is now the running cohort. Arm the stable-run
|
|
710
|
+
// reset against its generation: if it survives `crashLoopStableResetMs`
|
|
711
|
+
// the crash counter and backoff reset, so a flap followed by stability
|
|
712
|
+
// does not latch on a later, unrelated crash.
|
|
713
|
+
armStableRunResetTimer(childGeneration);
|
|
714
|
+
}
|
|
715
|
+
// Resolves once the inbound mail is durably accepted (its inbox write landed
|
|
716
|
+
// or the message was already durably present); rejects when it was not (a
|
|
717
|
+
// phase where the deployment is not accepting mail, a transient enqueue
|
|
718
|
+
// failure, or a stale refusal). The host propagates that settlement to the
|
|
719
|
+
// wire, so resolution is the durable-receipt ACK signal and rejection is the
|
|
720
|
+
// WITHHOLD signal -- a withheld message is redelivered by the hub.
|
|
721
|
+
async function onMailMessage(rawMessage) {
|
|
341
722
|
// Every inbound mail flows through the FIFO inbox claim-check
|
|
342
723
|
// queue, regardless of the supervisor's current phase. The
|
|
343
724
|
// dispatch loop (started by `spawn()` and restarted by the
|
|
@@ -349,16 +730,33 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
349
730
|
// the envelope's `receivedAt` prefix on the inbox filename.
|
|
350
731
|
if (state.phase === "idle" ||
|
|
351
732
|
state.phase === "stopping" ||
|
|
352
|
-
state.phase === "stopped"
|
|
353
|
-
|
|
354
|
-
//
|
|
355
|
-
//
|
|
356
|
-
|
|
733
|
+
state.phase === "stopped" ||
|
|
734
|
+
state.phase === "crash-looping") {
|
|
735
|
+
// The host's higher-level lifecycle is already tearing the deployment
|
|
736
|
+
// down; nothing is enqueued. Reject rather than silently drop so the
|
|
737
|
+
// ack is WITHHELD and the hub redelivers -- a later generation (or a
|
|
738
|
+
// recycle-installed dispatch loop) may accept it, and a permanently
|
|
739
|
+
// torn-down address exhausts the hub's bounded retry budget instead of
|
|
740
|
+
// losing a message a transiently-idle deployment would have taken.
|
|
741
|
+
throw new Error(`inbound mail not accepted: supervisor phase is "${state.phase}"`);
|
|
742
|
+
}
|
|
743
|
+
try {
|
|
744
|
+
await enqueueInboundMail(rawMessage);
|
|
745
|
+
}
|
|
746
|
+
catch (cause) {
|
|
747
|
+
// Both branches WITHHOLD (rethrow); the split only sets log severity so
|
|
748
|
+
// a stale refusal surfaces as its own loud signal rather than blending
|
|
749
|
+
// into ordinary enqueue-failure noise. The ack/withhold decision is the
|
|
750
|
+
// rethrow itself, never this classification.
|
|
751
|
+
if (cause instanceof StaleInboxEnqueueError) {
|
|
752
|
+
logger.error `inbound mail refused as stale, withholding ack (hub will redeliver): ${cause.message}`;
|
|
753
|
+
}
|
|
754
|
+
else {
|
|
755
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
756
|
+
logger.error `enqueueInbox failed, withholding ack (hub will redeliver): ${message}`;
|
|
757
|
+
}
|
|
758
|
+
throw cause;
|
|
357
759
|
}
|
|
358
|
-
void enqueueInboundMail(rawMessage).catch((cause) => {
|
|
359
|
-
const message = cause instanceof Error ? cause.message : String(cause);
|
|
360
|
-
logger.error `enqueueInbox failed: ${message}`;
|
|
361
|
-
});
|
|
362
760
|
}
|
|
363
761
|
async function enqueueInboundMail(rawMessage) {
|
|
364
762
|
const messageId = await deriveMessageId(rawMessage);
|
|
@@ -373,11 +771,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
373
771
|
const rawMessageBase64 = base64Encode(rawMessage);
|
|
374
772
|
// D2 leg: `enqueueInbox` runs in `onMailMessage` BEFORE dispatch, so
|
|
375
773
|
// it is paid OUTSIDE the dispatch-start..reply-produced window -- its
|
|
376
|
-
// growth is invisible to the 4.7 bracket.
|
|
377
|
-
//
|
|
378
|
-
//
|
|
774
|
+
// growth is invisible to the 4.7 bracket. This leg mark is keyed by the
|
|
775
|
+
// messageId, the same per-message key every in-window leg uses, so the
|
|
776
|
+
// D2 per-message OLS fit groups the enqueue leg with the rest.
|
|
379
777
|
legMarkStart(messageId, "enqueue");
|
|
380
|
-
await inboxPrimitives.enqueueInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
|
|
778
|
+
const outcome = await inboxPrimitives.enqueueInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
|
|
381
779
|
address: bindings.deploymentMailAddress,
|
|
382
780
|
messageId,
|
|
383
781
|
receivedAt,
|
|
@@ -385,7 +783,20 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
385
783
|
rawMessage: rawMessageBase64,
|
|
386
784
|
});
|
|
387
785
|
legMarkEnd(messageId, "enqueue");
|
|
388
|
-
|
|
786
|
+
// Only a fresh enqueue added a new inbox entry; wake the dispatch loop for
|
|
787
|
+
// it alone. An `already-present` outcome landed nothing new -- returning
|
|
788
|
+
// (which acks) without waking is correct, since the earlier delivery of
|
|
789
|
+
// the same messageId already drives dispatch. This resolves for both
|
|
790
|
+
// outcomes: both mean the bytes are durably accounted for, so both ack.
|
|
791
|
+
if (outcome.outcome === "enqueued") {
|
|
792
|
+
wakeDispatch();
|
|
793
|
+
}
|
|
794
|
+
else {
|
|
795
|
+
// A redelivery of a message already durably present: the ack still
|
|
796
|
+
// fires (it is on disk), but no new run is dispatched. Surface it so an
|
|
797
|
+
// at-least-once redelivery being made effectively-once is observable.
|
|
798
|
+
logger.info `inbound mail ${messageId} already durably present (${outcome.reason}); acknowledging without re-dispatch`;
|
|
799
|
+
}
|
|
389
800
|
}
|
|
390
801
|
/**
|
|
391
802
|
* Pump child-initiated upstream control frames after `ready` has
|
|
@@ -456,16 +867,98 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
456
867
|
// to-one with its cohort's `controlIncoming` iterator: a
|
|
457
868
|
// buffered `terminal.event` the OLD child emitted before kill
|
|
458
869
|
// landed must NEVER route to the NEW cohort's broadcaster.
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
// (the
|
|
462
|
-
//
|
|
870
|
+
// Every run shares the stable runId (the deployment mail
|
|
871
|
+
// address), so an OLD-cohort frame and a NEW-cohort run collide
|
|
872
|
+
// on that id by construction (the recycle/replay case); without
|
|
873
|
+
// this binding the stale frame would falsely settle the NEW
|
|
874
|
+
// cohort's `waitForRunTerminalOrPark` and commit `markConsumed`
|
|
463
875
|
// on a run still in flight. The broadcaster's own `dispose()`
|
|
464
876
|
// on cohort teardown turns post-dispose notify into a no-op,
|
|
465
877
|
// so a stale frame dequeued after the cohort was torn down
|
|
466
878
|
// drops cleanly without leaking into any successor cohort.
|
|
467
879
|
const event = terminalEventFromPayload(payload.data);
|
|
468
880
|
cohortBroadcaster.notify(payload.data.runId, event);
|
|
881
|
+
terminalRunIds.add(payload.data.runId);
|
|
882
|
+
// Clean up cohort tracking for the terminated run. Self-discovered
|
|
883
|
+
// runs have no dispatch-loop entry, so their cleanup happens here.
|
|
884
|
+
cohortRunIds.delete(payload.data.runId);
|
|
885
|
+
runInputChannels.delete(payload.data.runId);
|
|
886
|
+
continue;
|
|
887
|
+
}
|
|
888
|
+
if (payload.type === "park.notify") {
|
|
889
|
+
// The workflow-process child reported a control-plane suspension: an
|
|
890
|
+
// agent step parked on a reserved `signalName(correlationId)` channel.
|
|
891
|
+
if (payload.data.parkKind === "input") {
|
|
892
|
+
// Input parks are owned by the supervisor, not the hub. Register
|
|
893
|
+
// cohort membership BEFORE caching the correlationId so a run is
|
|
894
|
+
// never a channel-without-cohort entry (the routing-hygiene
|
|
895
|
+
// invariant); a live run that parked is always in cohortRunIds
|
|
896
|
+
// already, so this is idempotent belt-and-suspenders. Cache the
|
|
897
|
+
// correlationId so the dispatch loop can fire signal.deliver on
|
|
898
|
+
// subsequent mail without a substrate round-trip.
|
|
899
|
+
cohortRunIds.add(payload.data.runId);
|
|
900
|
+
runInputChannels.set(payload.data.runId, {
|
|
901
|
+
correlationId: payload.data.correlationId,
|
|
902
|
+
parkKind: "input",
|
|
903
|
+
});
|
|
904
|
+
// Stop any drain accumulator for a run that has parked. Input parks
|
|
905
|
+
// own this because a drain arms no accumulator for a run that already
|
|
906
|
+
// holds an input channel; an approval park never carries one, so its
|
|
907
|
+
// drain interaction is a separate concern and stays out of this arm.
|
|
908
|
+
const accumulator = drainAccumulators.get(payload.data.runId);
|
|
909
|
+
if (accumulator !== undefined) {
|
|
910
|
+
accumulator.stop();
|
|
911
|
+
drainAccumulators.delete(payload.data.runId);
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
else if (payload.data.parkKind === "approval") {
|
|
915
|
+
// Approval parks are hub-registered through the shared
|
|
916
|
+
// `registerSuspension` transform.
|
|
917
|
+
registerSuspension({
|
|
918
|
+
runId: payload.data.runId,
|
|
919
|
+
correlationId: payload.data.correlationId,
|
|
920
|
+
parkKind: "approval",
|
|
921
|
+
...(payload.data.snapshot !== undefined
|
|
922
|
+
? { snapshot: payload.data.snapshot }
|
|
923
|
+
: {}),
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
else {
|
|
927
|
+
// A `signal-relay` park is relayed down into the body child by the
|
|
928
|
+
// section runtime and is never hub-registered, so it does not ride
|
|
929
|
+
// `park.notify`. One arriving here is a protocol violation; log and
|
|
930
|
+
// drop rather than mis-registering it as an approval.
|
|
931
|
+
logger.error `park.notify for run ${payload.data.runId} carried parkKind=${payload.data.parkKind}, which is not a hub-registered kind; dropping`;
|
|
932
|
+
}
|
|
933
|
+
// A park of ANY kind suspends the run, so a dispatch loop waiting on
|
|
934
|
+
// `waitForRunTerminalOrPark` after firing the trigger (or delivering
|
|
935
|
+
// the last signal) must be released here regardless of park kind. An
|
|
936
|
+
// approval park that only registered its suspension would leave that
|
|
937
|
+
// loop hanging to the terminal-or-park backstop. Bump the park
|
|
938
|
+
// generation BEFORE resolving the waiter: a dispatch loop that captured
|
|
939
|
+
// `sinceGen` for this run must see the newer generation both when its
|
|
940
|
+
// armed waiter fires here and when it re-reads the generation after
|
|
941
|
+
// arming (the check-after-register).
|
|
942
|
+
parkGenerations.set(payload.data.runId, (parkGenerations.get(payload.data.runId) ?? 0) + 1);
|
|
943
|
+
// Wake any dispatch loop waiting for this run to park.
|
|
944
|
+
resolveParkNotifyWaiter(payload.data.runId);
|
|
945
|
+
continue;
|
|
946
|
+
}
|
|
947
|
+
if (payload.type === "parked-correlations.response") {
|
|
948
|
+
// The child answered a `reEmitParkedCorrelations` query. Resolve the
|
|
949
|
+
// awaiting driver; a response with no pending entry (the query already
|
|
950
|
+
// timed out and dropped it) is logged and dropped, never thrown, so it
|
|
951
|
+
// cannot tear the pump down.
|
|
952
|
+
resolveParkedResponse(payload.data);
|
|
953
|
+
continue;
|
|
954
|
+
}
|
|
955
|
+
if (payload.type === "resumed.runs") {
|
|
956
|
+
// The child self-discovered runs from the substrate after reconnect
|
|
957
|
+
// or recycle. Seed cohort tracking so drain accumulators and dispatch
|
|
958
|
+
// routing account for runs the supervisor did not personally fire.
|
|
959
|
+
for (const runId of payload.data.runIds) {
|
|
960
|
+
cohortRunIds.add(runId);
|
|
961
|
+
}
|
|
469
962
|
continue;
|
|
470
963
|
}
|
|
471
964
|
logger.warn `workflow-process upstream control payload ignored: type=${payload.type}`;
|
|
@@ -473,22 +966,26 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
473
966
|
}
|
|
474
967
|
const pendingMerges = new Map();
|
|
475
968
|
/**
|
|
476
|
-
* Reject every pending merge round-trip and every
|
|
477
|
-
*
|
|
478
|
-
*
|
|
479
|
-
*
|
|
480
|
-
*
|
|
481
|
-
*
|
|
482
|
-
*
|
|
969
|
+
* Reject every pending merge round-trip and every park-notify
|
|
970
|
+
* waiter. Invoked on cohort transitions (shutdown, recycle's
|
|
971
|
+
* `installNewChild`) so closures awaiting these promises do not
|
|
972
|
+
* outlive the cohort that armed them. Without this, a
|
|
973
|
+
* `handleSubstrateWriteRequest` mid-merge or a dispatch loop
|
|
974
|
+
* waiting for park would sit on a resolver that the dying control
|
|
975
|
+
* channel will never invoke.
|
|
483
976
|
*/
|
|
484
977
|
function rejectCohortAwaiters(reason) {
|
|
485
978
|
for (const [requestId, entry] of pendingMerges) {
|
|
486
979
|
pendingMerges.delete(requestId);
|
|
487
980
|
entry.resolve({ ok: false, reason: `cohort aborted: ${reason}` });
|
|
488
981
|
}
|
|
489
|
-
for (const [
|
|
490
|
-
|
|
491
|
-
|
|
982
|
+
for (const [requestId, entry] of pendingParkedQueries) {
|
|
983
|
+
pendingParkedQueries.delete(requestId);
|
|
984
|
+
entry.settle(null);
|
|
985
|
+
}
|
|
986
|
+
for (const [runId, resolve] of parkNotifyWaiters.entries()) {
|
|
987
|
+
parkNotifyWaiters.delete(runId);
|
|
988
|
+
resolve();
|
|
492
989
|
}
|
|
493
990
|
}
|
|
494
991
|
function resolveMergeResponse(data) {
|
|
@@ -525,6 +1022,125 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
525
1022
|
}
|
|
526
1023
|
entry.resolve({ ok: false, reason: data.result.reason });
|
|
527
1024
|
}
|
|
1025
|
+
// Stamp the deployment identity the supervisor owns onto a child-supplied
|
|
1026
|
+
// park and hand it to the host's suspension-register sink. Shared by the
|
|
1027
|
+
// `park.notify` arm (the happy-path emit) and `reEmitParkedCorrelations`
|
|
1028
|
+
// (the re-establishment re-emit). Best-effort: a throwing sink is logged,
|
|
1029
|
+
// not rethrown, so it cannot tear the upstream pump down or abort a re-emit
|
|
1030
|
+
// partway through the parked set. The sink (production: the sidecar) turns
|
|
1031
|
+
// the stamped registration into a `signal.correlation.register` frame the
|
|
1032
|
+
// hub co-writes the run's routing + approval rows from.
|
|
1033
|
+
function registerSuspension(park) {
|
|
1034
|
+
if (bindings.onSuspensionRegister === undefined) {
|
|
1035
|
+
logger.warn `suspension register for runId=${park.runId} but no onSuspensionRegister sink is wired; correlation ${park.correlationId} not registered`;
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
try {
|
|
1039
|
+
bindings.onSuspensionRegister({
|
|
1040
|
+
runId: park.runId,
|
|
1041
|
+
correlationId: park.correlationId,
|
|
1042
|
+
kind: park.parkKind,
|
|
1043
|
+
anchorRunId: bindings.anchorRunId,
|
|
1044
|
+
agentAddress: bindings.deploymentMailAddress,
|
|
1045
|
+
...(park.snapshot !== undefined
|
|
1046
|
+
? { approvalSnapshot: park.snapshot }
|
|
1047
|
+
: {}),
|
|
1048
|
+
});
|
|
1049
|
+
}
|
|
1050
|
+
catch (cause) {
|
|
1051
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1052
|
+
logger.error `onSuspensionRegister sink threw for runId=${park.runId} correlationId=${park.correlationId}: ${message}`;
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
const pendingParkedQueries = new Map();
|
|
1056
|
+
let parkedQuerySeq = 0;
|
|
1057
|
+
function resolveParkedResponse(data) {
|
|
1058
|
+
const entry = pendingParkedQueries.get(data.requestId);
|
|
1059
|
+
if (entry === undefined) {
|
|
1060
|
+
logger.warn `parked-correlations.response landed with no pending entry; requestId=${data.requestId} dropped`;
|
|
1061
|
+
return;
|
|
1062
|
+
}
|
|
1063
|
+
pendingParkedQueries.delete(data.requestId);
|
|
1064
|
+
entry.settle(data.parked);
|
|
1065
|
+
}
|
|
1066
|
+
async function reEmitParkedCorrelations() {
|
|
1067
|
+
// No-op (NOT throw) when the child is not addressable. `deliverSignal`
|
|
1068
|
+
// throws on non-running/starting (including recycling) because it points a
|
|
1069
|
+
// write at the dying child's closing pipe and wants the caller to retry;
|
|
1070
|
+
// this driver's contract is the opposite -- the next re-establishment
|
|
1071
|
+
// re-drives it -- so skipping recycling and letting the next spawn's
|
|
1072
|
+
// re-emit cover it is correct here, not a missed guard.
|
|
1073
|
+
if (state.phase !== "running" && state.phase !== "starting") {
|
|
1074
|
+
logger.info `reEmitParkedCorrelations: child not addressable (phase=${state.phase}); skipping`;
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
const controlSender = state.controlSender;
|
|
1078
|
+
const requestId = `pc-${String((parkedQuerySeq += 1))}`;
|
|
1079
|
+
const responded = new Promise((resolve) => {
|
|
1080
|
+
pendingParkedQueries.set(requestId, { settle: resolve });
|
|
1081
|
+
});
|
|
1082
|
+
// Watchdog: a wedged-but-alive child never tears its cohort down, so the
|
|
1083
|
+
// cohort-abort settle would never fire and this await would hang the
|
|
1084
|
+
// re-establishment caller. On expiry, drop the pending entry and return;
|
|
1085
|
+
// the next re-establishment re-drives (the hub co-write is idempotent).
|
|
1086
|
+
let timeoutHandle = null;
|
|
1087
|
+
const watchdog = new Promise((resolve) => {
|
|
1088
|
+
timeoutHandle = setTimeout(() => {
|
|
1089
|
+
timeoutHandle = null;
|
|
1090
|
+
if (pendingParkedQueries.delete(requestId)) {
|
|
1091
|
+
logger.warn `reEmitParkedCorrelations: requestId=${requestId} did not respond within ${String(parkedQueryWatchdogMs)}ms; re-registration retries on the next re-establishment`;
|
|
1092
|
+
}
|
|
1093
|
+
resolve("timeout");
|
|
1094
|
+
}, parkedQueryWatchdogMs);
|
|
1095
|
+
});
|
|
1096
|
+
let outcome;
|
|
1097
|
+
try {
|
|
1098
|
+
await controlSender.send({
|
|
1099
|
+
type: "parked-correlations.request",
|
|
1100
|
+
data: { requestId },
|
|
1101
|
+
});
|
|
1102
|
+
outcome = await Promise.race([responded, watchdog]);
|
|
1103
|
+
}
|
|
1104
|
+
catch (cause) {
|
|
1105
|
+
// The downstream send failed (a closing pipe). Best-effort: drop the
|
|
1106
|
+
// pending entry, log, and return; the next re-establishment re-drives.
|
|
1107
|
+
pendingParkedQueries.delete(requestId);
|
|
1108
|
+
if (timeoutHandle !== null)
|
|
1109
|
+
clearTimeout(timeoutHandle);
|
|
1110
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1111
|
+
logger.warn `reEmitParkedCorrelations query failed: ${message}; re-registration retries on the next re-establishment`;
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
if (timeoutHandle !== null)
|
|
1115
|
+
clearTimeout(timeoutHandle);
|
|
1116
|
+
// `timeout` (watchdog fired) or `null` (cohort torn down before the child
|
|
1117
|
+
// answered): nothing to re-emit; the next re-establishment re-drives.
|
|
1118
|
+
if (outcome === "timeout" || outcome === null)
|
|
1119
|
+
return;
|
|
1120
|
+
for (const parked of outcome) {
|
|
1121
|
+
if (parked.parkKind === "input") {
|
|
1122
|
+
// Register cohort membership BEFORE the input channel so the run is
|
|
1123
|
+
// never a channel-without-cohort entry: the dispatch loop's routing
|
|
1124
|
+
// hygiene drops exactly such entries, and a live resumed run must not
|
|
1125
|
+
// be mistaken for a dead one and have its channel deleted.
|
|
1126
|
+
cohortRunIds.add(parked.runId);
|
|
1127
|
+
runInputChannels.set(parked.runId, {
|
|
1128
|
+
correlationId: parked.correlationId,
|
|
1129
|
+
parkKind: "input",
|
|
1130
|
+
});
|
|
1131
|
+
}
|
|
1132
|
+
else {
|
|
1133
|
+
registerSuspension({ ...parked, parkKind: "approval" });
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
function resolveParkNotifyWaiter(runId) {
|
|
1138
|
+
const resolve = parkNotifyWaiters.get(runId);
|
|
1139
|
+
if (resolve === undefined)
|
|
1140
|
+
return;
|
|
1141
|
+
parkNotifyWaiters.delete(runId);
|
|
1142
|
+
resolve();
|
|
1143
|
+
}
|
|
528
1144
|
/**
|
|
529
1145
|
* OUTBOUND half of mailbox ownership (§3a). The workflow-process child
|
|
530
1146
|
* never holds the agent's signing key; it forwards the structured
|
|
@@ -628,7 +1244,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
628
1244
|
// is which process owns the substrate write contract.
|
|
629
1245
|
const writePrincipal = {
|
|
630
1246
|
kind: "workflow-process",
|
|
631
|
-
|
|
1247
|
+
anchorRunId: bindings.anchorRunId,
|
|
632
1248
|
};
|
|
633
1249
|
// The commit's terminal detection comes from the kind handler's
|
|
634
1250
|
// typed `newlyTerminalRuns` signal (returned below), not a sniff of
|
|
@@ -656,7 +1272,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
656
1272
|
// benchmark's per-message OLS fit groups on.
|
|
657
1273
|
const legClassification = classifyProxiedWriteLeg(data.preservePrefix);
|
|
658
1274
|
if (legClassification !== null) {
|
|
659
|
-
legMarkStart(legClassification.
|
|
1275
|
+
legMarkStart(legClassification.messageId, legClassification.leg);
|
|
660
1276
|
}
|
|
661
1277
|
try {
|
|
662
1278
|
const { commitSha, newlyTerminalRuns } = await bindings.repoStore.writeTreePreservingPrefix(writePrincipal, validatedRepoId, data.ref, {
|
|
@@ -701,22 +1317,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
701
1317
|
});
|
|
702
1318
|
// D2 leg end: the substrate commit (hash objects, write tree,
|
|
703
1319
|
// advance ref under the per-repo lock) just resolved. Stamped here,
|
|
704
|
-
// before the
|
|
705
|
-
//
|
|
706
|
-
//
|
|
1320
|
+
// before the response, so the run-event/wal leg measures only its own
|
|
1321
|
+
// commit and not the dispatch loop's markConsumed (which the
|
|
1322
|
+
// `markconsumed` leg owns).
|
|
707
1323
|
if (legClassification !== null) {
|
|
708
|
-
legMarkEnd(legClassification.
|
|
709
|
-
}
|
|
710
|
-
const watchdog = await synchronouslyDispatchTerminalWrite(newlyTerminalRuns);
|
|
711
|
-
if (!watchdog.ok) {
|
|
712
|
-
await controlSender.send({
|
|
713
|
-
type: "substrate.write.response",
|
|
714
|
-
data: {
|
|
715
|
-
requestId: data.requestId,
|
|
716
|
-
result: { ok: false, reason: watchdog.reason },
|
|
717
|
-
},
|
|
718
|
-
});
|
|
719
|
-
return;
|
|
1324
|
+
legMarkEnd(legClassification.messageId, legClassification.leg);
|
|
720
1325
|
}
|
|
721
1326
|
await controlSender.send({
|
|
722
1327
|
type: "substrate.write.response",
|
|
@@ -740,7 +1345,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
740
1345
|
substrate: bindings.repoStore,
|
|
741
1346
|
repoId: validatedRepoId,
|
|
742
1347
|
ref: data.ref,
|
|
743
|
-
|
|
1348
|
+
anchorRunId: bindings.anchorRunId,
|
|
744
1349
|
runId,
|
|
745
1350
|
}).catch((cause) => {
|
|
746
1351
|
logger.warn `compaction of run ${runId} failed: ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
@@ -765,129 +1370,6 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
765
1370
|
});
|
|
766
1371
|
}
|
|
767
1372
|
}
|
|
768
|
-
// Per-runId synchronization between the substrate-write handler
|
|
769
|
-
// and the dispatch loop's `markConsumed`. The handler arms a
|
|
770
|
-
// waiter when it commits a terminal-event blob and waits for the
|
|
771
|
-
// dispatch loop to fire `resolveMarkConsumedWaiter(runId)` before
|
|
772
|
-
// sending the substrate.write.response back to the child.
|
|
773
|
-
const markConsumedCompletionWaiters = new Map();
|
|
774
|
-
function resolveMarkConsumedWaiter(runId) {
|
|
775
|
-
const waiter = markConsumedCompletionWaiters.get(runId);
|
|
776
|
-
if (waiter === undefined)
|
|
777
|
-
return;
|
|
778
|
-
markConsumedCompletionWaiters.delete(runId);
|
|
779
|
-
waiter.resolve();
|
|
780
|
-
}
|
|
781
|
-
/**
|
|
782
|
-
* Hold the substrate.write.response until the dispatch loop's
|
|
783
|
-
* markConsumed settles for each run the kind handler reports as newly
|
|
784
|
-
* terminal in this commit. Terminal-ness comes from the handler's typed
|
|
785
|
-
* `newlyTerminalRuns` signal -- determined authoritatively during
|
|
786
|
-
* validation -- not re-derived from the committed path shape, so it
|
|
787
|
-
* survives the run-event layout changing (e.g. compaction folding a
|
|
788
|
-
* run's per-event files into one combined file). The wait is per-runId
|
|
789
|
-
* so multiple runs can proceed concurrently if a future dispatch loop
|
|
790
|
-
* ever processes more than one mail in parallel.
|
|
791
|
-
*
|
|
792
|
-
* A watchdog timeout (`terminalWriteWatchdogMs`) caps each wait so a
|
|
793
|
-
* never-arming markConsumed (a bug in the dispatch loop, a torn-down
|
|
794
|
-
* cohort, a stalled inbox primitive) does not deadlock the child's
|
|
795
|
-
* write -- and therefore the runtime body, and therefore the dispatch
|
|
796
|
-
* loop. On expiry the waiter is force-released and a structured failure
|
|
797
|
-
* propagates back to the child as
|
|
798
|
-
* `{ ok: false, reason: "terminal-write watchdog timeout: ..." }`.
|
|
799
|
-
*/
|
|
800
|
-
async function synchronouslyDispatchTerminalWrite(newlyTerminalRuns) {
|
|
801
|
-
const holds = [];
|
|
802
|
-
for (const { runId, terminalEventJson } of newlyTerminalRuns) {
|
|
803
|
-
if (!inFlightRuns.has(runId))
|
|
804
|
-
continue;
|
|
805
|
-
holds.push(holdResponseForMarkConsumed(runId, terminalEventJson));
|
|
806
|
-
}
|
|
807
|
-
if (holds.length === 0)
|
|
808
|
-
return { ok: true };
|
|
809
|
-
const results = await Promise.all(holds);
|
|
810
|
-
return results.find((r) => !r.ok) ?? { ok: true };
|
|
811
|
-
}
|
|
812
|
-
async function holdResponseForMarkConsumed(runId, terminalEventJson) {
|
|
813
|
-
const completed = new Promise((resolve, reject) => {
|
|
814
|
-
markConsumedCompletionWaiters.set(runId, { resolve, reject });
|
|
815
|
-
});
|
|
816
|
-
const broadcaster = activeTerminalBroadcaster();
|
|
817
|
-
if (broadcaster !== null) {
|
|
818
|
-
const synthetic = synthesizeTerminalEvent(terminalEventJson);
|
|
819
|
-
if (synthetic !== null) {
|
|
820
|
-
broadcaster.notify(runId, synthetic);
|
|
821
|
-
}
|
|
822
|
-
}
|
|
823
|
-
let timeoutHandle = null;
|
|
824
|
-
const watchdog = new Promise((resolve) => {
|
|
825
|
-
timeoutHandle = setTimeout(() => {
|
|
826
|
-
timeoutHandle = null;
|
|
827
|
-
// Force-release the waiter so the dispatch loop's eventual
|
|
828
|
-
// resolve does not strand a dangling map entry, then surface
|
|
829
|
-
// the structured failure to the caller. The reason text is
|
|
830
|
-
// logged through the package logger so the watchdog is not
|
|
831
|
-
// silent on the host side.
|
|
832
|
-
const stillPending = markConsumedCompletionWaiters.get(runId) !== undefined;
|
|
833
|
-
if (stillPending) {
|
|
834
|
-
markConsumedCompletionWaiters.delete(runId);
|
|
835
|
-
}
|
|
836
|
-
const reason = `terminal-write watchdog timeout: markConsumed for runId=${runId} did not settle within ${String(terminalWriteWatchdogMs)}ms`;
|
|
837
|
-
logger.error `${reason}`;
|
|
838
|
-
resolve({ ok: false, reason });
|
|
839
|
-
}, terminalWriteWatchdogMs);
|
|
840
|
-
});
|
|
841
|
-
const result = await Promise.race([
|
|
842
|
-
completed.then(() => ({ ok: true })),
|
|
843
|
-
watchdog,
|
|
844
|
-
]);
|
|
845
|
-
if (timeoutHandle !== null) {
|
|
846
|
-
clearTimeout(timeoutHandle);
|
|
847
|
-
}
|
|
848
|
-
return result;
|
|
849
|
-
}
|
|
850
|
-
function synthesizeTerminalEvent(terminalEventJson) {
|
|
851
|
-
let parsed;
|
|
852
|
-
try {
|
|
853
|
-
parsed = JSON.parse(terminalEventJson);
|
|
854
|
-
}
|
|
855
|
-
catch {
|
|
856
|
-
return null;
|
|
857
|
-
}
|
|
858
|
-
if (typeof parsed !== "object" ||
|
|
859
|
-
parsed === null ||
|
|
860
|
-
!("type" in parsed) ||
|
|
861
|
-
!("seq" in parsed)) {
|
|
862
|
-
return null;
|
|
863
|
-
}
|
|
864
|
-
const body = parsed;
|
|
865
|
-
if (typeof body.seq !== "number")
|
|
866
|
-
return null;
|
|
867
|
-
const at = typeof body.at === "string" ? body.at : new Date().toISOString();
|
|
868
|
-
if (body.type === "RunCompleted") {
|
|
869
|
-
return { kind: "RunCompleted", seq: body.seq, at };
|
|
870
|
-
}
|
|
871
|
-
if (body.type === "RunCancelled") {
|
|
872
|
-
return { kind: "RunCancelled", seq: body.seq, at };
|
|
873
|
-
}
|
|
874
|
-
if (body.type === "RunFailed") {
|
|
875
|
-
// The wire schema makes `error.message` required when the event
|
|
876
|
-
// type is `RunFailed`. An event that doesn't carry one is a
|
|
877
|
-
// contract violation upstream of the supervisor; coercing it to an
|
|
878
|
-
// empty string would silently hide the producer bug.
|
|
879
|
-
if (typeof body.error?.message !== "string") {
|
|
880
|
-
throw new Error(`synthesizeTerminalEvent: RunFailed event missing required error.message`);
|
|
881
|
-
}
|
|
882
|
-
return {
|
|
883
|
-
kind: "RunFailed",
|
|
884
|
-
seq: body.seq,
|
|
885
|
-
at,
|
|
886
|
-
error: { message: body.error.message },
|
|
887
|
-
};
|
|
888
|
-
}
|
|
889
|
-
return null;
|
|
890
|
-
}
|
|
891
1373
|
function activeControlSender() {
|
|
892
1374
|
if (state.phase === "starting" ||
|
|
893
1375
|
state.phase === "running" ||
|
|
@@ -896,14 +1378,6 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
896
1378
|
}
|
|
897
1379
|
return null;
|
|
898
1380
|
}
|
|
899
|
-
function activeTerminalBroadcaster() {
|
|
900
|
-
if (state.phase === "starting" ||
|
|
901
|
-
state.phase === "running" ||
|
|
902
|
-
state.phase === "recycling") {
|
|
903
|
-
return state.terminalBroadcaster;
|
|
904
|
-
}
|
|
905
|
-
return null;
|
|
906
|
-
}
|
|
907
1381
|
async function wireChild(args) {
|
|
908
1382
|
const controlSender = createControlChannelSender({
|
|
909
1383
|
privateKeySeed: args.ipcKeypair.privateKey,
|
|
@@ -921,10 +1395,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
921
1395
|
hmacKey: args.hmacKey,
|
|
922
1396
|
channelId: args.channelId,
|
|
923
1397
|
reader: args.handle.eventReader,
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1398
|
+
// Route event-channel crashes through the same funnel as
|
|
1399
|
+
// control-channel crashes so both drive the respawn/crash-loop path
|
|
1400
|
+
// uniformly on the live cohort (and defer to the owning lifecycle
|
|
1401
|
+
// path in every other phase).
|
|
1402
|
+
onCrash: onChildCrash,
|
|
928
1403
|
});
|
|
929
1404
|
const eventPump = pumpEvents(eventIter, args.onInferenceEvent);
|
|
930
1405
|
return {
|
|
@@ -951,7 +1426,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
951
1426
|
channelId,
|
|
952
1427
|
hmacKey,
|
|
953
1428
|
hostPublicKey: ipcKeypair.publicKey,
|
|
954
|
-
|
|
1429
|
+
anchorRunId: bindings.anchorRunId,
|
|
955
1430
|
deploymentMailAddress: bindings.deploymentMailAddress,
|
|
956
1431
|
stepCount: bindings.stepCount,
|
|
957
1432
|
definitionHash: opts.definitionHash,
|
|
@@ -1029,7 +1504,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1029
1504
|
repoStore: bindings.repoStore,
|
|
1030
1505
|
principal: bindings.readPrincipal,
|
|
1031
1506
|
stepOrder: opts.stepOrder,
|
|
1032
|
-
|
|
1507
|
+
anchorRunId: bindings.anchorRunId,
|
|
1033
1508
|
deriveStepAddress: bindings.deriveStepAddress,
|
|
1034
1509
|
...(bindings.deriveStepRepoId !== undefined
|
|
1035
1510
|
? { deriveStepRepoId: bindings.deriveStepRepoId }
|
|
@@ -1124,19 +1599,27 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1124
1599
|
// same control channel `trigger.fire` uses, so the ordering
|
|
1125
1600
|
// guarantee (`grants-updated` lands before `trigger.fire`) holds
|
|
1126
1601
|
// for buffered and post-ready inbound mail alike.
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1602
|
+
//
|
|
1603
|
+
// Suppressed when `onRunStart` is wired: that binding makes the
|
|
1604
|
+
// dispatch loop push a per-run snapshot before each `trigger.fire`,
|
|
1605
|
+
// so the spawn-time push would only mask a broken per-run barrier
|
|
1606
|
+
// (the child would already hold grants and never hit its throw-on-
|
|
1607
|
+
// null guard). The per-run push is then the sole grants source.
|
|
1608
|
+
if (bindings.onRunStart === undefined) {
|
|
1609
|
+
await wired.wiring.controlSender.send({
|
|
1610
|
+
type: "grants-updated",
|
|
1611
|
+
data: {
|
|
1612
|
+
snapshot: {
|
|
1613
|
+
steps: credentialsSnapshot.steps.map((s) => ({
|
|
1614
|
+
stepId: s.stepId,
|
|
1615
|
+
address: s.address,
|
|
1616
|
+
grants: [...s.grants],
|
|
1617
|
+
contentHash: s.contentHash,
|
|
1618
|
+
})),
|
|
1619
|
+
},
|
|
1137
1620
|
},
|
|
1138
|
-
}
|
|
1139
|
-
}
|
|
1621
|
+
});
|
|
1622
|
+
}
|
|
1140
1623
|
// Transition to running. The dispatch loop (started below)
|
|
1141
1624
|
// picks up any pre-ready buffered mail through the FIFO inbox
|
|
1142
1625
|
// queue rather than through an in-memory buffer; arrival order
|
|
@@ -1170,6 +1653,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1170
1653
|
dispatchLoop,
|
|
1171
1654
|
replayDone,
|
|
1172
1655
|
};
|
|
1656
|
+
// Bump the generation and arm the exit-watcher atomically with the
|
|
1657
|
+
// running transition (no await between the swap above and this call)
|
|
1658
|
+
// so an unexpected exit of this child is classified against the
|
|
1659
|
+
// right generation.
|
|
1660
|
+
armChildForRunning(handle);
|
|
1173
1661
|
// Kick the dispatch loop in case mail landed in the inbox
|
|
1174
1662
|
// before the loop's first `await dispatchWake`. A wake against a
|
|
1175
1663
|
// freshly-minted promise is a no-op; the dispatch loop's first
|
|
@@ -1200,6 +1688,18 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1200
1688
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1201
1689
|
logger.error `upstream control pump failed: ${message}`;
|
|
1202
1690
|
});
|
|
1691
|
+
// Trigger A: re-register every correlation the freshly-ready child is
|
|
1692
|
+
// parked on. A `park.notify` register can be lost while the hub is down
|
|
1693
|
+
// at the original suspend; a child that resumes such a parked run (a
|
|
1694
|
+
// sidecar restart re-spawning this deployment, or a recycle -- see the
|
|
1695
|
+
// matching call in `installNewChild`) re-parks without re-emitting, so
|
|
1696
|
+
// the supervisor re-drives it from every re-establishment. Fire-and-
|
|
1697
|
+
// forget after the pump is armed to route the response: best-effort,
|
|
1698
|
+
// watchdog-bounded, and an empty round-trip when nothing is parked.
|
|
1699
|
+
void reEmitParkedCorrelations().catch((cause) => {
|
|
1700
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1701
|
+
logger.warn `re-emit of parked correlations on re-establishment failed: ${message}`;
|
|
1702
|
+
});
|
|
1203
1703
|
// Arm the recycle policy. The policy is a no-op when all bounds
|
|
1204
1704
|
// are `undefined`; bounds resolution lives inside `createRecyclePolicy`.
|
|
1205
1705
|
if (bindings.recyclePolicy !== undefined) {
|
|
@@ -1318,60 +1818,132 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1318
1818
|
}
|
|
1319
1819
|
/**
|
|
1320
1820
|
* Forward one dequeued inbox entry to the child as `trigger.fire`
|
|
1321
|
-
* and record its runId as in-flight. The runId is the
|
|
1322
|
-
*
|
|
1323
|
-
*
|
|
1324
|
-
*
|
|
1821
|
+
* and record its runId as in-flight. The runId is the local part of the
|
|
1822
|
+
* deployment's mail address (see `deriveWorkflowRunId`), identifying its one
|
|
1823
|
+
* top-level run; the `messageId` rides alongside it so the child can
|
|
1824
|
+
* recover the trigger's mail bytes by claim-check. The runId is the
|
|
1825
|
+
* same value the dispatch loop waits on via `terminalEventSource`.
|
|
1325
1826
|
*/
|
|
1326
|
-
async function forwardDispatchedEntry(sender, messageId, receivedAt) {
|
|
1827
|
+
async function forwardDispatchedEntry(sender, messageId, receivedAt, runId) {
|
|
1327
1828
|
await sender.send({
|
|
1328
1829
|
type: "trigger.fire",
|
|
1329
1830
|
data: {
|
|
1330
|
-
runId
|
|
1831
|
+
runId,
|
|
1331
1832
|
messageId,
|
|
1332
1833
|
receivedAt,
|
|
1333
1834
|
},
|
|
1334
1835
|
});
|
|
1335
|
-
|
|
1336
|
-
return
|
|
1836
|
+
cohortRunIds.add(runId);
|
|
1837
|
+
return runId;
|
|
1838
|
+
}
|
|
1839
|
+
/**
|
|
1840
|
+
* Push the run's grants snapshot to the child ahead of its
|
|
1841
|
+
* `trigger.fire`. Returns `true` if the barrier FAILED (the caller must
|
|
1842
|
+
* skip the fire; the run has already been settled as `RunFailed`) and
|
|
1843
|
+
* `false` if the barrier passed or is not armed (`onRunStart` unwired,
|
|
1844
|
+
* where `spawn` supplied the snapshot instead).
|
|
1845
|
+
*
|
|
1846
|
+
* The sink is a request/response contract: the supervisor awaits the
|
|
1847
|
+
* returned snapshot and awaits the `grants-updated` send so both land on
|
|
1848
|
+
* the child's control channel before the trigger. A throw from either --
|
|
1849
|
+
* the sink itself or the control send -- is surfaced as a synthesized
|
|
1850
|
+
* `RunFailed` fanned out to this run's broadcaster watcher, never
|
|
1851
|
+
* swallowed, so the run fails deterministically instead of the child
|
|
1852
|
+
* authorizing against a stale or absent snapshot.
|
|
1853
|
+
*/
|
|
1854
|
+
async function pushRunGrants(sender, runId, broadcaster) {
|
|
1855
|
+
if (bindings.onRunStart === undefined)
|
|
1856
|
+
return false;
|
|
1857
|
+
try {
|
|
1858
|
+
const snapshot = await bindings.onRunStart({
|
|
1859
|
+
runId,
|
|
1860
|
+
anchorRunId: bindings.anchorRunId,
|
|
1861
|
+
});
|
|
1862
|
+
await sender.send({
|
|
1863
|
+
type: "grants-updated",
|
|
1864
|
+
data: {
|
|
1865
|
+
snapshot: {
|
|
1866
|
+
steps: snapshot.steps.map((s) => ({
|
|
1867
|
+
stepId: s.stepId,
|
|
1868
|
+
address: s.address,
|
|
1869
|
+
grants: [...s.grants],
|
|
1870
|
+
contentHash: s.contentHash,
|
|
1871
|
+
})),
|
|
1872
|
+
},
|
|
1873
|
+
},
|
|
1874
|
+
});
|
|
1875
|
+
// Deliver the deployment's credential material on the same pre-trigger
|
|
1876
|
+
// barrier, so a tool that resolves a credential on the first step already
|
|
1877
|
+
// has it in the child's cell. The material is the decrypted delivery the
|
|
1878
|
+
// hub put on the deploy frame; a later rotation flows through
|
|
1879
|
+
// `deliverCredentials` instead. Absent when the deployment binds none.
|
|
1880
|
+
if (bindings.credentialDelivery !== undefined) {
|
|
1881
|
+
await sender.send({
|
|
1882
|
+
type: "credentials-updated",
|
|
1883
|
+
data: { delivery: bindings.credentialDelivery },
|
|
1884
|
+
});
|
|
1885
|
+
}
|
|
1886
|
+
return false;
|
|
1887
|
+
}
|
|
1888
|
+
catch (cause) {
|
|
1889
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1890
|
+
logger.error `onRunStart grants barrier failed for run ${runId}; failing the run: ${message}`;
|
|
1891
|
+
broadcaster.notify(runId, {
|
|
1892
|
+
kind: "RunFailed",
|
|
1893
|
+
seq: 0,
|
|
1894
|
+
at: new Date().toISOString(),
|
|
1895
|
+
error: {
|
|
1896
|
+
message: `workflow-host supervisor: run ${runId} not authorized; grants barrier failed before trigger.fire: ${message}`,
|
|
1897
|
+
},
|
|
1898
|
+
});
|
|
1899
|
+
return true;
|
|
1900
|
+
}
|
|
1337
1901
|
}
|
|
1338
1902
|
/**
|
|
1339
1903
|
* One iteration of the dispatch loop: dequeue the FIFO-first inbox
|
|
1340
|
-
* entry,
|
|
1341
|
-
*
|
|
1342
|
-
*
|
|
1343
|
-
*
|
|
1344
|
-
*
|
|
1904
|
+
* entry, decide whether to `signal.deliver` or `trigger.fire` (or wait
|
|
1905
|
+
* if the run is in-flight but not yet parked), then `markConsumed`
|
|
1906
|
+
* once the child has taken up the message -- for a `trigger.fire` that
|
|
1907
|
+
* means after the run reaches a terminal event or parks, so the
|
|
1908
|
+
* claim-check entry the child still needs to read is not deleted out
|
|
1909
|
+
* from under it. Returns `true` if a dispatch landed (caller should
|
|
1910
|
+
* loop immediately) and `false` if the inbox was empty (caller should
|
|
1911
|
+
* await the next wake).
|
|
1345
1912
|
*/
|
|
1346
1913
|
async function dispatchOne(sender, cohortAbort, broadcaster) {
|
|
1347
1914
|
if (cohortAbort.signal.aborted)
|
|
1348
1915
|
return false;
|
|
1349
|
-
// Subscribe to the terminal broadcaster BEFORE forwarding the
|
|
1350
|
-
// trigger.fire so a terminal event the child notifies between
|
|
1351
|
-
// forward and subscribe cannot be missed. The broadcaster fires
|
|
1352
|
-
// its listeners synchronously inside `notify`; with the subscribe
|
|
1353
|
-
// ordered first the listener buffers the event until the
|
|
1354
|
-
// dispatch loop's `iter.next()` consumes it.
|
|
1355
1916
|
const beforeDequeueMs = dispatchTimingEnabled() ? performance.now() : 0;
|
|
1356
1917
|
const dequeued = await inboxPrimitives.dequeueToProcessing(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress);
|
|
1357
1918
|
if (dequeued === null)
|
|
1358
1919
|
return false;
|
|
1359
1920
|
const envelope = dequeued.envelope;
|
|
1360
|
-
const runId =
|
|
1361
|
-
|
|
1362
|
-
|
|
1921
|
+
const runId = deriveWorkflowRunId(bindings.deploymentMailAddress);
|
|
1922
|
+
const messageId = envelope.messageId;
|
|
1923
|
+
let rejection;
|
|
1924
|
+
const rejectTerminalRun = () => {
|
|
1925
|
+
if (rejection !== undefined)
|
|
1926
|
+
return;
|
|
1927
|
+
rejection = {
|
|
1928
|
+
code: "workflow_run_terminal",
|
|
1929
|
+
message: `Workflow run ${runId} is terminal and cannot be fired again`,
|
|
1930
|
+
};
|
|
1931
|
+
logger.warn `rejecting inbound mail ${messageId}: workflow run ${runId} is terminal`;
|
|
1932
|
+
};
|
|
1933
|
+
currentDispatchMessageId = messageId;
|
|
1934
|
+
emitDispatchTiming(messageId, "dispatch-start", beforeDequeueMs);
|
|
1363
1935
|
// D2 leg: the claim-check dequeue READ. `dispatch-start` is sampled
|
|
1364
1936
|
// BEFORE the dequeue (so the roundtrip bracket includes the read);
|
|
1365
1937
|
// the dequeue leg's own start mark is that same pre-dequeue sample
|
|
1366
1938
|
// re-stamped under the leg channel, and its end is now (the read just
|
|
1367
1939
|
// completed). Emitting the start retroactively here -- rather than
|
|
1368
|
-
// before the await -- keeps the leg keyed by the
|
|
1369
|
-
// known after the dequeue resolves.
|
|
1940
|
+
// before the await -- keeps the leg keyed by the messageId, which is
|
|
1941
|
+
// only known after the dequeue resolves.
|
|
1370
1942
|
if (bindings.onDispatchTiming !== undefined) {
|
|
1371
1943
|
try {
|
|
1372
1944
|
bindings.onDispatchTiming({
|
|
1373
1945
|
kind: "leg",
|
|
1374
|
-
|
|
1946
|
+
messageId,
|
|
1375
1947
|
leg: "dequeue",
|
|
1376
1948
|
phase: "start",
|
|
1377
1949
|
atMs: beforeDequeueMs,
|
|
@@ -1379,28 +1951,212 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1379
1951
|
}
|
|
1380
1952
|
catch (cause) {
|
|
1381
1953
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1382
|
-
logger.warn `onDispatchTiming leg observer threw for ${
|
|
1954
|
+
logger.warn `onDispatchTiming leg observer threw for ${messageId} (dequeue start): ${message}`;
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
legMarkEnd(messageId, "dequeue");
|
|
1958
|
+
// In-memory cohort membership is deliberately not the lifecycle
|
|
1959
|
+
// authority: it is empty for a new run, after a terminal frame, and while
|
|
1960
|
+
// a child is rediscovering a live run after restart. Consult the durable
|
|
1961
|
+
// log before deciding that "no cohort" means "fire". A live log is added
|
|
1962
|
+
// back to cohort tracking so the existing terminal/park wait handles the
|
|
1963
|
+
// recovery window; a terminal log is rejected permanently.
|
|
1964
|
+
if (!cohortRunIds.has(runId)) {
|
|
1965
|
+
const lifecycle = terminalRunIds.has(runId)
|
|
1966
|
+
? "terminal"
|
|
1967
|
+
: await readWorkflowRunLifecycle(bindings.repoStore, bindings.workflowRunRepoId, runId);
|
|
1968
|
+
if (lifecycle === "terminal") {
|
|
1969
|
+
rejectTerminalRun();
|
|
1970
|
+
}
|
|
1971
|
+
else if (lifecycle === "live") {
|
|
1972
|
+
cohortRunIds.add(runId);
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
if (rejection === undefined) {
|
|
1976
|
+
// Subscribe to the terminal broadcaster BEFORE the grants barrier so
|
|
1977
|
+
// a synthetic `RunFailed` from a barrier failure can be captured.
|
|
1978
|
+
const preIter = broadcaster.source(runId)[Symbol.asyncIterator]();
|
|
1979
|
+
// Per-run grants barrier. When `onRunStart` is wired, push this run's
|
|
1980
|
+
// grants snapshot BEFORE the trigger/signal so the child's authorize
|
|
1981
|
+
// closure binds to it rather than throwing on a null snapshot. The push
|
|
1982
|
+
// and the fire share the child's control channel, so a `grants-updated`
|
|
1983
|
+
// awaited here is observed by the child ahead of the trigger. A barrier
|
|
1984
|
+
// failure (the sink throws, or the push fails) fails the run loudly --
|
|
1985
|
+
// a synthesized `RunFailed` fanned out to this run's watcher -- and the
|
|
1986
|
+
// trigger is NOT fired, so no step ever runs against absent grants.
|
|
1987
|
+
const barrierFailed = await pushRunGrants(sender, runId, broadcaster);
|
|
1988
|
+
if (barrierFailed) {
|
|
1989
|
+
// Wait for the synthetic RunFailed before consuming the message.
|
|
1990
|
+
await waitForRunTerminal(preIter, cohortAbort.signal);
|
|
1991
|
+
// Clean up for synthetic barrier failure (real terminal events are
|
|
1992
|
+
// cleaned up by pumpUpstreamControl, but synthetic ones are not).
|
|
1993
|
+
cohortRunIds.delete(runId);
|
|
1994
|
+
runInputChannels.delete(runId);
|
|
1995
|
+
}
|
|
1996
|
+
else {
|
|
1997
|
+
// Dispose the pre-created iterator; the normal path creates fresh
|
|
1998
|
+
// iterators inside the dispatch-decision loop when waiting.
|
|
1999
|
+
if (typeof preIter.return === "function") {
|
|
2000
|
+
await preIter.return();
|
|
2001
|
+
}
|
|
2002
|
+
// Unified dispatch: park → signal.deliver; no live run → trigger.fire;
|
|
2003
|
+
// in-flight but undecided → wait for terminal or park, then re-evaluate.
|
|
2004
|
+
while (!cohortAbort.signal.aborted) {
|
|
2005
|
+
if (terminalRunIds.has(runId)) {
|
|
2006
|
+
rejectTerminalRun();
|
|
2007
|
+
break;
|
|
2008
|
+
}
|
|
2009
|
+
// Capture the park generation BEFORE this iteration's pre-wait awaits
|
|
2010
|
+
// so `waitForRunTerminalOrPark` can accept a strictly-newer park that
|
|
2011
|
+
// fires during them (see the latch there). Re-captured each iteration
|
|
2012
|
+
// so a wait that returned "parked" is not re-counted next time around.
|
|
2013
|
+
const sinceGen = parkGenerations.get(runId) ?? 0;
|
|
2014
|
+
// Routing hygiene (defense-in-depth the latch never depends on): a
|
|
2015
|
+
// runInputChannels entry with no live run in this cohort is a stale
|
|
2016
|
+
// routing hazard left by a dead incarnation. Drop it BEFORE the signal
|
|
2017
|
+
// branch so a fresh mail cannot be routed onto a dead run's
|
|
2018
|
+
// correlation. The resumed.runs invariant (cohortRunIds registered
|
|
2019
|
+
// before its input channel) keeps a LIVE resumed run out of this
|
|
2020
|
+
// branch, so this only ever drops genuinely-dead entries.
|
|
2021
|
+
if (!cohortRunIds.has(runId) && runInputChannels.has(runId)) {
|
|
2022
|
+
runInputChannels.delete(runId);
|
|
2023
|
+
}
|
|
2024
|
+
const inputChannel = runInputChannels.get(runId);
|
|
2025
|
+
if (inputChannel !== undefined) {
|
|
2026
|
+
// Resolve the inbound mail to conversation text HERE, the single
|
|
2027
|
+
// site that knows this payload's provenance is mail, applying the
|
|
2028
|
+
// SAME extraction the turn-1 trigger does (resolveTriggerPayload).
|
|
2029
|
+
// The signal.deliver frame's payload is the resume decision in FINAL
|
|
2030
|
+
// form; deliverSignal's structured signals ship their own payload
|
|
2031
|
+
// unchanged. Done BEFORE minting the terminal watcher so a failure
|
|
2032
|
+
// here cannot leak an un-finalized iterator.
|
|
2033
|
+
let inputText;
|
|
2034
|
+
try {
|
|
2035
|
+
if (envelope.rawMessage === undefined) {
|
|
2036
|
+
throw new Error("inbound mail carries no rawMessage bytes");
|
|
2037
|
+
}
|
|
2038
|
+
inputText = extractConversationText(base64Decode(envelope.rawMessage), envelope.messageId);
|
|
2039
|
+
}
|
|
2040
|
+
catch (cause) {
|
|
2041
|
+
// A malformed turn-2 mail cannot resume the parked agent. DROP it:
|
|
2042
|
+
// log loudly and consume it (break to the post-loop markConsumed)
|
|
2043
|
+
// rather than throwing -- a throw aborts the dispatch without
|
|
2044
|
+
// consuming, and replay re-delivers the same poison mail forever.
|
|
2045
|
+
// The run stays parked on its current correlation, ready for the
|
|
2046
|
+
// next valid mail; one bad mail must not tear down a long-lived
|
|
2047
|
+
// conversation.
|
|
2048
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
2049
|
+
logger.error `signal.deliver for run ${runId}: dropping malformed inbound mail ${envelope.messageId}: ${message}`;
|
|
2050
|
+
break;
|
|
2051
|
+
}
|
|
2052
|
+
// Mint the terminal watcher only now, after the payload resolved, so
|
|
2053
|
+
// a terminal the resumed run reaches right after applying the signal
|
|
2054
|
+
// is not missed; the park watcher is armed inside
|
|
2055
|
+
// waitForRunTerminalOrPark.
|
|
2056
|
+
const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
|
|
2057
|
+
let waitEntered = false;
|
|
2058
|
+
try {
|
|
2059
|
+
await sender.send({
|
|
2060
|
+
type: "signal.deliver",
|
|
2061
|
+
data: {
|
|
2062
|
+
runId,
|
|
2063
|
+
signalName: signalName(inputChannel.correlationId),
|
|
2064
|
+
signalId: envelope.messageId,
|
|
2065
|
+
payload: inputText,
|
|
2066
|
+
},
|
|
2067
|
+
});
|
|
2068
|
+
// Invalidate the cached input channel: its correlation is now
|
|
2069
|
+
// consumed by this delivery, so the NEXT mail must not reuse it.
|
|
2070
|
+
// The resumed run re-parks on a FRESH correlation (a new
|
|
2071
|
+
// park.notify repopulates runInputChannels); a mail arriving before
|
|
2072
|
+
// that re-park waits via the in-flight branch rather than
|
|
2073
|
+
// delivering onto the stale channel. Routing hygiene only -- the
|
|
2074
|
+
// wait keys on the park-generation edge, not this level state.
|
|
2075
|
+
runInputChannels.delete(runId);
|
|
2076
|
+
// Durable-consume contract, mirroring the trigger.fire path: hold
|
|
2077
|
+
// markConsumed until the child has durably taken up the signal --
|
|
2078
|
+
// the resumed run re-parks or reaches a terminal event. That gate
|
|
2079
|
+
// is downstream of durability DESPITE the child's fire-and-forget
|
|
2080
|
+
// SignalReceived writer: the runtime reaches re-park/terminal only
|
|
2081
|
+
// by resuming from the COMMITTED SignalReceived, which its per-run
|
|
2082
|
+
// subscribeKind substrate subscription surfaces only after the
|
|
2083
|
+
// commit lands -- so the substrate subscription IS the ack, and a
|
|
2084
|
+
// failed deliver commit is observed by nothing, never re-parks, and
|
|
2085
|
+
// never releases markConsumed (the mail stays reclaimable). A crash
|
|
2086
|
+
// before the re-park/terminal leaves the claim-check entry in
|
|
2087
|
+
// processing/, so replayProcessingToInbox re-delivers the signal on
|
|
2088
|
+
// restart. On cohort abort the wait returns and the post-loop guard
|
|
2089
|
+
// skips markConsumed.
|
|
2090
|
+
waitEntered = true;
|
|
2091
|
+
await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
|
|
2092
|
+
}
|
|
2093
|
+
finally {
|
|
2094
|
+
// waitForRunTerminalOrPark finalizes the iterator it consumes; the
|
|
2095
|
+
// only leak is when `sender.send` throws before the wait is
|
|
2096
|
+
// entered, so finalize only in that case.
|
|
2097
|
+
if (!waitEntered && typeof iter.return === "function") {
|
|
2098
|
+
await iter.return(undefined).catch(() => {
|
|
2099
|
+
/* best-effort finalisation of the watcher iterator. */
|
|
2100
|
+
});
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
break;
|
|
2104
|
+
}
|
|
2105
|
+
if (!cohortRunIds.has(runId)) {
|
|
2106
|
+
// Subscribe the terminal watcher BEFORE the trigger fires. The
|
|
2107
|
+
// broadcaster drops a notify that has no listener (its subscribe-
|
|
2108
|
+
// before-fire contract), so a terminal that lands while
|
|
2109
|
+
// forwardDispatchedEntry is in flight would be lost and the wait
|
|
2110
|
+
// would hang to the backstop.
|
|
2111
|
+
const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
|
|
2112
|
+
let waitEntered = false;
|
|
2113
|
+
try {
|
|
2114
|
+
await forwardDispatchedEntry(sender, envelope.messageId, envelope.receivedAt, runId);
|
|
2115
|
+
// Wait for the child to process this trigger before allowing
|
|
2116
|
+
// `markConsumed` to move the claim-check entry out of
|
|
2117
|
+
// `processing/`. The child reads the trigger payload from that
|
|
2118
|
+
// entry; racing `markConsumed` would delete the entry before the
|
|
2119
|
+
// child resolves it. On cohort abort the wait returns and the
|
|
2120
|
+
// post-loop guard skips markConsumed.
|
|
2121
|
+
waitEntered = true;
|
|
2122
|
+
await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
|
|
2123
|
+
}
|
|
2124
|
+
finally {
|
|
2125
|
+
// waitForRunTerminalOrPark finalizes the iterator it consumes; the
|
|
2126
|
+
// only leak is when forward throws before the wait is
|
|
2127
|
+
// entered, so finalize only in that case.
|
|
2128
|
+
if (!waitEntered && typeof iter.return === "function") {
|
|
2129
|
+
await iter.return(undefined).catch(() => {
|
|
2130
|
+
/* best-effort finalisation of the watcher iterator. */
|
|
2131
|
+
});
|
|
2132
|
+
}
|
|
2133
|
+
}
|
|
2134
|
+
break;
|
|
2135
|
+
}
|
|
2136
|
+
const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
|
|
2137
|
+
const outcome = await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
|
|
2138
|
+
if (outcome === "aborted")
|
|
2139
|
+
break;
|
|
2140
|
+
if (outcome === "terminal") {
|
|
2141
|
+
// This message was waiting for an already-live run to expose its
|
|
2142
|
+
// next input correlation. The run terminated first, so the mail
|
|
2143
|
+
// was never delivered and must not fall through to trigger.fire.
|
|
2144
|
+
rejectTerminalRun();
|
|
2145
|
+
break;
|
|
2146
|
+
}
|
|
2147
|
+
// Continue loop: re-evaluate runInputChannels / cohortRunIds
|
|
2148
|
+
}
|
|
1383
2149
|
}
|
|
1384
2150
|
}
|
|
1385
|
-
legMarkEnd(runId, "dequeue");
|
|
1386
|
-
const iterable = broadcaster.source(runId);
|
|
1387
|
-
const iter = iterable[Symbol.asyncIterator]();
|
|
1388
|
-
await forwardDispatchedEntry(sender, envelope.messageId, envelope.receivedAt);
|
|
1389
|
-
await waitForRunTerminal(iter, cohortAbort.signal);
|
|
1390
|
-
emitDispatchTiming(runId, "reply-produced", performance.now());
|
|
1391
|
-
inFlightRuns.delete(runId);
|
|
1392
2151
|
if (cohortAbort.signal.aborted) {
|
|
1393
|
-
|
|
1394
|
-
// alongside it). Skip `markConsumed` so the recycle path's
|
|
1395
|
-
// drain-side replay can reclaim the processing entry.
|
|
1396
|
-
currentDispatchRunId = null;
|
|
1397
|
-
resolveMarkConsumedWaiter(runId);
|
|
2152
|
+
currentDispatchMessageId = null;
|
|
1398
2153
|
return false;
|
|
1399
2154
|
}
|
|
1400
|
-
|
|
1401
|
-
//
|
|
1402
|
-
// the leg mark makes the
|
|
1403
|
-
|
|
2155
|
+
emitDispatchTiming(messageId, "reply-produced", performance.now());
|
|
2156
|
+
// D2 leg: `markConsumed` is paid AFTER `reply-produced`, so its growth
|
|
2157
|
+
// is invisible to the 4.7 round-trip bracket -- the leg mark makes the
|
|
2158
|
+
// out-of-window cost visible.
|
|
2159
|
+
legMarkStart(messageId, "markconsumed");
|
|
1404
2160
|
try {
|
|
1405
2161
|
await inboxPrimitives.markConsumed(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
|
|
1406
2162
|
address: bindings.deploymentMailAddress,
|
|
@@ -1408,17 +2164,21 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1408
2164
|
runId,
|
|
1409
2165
|
consumedAt: Date.now(),
|
|
1410
2166
|
retentionHorizonMs: consumedRetentionMs,
|
|
2167
|
+
...(rejection !== undefined ? { rejection } : {}),
|
|
1411
2168
|
});
|
|
1412
2169
|
}
|
|
1413
2170
|
catch (cause) {
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
2171
|
+
// A markConsumed failure is fatal:
|
|
2172
|
+
// swallowing it treats the dispatch as complete while the mail is NOT
|
|
2173
|
+
// durably recorded consumed, hiding the failure and leaving a mail that
|
|
2174
|
+
// is neither cleanly consumed nor visibly failed. Propagate into the
|
|
2175
|
+
// dispatch fault handler so the failure surfaces and the mail stays
|
|
2176
|
+
// reclaimable.
|
|
2177
|
+
throw new Error(`failed to markConsumed for run ${runId}`, { cause });
|
|
2178
|
+
}
|
|
2179
|
+
legMarkEnd(messageId, "markconsumed");
|
|
1420
2180
|
maybeRepack(runId);
|
|
1421
|
-
|
|
2181
|
+
currentDispatchMessageId = null;
|
|
1422
2182
|
return true;
|
|
1423
2183
|
}
|
|
1424
2184
|
/**
|
|
@@ -1460,6 +2220,94 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1460
2220
|
}
|
|
1461
2221
|
}
|
|
1462
2222
|
}
|
|
2223
|
+
/**
|
|
2224
|
+
* Wait until the run's terminal event lands on the cohort
|
|
2225
|
+
* broadcaster's iterator, the child parks the run (bumping the park
|
|
2226
|
+
* generation past `sinceGen`), or the cohort aborts. Returns `"terminal"`
|
|
2227
|
+
* when a terminal event arrived, `"parked"` when the run parked,
|
|
2228
|
+
* and `"aborted"` when the cohort tore down. Throws when the backstop
|
|
2229
|
+
* fires (see `TERMINAL_OR_PARK_BACKSTOP_MS`).
|
|
2230
|
+
*
|
|
2231
|
+
* `sinceGen` is the park generation the CALLER captured before its pre-wait
|
|
2232
|
+
* awaits; the wait accepts only a STRICTLY NEWER park (`generation >
|
|
2233
|
+
* sinceGen`). Keying on that edge -- not `runInputChannels`' level state --
|
|
2234
|
+
* makes the wait's correctness local: a park during the pre-wait awaits is
|
|
2235
|
+
* observed even though its `resolveParkNotifyWaiter` no-op'd, and a stale
|
|
2236
|
+
* channel entry from a prior run or incarnation cannot false-positive.
|
|
2237
|
+
*/
|
|
2238
|
+
async function waitForRunTerminalOrPark(iter, abortSignal, runId, sinceGen) {
|
|
2239
|
+
let onAbort = null;
|
|
2240
|
+
const abortPromise = new Promise((resolve) => {
|
|
2241
|
+
if (abortSignal.aborted) {
|
|
2242
|
+
resolve({ source: "abort" });
|
|
2243
|
+
return;
|
|
2244
|
+
}
|
|
2245
|
+
onAbort = () => resolve({ source: "abort" });
|
|
2246
|
+
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
2247
|
+
});
|
|
2248
|
+
let parkResolve = null;
|
|
2249
|
+
const parkPromise = new Promise((resolve) => {
|
|
2250
|
+
parkResolve = () => resolve({ source: "park" });
|
|
2251
|
+
parkNotifyWaiters.set(runId, parkResolve);
|
|
2252
|
+
});
|
|
2253
|
+
let timeoutHandle;
|
|
2254
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
2255
|
+
timeoutHandle = setTimeout(() => resolve({ source: "timeout" }), TERMINAL_OR_PARK_BACKSTOP_MS);
|
|
2256
|
+
});
|
|
2257
|
+
try {
|
|
2258
|
+
if (abortSignal.aborted)
|
|
2259
|
+
return "aborted";
|
|
2260
|
+
// Check-after-register: read the generation now that the waiter above is
|
|
2261
|
+
// armed, SYNCHRONOUSLY (no await between arming and this read, so no
|
|
2262
|
+
// `park.notify` can interleave). A generation past `sinceGen` means the
|
|
2263
|
+
// run already parked -- during the caller's pre-wait awaits, before the
|
|
2264
|
+
// waiter armed, so `resolveParkNotifyWaiter` no-op'd and the armed
|
|
2265
|
+
// parkPromise would never fire -- and this catches it rather than hanging
|
|
2266
|
+
// to the backstop.
|
|
2267
|
+
if ((parkGenerations.get(runId) ?? 0) > sinceGen)
|
|
2268
|
+
return "parked";
|
|
2269
|
+
const result = await Promise.race([
|
|
2270
|
+
iter.next().then((r) => ({ source: "iter", r })),
|
|
2271
|
+
abortPromise,
|
|
2272
|
+
parkPromise,
|
|
2273
|
+
timeoutPromise,
|
|
2274
|
+
]);
|
|
2275
|
+
if (result.source === "abort")
|
|
2276
|
+
return "aborted";
|
|
2277
|
+
if (result.source === "park")
|
|
2278
|
+
return "parked";
|
|
2279
|
+
if (result.source === "timeout") {
|
|
2280
|
+
// Backstop against a lost wake or a wedged child: the run neither
|
|
2281
|
+
// parked, terminated, nor aborted within a generous window. Surface it
|
|
2282
|
+
// LOUDLY and throw so the dispatch fails -- the caller does not
|
|
2283
|
+
// markConsumed on a throw, so the mail stays reclaimable in
|
|
2284
|
+
// processing/ and is never consumed on the assumption the run
|
|
2285
|
+
// progressed.
|
|
2286
|
+
logger.error `waitForRunTerminalOrPark backstop fired for run ${runId} after ${TERMINAL_OR_PARK_BACKSTOP_MS}ms; failing the dispatch so the mail stays reclaimable`;
|
|
2287
|
+
throw new Error(`waitForRunTerminalOrPark backstop: run ${runId} did not park or terminate within ${TERMINAL_OR_PARK_BACKSTOP_MS}ms`);
|
|
2288
|
+
}
|
|
2289
|
+
if (result.r.done === true)
|
|
2290
|
+
return "aborted";
|
|
2291
|
+
// A terminal event for this runId arrived; stop waiting.
|
|
2292
|
+
return "terminal";
|
|
2293
|
+
}
|
|
2294
|
+
finally {
|
|
2295
|
+
if (timeoutHandle !== undefined) {
|
|
2296
|
+
clearTimeout(timeoutHandle);
|
|
2297
|
+
}
|
|
2298
|
+
if (parkResolve !== null) {
|
|
2299
|
+
parkNotifyWaiters.delete(runId);
|
|
2300
|
+
}
|
|
2301
|
+
if (onAbort !== null) {
|
|
2302
|
+
abortSignal.removeEventListener("abort", onAbort);
|
|
2303
|
+
}
|
|
2304
|
+
if (typeof iter.return === "function") {
|
|
2305
|
+
await iter.return(undefined).catch(() => {
|
|
2306
|
+
/* swallowed: best-effort finalisation of the watcher iterator. */
|
|
2307
|
+
});
|
|
2308
|
+
}
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
1463
2311
|
/**
|
|
1464
2312
|
* The dispatch loop body. Runs until the cohort aborts; each
|
|
1465
2313
|
* iteration drains one inbox entry through the FIFO claim-check
|
|
@@ -1482,6 +2330,13 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1482
2330
|
return;
|
|
1483
2331
|
}
|
|
1484
2332
|
while (!cohortAbort.signal.aborted) {
|
|
2333
|
+
// Capture the wake BEFORE the dispatch iteration (capture-before-check,
|
|
2334
|
+
// same discipline as the park-generation latch). `wakeDispatch` resolves
|
|
2335
|
+
// the CURRENT promise and swaps in a fresh one, so a mail that enqueues
|
|
2336
|
+
// DURING dispatchOne resolves THIS captured promise; capturing it after
|
|
2337
|
+
// dispatchOne would await the fresh, unresolved promise and strand that
|
|
2338
|
+
// mail until some later wake.
|
|
2339
|
+
const wake = dispatchWake.promise;
|
|
1485
2340
|
let dispatched;
|
|
1486
2341
|
try {
|
|
1487
2342
|
dispatched = await dispatchOne(sender, cohortAbort, broadcaster);
|
|
@@ -1499,7 +2354,6 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1499
2354
|
continue;
|
|
1500
2355
|
if (cohortAbort.signal.aborted)
|
|
1501
2356
|
return;
|
|
1502
|
-
const wake = dispatchWake.promise;
|
|
1503
2357
|
const abortPromise = new Promise((resolve) => {
|
|
1504
2358
|
if (cohortAbort.signal.aborted) {
|
|
1505
2359
|
resolve();
|
|
@@ -1517,7 +2371,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1517
2371
|
substrate: bindings.repoStore,
|
|
1518
2372
|
repoId: bindings.workflowRunRepoId,
|
|
1519
2373
|
ref: bindings.workflowRunRef,
|
|
1520
|
-
|
|
2374
|
+
anchorRunId: bindings.anchorRunId,
|
|
1521
2375
|
runId: opts.runId,
|
|
1522
2376
|
origin: opts.origin,
|
|
1523
2377
|
reason: opts.reason,
|
|
@@ -1530,7 +2384,9 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1530
2384
|
await shutdownInternal({ reason: "shutdown requested" });
|
|
1531
2385
|
}
|
|
1532
2386
|
async function shutdownInternal(opts) {
|
|
1533
|
-
if (state.phase === "idle" ||
|
|
2387
|
+
if (state.phase === "idle" ||
|
|
2388
|
+
state.phase === "stopped" ||
|
|
2389
|
+
state.phase === "crash-looping")
|
|
1534
2390
|
return;
|
|
1535
2391
|
const prior = state;
|
|
1536
2392
|
state = { phase: "stopping" };
|
|
@@ -1566,11 +2422,15 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1566
2422
|
}
|
|
1567
2423
|
}
|
|
1568
2424
|
drainAccumulators.clear();
|
|
2425
|
+
cohortRunIds.clear();
|
|
2426
|
+
runInputChannels.clear();
|
|
2427
|
+
parkNotifyWaiters.clear();
|
|
2428
|
+
parkGenerations.clear();
|
|
1569
2429
|
if (prior.phase === "starting" ||
|
|
1570
2430
|
prior.phase === "running" ||
|
|
1571
2431
|
prior.phase === "recycling") {
|
|
1572
2432
|
prior.terminalCohortAbort.abort();
|
|
1573
|
-
// Reject every pending merge round-trip and
|
|
2433
|
+
// Reject every pending merge round-trip and park-notify waiter
|
|
1574
2434
|
// so handler closures awaiting them (including fire-and-forget
|
|
1575
2435
|
// `handleSubstrateWriteRequest` instances) cannot outlive the
|
|
1576
2436
|
// dying cohort. Without this, the `await new Promise` inside
|
|
@@ -1628,6 +2488,24 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1628
2488
|
}
|
|
1629
2489
|
recyclePolicy = null;
|
|
1630
2490
|
}
|
|
2491
|
+
// Disarm the crash-loop stable-run reset timer so it cannot fire
|
|
2492
|
+
// against a torn-down supervisor. Also drop any pending child exit
|
|
2493
|
+
// that `maybeHandleChildExit` recorded but declined to act on -- it
|
|
2494
|
+
// leaves one pending when a respawn was in flight, or when the phase
|
|
2495
|
+
// had already left `running`. (The crash-loop latch path does NOT
|
|
2496
|
+
// leave one pending: `maybeHandleChildExit` nulls `pendingChildExit`
|
|
2497
|
+
// before invoking the handler that latches.) A shutdown-initiated
|
|
2498
|
+
// kill of a live child in the `finally` below resolves its
|
|
2499
|
+
// `handle.exited`, so the watcher may re-record a pending exit AFTER
|
|
2500
|
+
// this clear -- harmless: the phase is terminal, so
|
|
2501
|
+
// `maybeHandleChildExit` no-ops on it, and `spawn()` requires `idle`,
|
|
2502
|
+
// so the stale slot is never re-examined.
|
|
2503
|
+
clearStableRunResetTimer();
|
|
2504
|
+
// Cancel every armed respawn backoff wait. The phase was flipped to
|
|
2505
|
+
// `stopping` synchronously above, so each parked respawn coroutine
|
|
2506
|
+
// this unblocks re-checks the phase and bails without respawning.
|
|
2507
|
+
cancelRespawnBackoffWaits();
|
|
2508
|
+
pendingChildExit = null;
|
|
1631
2509
|
spawnContext = null;
|
|
1632
2510
|
if (prior.phase === "starting" ||
|
|
1633
2511
|
prior.phase === "running" ||
|
|
@@ -1675,7 +2553,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1675
2553
|
/* swallowed for the same reason as above. */
|
|
1676
2554
|
});
|
|
1677
2555
|
}
|
|
1678
|
-
state = { phase: "stopped" };
|
|
2556
|
+
state = { phase: opts.terminalPhase ?? "stopped" };
|
|
1679
2557
|
}
|
|
1680
2558
|
logger.info `supervisor shutdown complete (${opts.reason})`;
|
|
1681
2559
|
}
|
|
@@ -1702,11 +2580,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1702
2580
|
*/
|
|
1703
2581
|
async function drainImpl(opts, ctx) {
|
|
1704
2582
|
// Drain is meaningful only when a workflow-process child is up;
|
|
1705
|
-
// calling it from `idle`/`stopping`/`stopped
|
|
1706
|
-
// higher-level host shutdown
|
|
1707
|
-
// unconditionally without sniffing the phase.
|
|
1708
|
-
// calls drain via `drainImpl({}, { fromRecycle: true })`
|
|
1709
|
-
// admits `recycling` because the drain step runs against a
|
|
2583
|
+
// calling it from any non-active phase (`idle`/`stopping`/`stopped`/
|
|
2584
|
+
// `crash-looping`) is a no-op so the higher-level host shutdown
|
|
2585
|
+
// sequence can call drain unconditionally without sniffing the phase.
|
|
2586
|
+
// The recycle path calls drain via `drainImpl({}, { fromRecycle: true })`
|
|
2587
|
+
// and admits `recycling` because the drain step runs against a
|
|
1710
2588
|
// still-live controlSender before the kill lands.
|
|
1711
2589
|
if (state.phase !== "running" &&
|
|
1712
2590
|
state.phase !== "starting" &&
|
|
@@ -1730,15 +2608,22 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1730
2608
|
// body's cancellation cascade tears the run down without the
|
|
1731
2609
|
// supervisor having to thread any per-run wiring beyond what the
|
|
1732
2610
|
// accumulator already encapsulates.
|
|
2611
|
+
//
|
|
2612
|
+
// Runs that are already parked do not need drain escalation; the
|
|
2613
|
+
// runtime parks the run and the supervisor simply stops delivering
|
|
2614
|
+
// new mail. The cohort abort on shutdown/recycle will eventually
|
|
2615
|
+
// tear the run down.
|
|
1733
2616
|
const cohortSource = perCohortTerminalSource(state.terminalCohortAbort, state.terminalBroadcaster);
|
|
1734
|
-
for (const runId of
|
|
2617
|
+
for (const runId of cohortRunIds) {
|
|
1735
2618
|
if (drainAccumulators.has(runId))
|
|
1736
2619
|
continue;
|
|
2620
|
+
if (runInputChannels.has(runId))
|
|
2621
|
+
continue;
|
|
1737
2622
|
const accumulator = accumulatorFactory({
|
|
1738
2623
|
substrate: bindings.repoStore,
|
|
1739
2624
|
repoId: bindings.workflowRunRepoId,
|
|
1740
2625
|
ref: bindings.workflowRunRef,
|
|
1741
|
-
|
|
2626
|
+
anchorRunId: bindings.anchorRunId,
|
|
1742
2627
|
runId,
|
|
1743
2628
|
signAsPrincipal: bindings.signAsPrincipal,
|
|
1744
2629
|
drainTimeoutMs,
|
|
@@ -1752,7 +2637,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1752
2637
|
}
|
|
1753
2638
|
}
|
|
1754
2639
|
async function recycle(opts) {
|
|
1755
|
-
if (
|
|
2640
|
+
if (respawnInProgress) {
|
|
1756
2641
|
throw new Error("supervisor: recycle already in progress");
|
|
1757
2642
|
}
|
|
1758
2643
|
if (state.phase !== "running") {
|
|
@@ -1761,10 +2646,43 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1761
2646
|
if (spawnContext === null) {
|
|
1762
2647
|
throw new Error("supervisor: recycle called without a spawn context; spawn() must complete first");
|
|
1763
2648
|
}
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
2649
|
+
// The contention read (`respawnInProgress`) stays here in the caller's
|
|
2650
|
+
// precondition zone rather than inside `runRespawn`: an operator double-
|
|
2651
|
+
// recycle is an error and must throw. `runRespawn` sets the latch
|
|
2652
|
+
// synchronously at entry, so this read and that set are never separated
|
|
2653
|
+
// by an await, and no second respawn can interleave between them.
|
|
2654
|
+
return runRespawn({
|
|
2655
|
+
origin: opts.origin ?? "operator",
|
|
2656
|
+
reason: opts.reason,
|
|
2657
|
+
prior: state,
|
|
2658
|
+
priorContext: spawnContext,
|
|
2659
|
+
drain: async (deadlineMs) => {
|
|
2660
|
+
// The recycle path's drain step shares the drain primitive but
|
|
2661
|
+
// bypasses the public surface's `recycling` silent-no-op so the
|
|
2662
|
+
// still-live controlSender (this step runs BEFORE abortPriorCohort
|
|
2663
|
+
// + kill) receives the frame. The public `drain()` silently no-ops
|
|
2664
|
+
// on `recycling` for external callers because the kill/respawn gap
|
|
2665
|
+
// can leave the controlSender dying.
|
|
2666
|
+
await drainImpl({ deadlineMs }, { fromRecycle: true });
|
|
2667
|
+
},
|
|
2668
|
+
});
|
|
2669
|
+
}
|
|
2670
|
+
/**
|
|
2671
|
+
* Shared kill/replay/respawn/install driver. The caller has already
|
|
2672
|
+
* verified the supervisor is `running` with a live spawn context and
|
|
2673
|
+
* snapshotted both as `prior`/`priorContext`; this function transitions
|
|
2674
|
+
* to `recycling`, runs the six-step `triggerRecycle` sequence with the
|
|
2675
|
+
* caller-supplied `drain` step, and swaps in the new cohort via the
|
|
2676
|
+
* inline `installNewChild` callback. The operator/policy/self recycle
|
|
2677
|
+
* path is the sole caller today; the crash-respawn path will call it with
|
|
2678
|
+
* a no-op drain (its child is already dead).
|
|
2679
|
+
*/
|
|
2680
|
+
async function runRespawn(args) {
|
|
2681
|
+
// Set synchronously at entry (before any await) so the caller's
|
|
2682
|
+
// contention read and this set cannot be separated by an event-loop
|
|
2683
|
+
// turn; two respawns can never interleave.
|
|
2684
|
+
respawnInProgress = true;
|
|
2685
|
+
const { origin, reason, prior, priorContext, drain } = args;
|
|
1768
2686
|
// The cohort abort no longer fires up-front. triggerRecycle drives
|
|
1769
2687
|
// the drain and replay steps against a LIVE cohort first, then
|
|
1770
2688
|
// invokes `abortPriorCohort` (the callback below) between replay
|
|
@@ -1808,16 +2726,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1808
2726
|
channelId: prior.channelId,
|
|
1809
2727
|
eventPump: prior.eventPump,
|
|
1810
2728
|
},
|
|
1811
|
-
drain
|
|
1812
|
-
// The recycle path's drain step shares the drain
|
|
1813
|
-
// primitive but bypasses the public surface's `recycling`
|
|
1814
|
-
// silent-no-op so the still-live controlSender (this
|
|
1815
|
-
// step runs BEFORE abortPriorCohort + kill) receives the
|
|
1816
|
-
// frame. The public `drain()` silently no-ops on
|
|
1817
|
-
// `recycling` for external callers because the
|
|
1818
|
-
// kill/respawn gap can leave the controlSender dying.
|
|
1819
|
-
await drainImpl({ deadlineMs }, { fromRecycle: true });
|
|
1820
|
-
},
|
|
2729
|
+
drain,
|
|
1821
2730
|
replayProcessingToInbox: async () => {
|
|
1822
2731
|
await inboxPrimitives.replayProcessingToInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress);
|
|
1823
2732
|
},
|
|
@@ -1828,6 +2737,13 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1828
2737
|
prior.terminalCohortAbort.abort();
|
|
1829
2738
|
wakeDispatch();
|
|
1830
2739
|
},
|
|
2740
|
+
// Kept inline rather than extracted: this cohort-swap closes over
|
|
2741
|
+
// the supervisor's full mutable cohort state (drainAccumulators,
|
|
2742
|
+
// cohortRunIds, runInputChannels, parkNotifyWaiters, parkGenerations,
|
|
2743
|
+
// rejectCohortAwaiters, spawnContext, plus `prior`/`priorContext`).
|
|
2744
|
+
// A standalone helper would take all of it as parameters for zero
|
|
2745
|
+
// reuse -- the crash-respawn path reaches this callback transitively
|
|
2746
|
+
// through `runRespawn`, so it needs no separate extraction.
|
|
1831
2747
|
installNewChild: ({ wiring, credentialsSnapshot, controlIncoming, }) => {
|
|
1832
2748
|
// Phase guard: a `shutdown()` that landed during the
|
|
1833
2749
|
// kill/respawn gap (between `subprocessSpawner` and this
|
|
@@ -1871,7 +2787,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1871
2787
|
accumulator.stop();
|
|
1872
2788
|
}
|
|
1873
2789
|
drainAccumulators.clear();
|
|
1874
|
-
|
|
2790
|
+
cohortRunIds.clear();
|
|
2791
|
+
runInputChannels.clear();
|
|
2792
|
+
parkNotifyWaiters.clear();
|
|
2793
|
+
parkGenerations.clear();
|
|
2794
|
+
// Reject every pending merge round-trip and park-notify
|
|
1875
2795
|
// waiter registered against the dying cohort so handler
|
|
1876
2796
|
// closures cannot survive the kill/respawn gap. The new
|
|
1877
2797
|
// child will re-issue substrate writes through fresh
|
|
@@ -1907,6 +2827,12 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1907
2827
|
dispatchLoop: newDispatchLoop,
|
|
1908
2828
|
replayDone: null,
|
|
1909
2829
|
};
|
|
2830
|
+
// Bump the generation and arm the exit-watcher for the
|
|
2831
|
+
// respawned child atomically with this running transition, so
|
|
2832
|
+
// the predecessor's watcher (already stale by generation) never
|
|
2833
|
+
// drives a spurious respawn and a crash of THIS child is
|
|
2834
|
+
// classified against the new generation.
|
|
2835
|
+
armChildForRunning(wiring.handle);
|
|
1910
2836
|
// Cache fresh spawn context with the updated spawnedAt
|
|
1911
2837
|
// so the policy timer's uptime check resets on recycle.
|
|
1912
2838
|
const now = bindings.recyclePolicyNow ?? defaultNow;
|
|
@@ -1934,6 +2860,16 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1934
2860
|
// entries the previous cohort's replayProcessingToInbox
|
|
1935
2861
|
// just moved back.
|
|
1936
2862
|
wakeDispatch();
|
|
2863
|
+
// Trigger A on the recycle seam: the respawned child re-parks any
|
|
2864
|
+
// surviving parked run without re-emitting, and a recycle leaves
|
|
2865
|
+
// the hub link untouched so the reconnect trigger never fires --
|
|
2866
|
+
// so re-drive the re-registration here too. Same fire-and-forget
|
|
2867
|
+
// contract as the spawn seam; the fresh cohort's controlSender is
|
|
2868
|
+
// in `state` now, and its pump (armed above) routes the response.
|
|
2869
|
+
void reEmitParkedCorrelations().catch((cause) => {
|
|
2870
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
2871
|
+
logger.warn `re-emit of parked correlations on re-establishment failed: ${message}`;
|
|
2872
|
+
});
|
|
1937
2873
|
},
|
|
1938
2874
|
onCrash: onChildCrash,
|
|
1939
2875
|
// Edge-resolved once at the supervisor factory; recycle bounds
|
|
@@ -1945,7 +2881,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1945
2881
|
...(bindings.recyclePolicyClearTimer !== undefined
|
|
1946
2882
|
? { clearTimer: bindings.recyclePolicyClearTimer }
|
|
1947
2883
|
: {}),
|
|
1948
|
-
}, { origin, reason
|
|
2884
|
+
}, { origin, reason });
|
|
1949
2885
|
// After the recycle, await the previous cohort's dispatch
|
|
1950
2886
|
// loop so a teardown coroutine cannot survive past the
|
|
1951
2887
|
// recycle's return point.
|
|
@@ -1963,7 +2899,18 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1963
2899
|
// level shutdown. Tear the prior cohort down through the same
|
|
1964
2900
|
// path a real shutdown uses so the supervisor reaches a clean
|
|
1965
2901
|
// `stopped` state, then re-throw so the operator sees the
|
|
1966
|
-
//
|
|
2902
|
+
// failure and can redeploy.
|
|
2903
|
+
//
|
|
2904
|
+
// This teardown-to-`stopped` is shared by both callers, and that is
|
|
2905
|
+
// deliberate. A crash-origin respawn whose spawn/wire/handshake fails
|
|
2906
|
+
// is a broken deploy (a bad binary, unreadable credentials), NOT a
|
|
2907
|
+
// flapping child, so it does NOT feed the crash-loop exit counter and
|
|
2908
|
+
// does NOT reach `crash-looping`; conflating a mechanism failure with
|
|
2909
|
+
// a flap would muddy what that counter means. It reaches `stopped`,
|
|
2910
|
+
// the same terminal a failed operator recycle reaches. (Consequence:
|
|
2911
|
+
// this path leaves no `RunFailed` tombstone, unlike the exit-count
|
|
2912
|
+
// latch; the crash-respawn happy path -- a clean child death whose
|
|
2913
|
+
// respawn succeeds -- is what the crash-loop guard bounds.)
|
|
1967
2914
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1968
2915
|
logger.error `recycle failed; tearing supervisor down: ${message}`;
|
|
1969
2916
|
await shutdownInternal({
|
|
@@ -1977,7 +2924,12 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1977
2924
|
throw cause;
|
|
1978
2925
|
}
|
|
1979
2926
|
finally {
|
|
1980
|
-
|
|
2927
|
+
respawnInProgress = false;
|
|
2928
|
+
// Drain any child exit observed during the respawn. A crash of the
|
|
2929
|
+
// freshly-installed child that raced this respawn's completion was
|
|
2930
|
+
// deferred (respawnInProgress was set); handle it now that the latch
|
|
2931
|
+
// is clear. A pending exit for a superseded generation drops as stale.
|
|
2932
|
+
maybeHandleChildExit();
|
|
1981
2933
|
}
|
|
1982
2934
|
return attempt;
|
|
1983
2935
|
}
|
|
@@ -2026,6 +2978,20 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
2026
2978
|
},
|
|
2027
2979
|
});
|
|
2028
2980
|
}
|
|
2981
|
+
async function deliverCredentials(opts) {
|
|
2982
|
+
// The supervisor is the single producer of `credentials-updated` control
|
|
2983
|
+
// frames. Phase-guarded exactly like `deliverSources`: outside
|
|
2984
|
+
// starting/running the control sender points at a dying child, so a frame
|
|
2985
|
+
// would buffer behind the SIGTERM or write into a closed pipe. Rejecting
|
|
2986
|
+
// surfaces the race so the caller can retry once the recycle completes.
|
|
2987
|
+
if (state.phase !== "running" && state.phase !== "starting") {
|
|
2988
|
+
throw new Error(`supervisor: deliverCredentials called in phase ${state.phase}; expected starting/running`);
|
|
2989
|
+
}
|
|
2990
|
+
await state.controlSender.send({
|
|
2991
|
+
type: "credentials-updated",
|
|
2992
|
+
data: { delivery: opts.delivery },
|
|
2993
|
+
});
|
|
2994
|
+
}
|
|
2029
2995
|
function getCredentialsSnapshot() {
|
|
2030
2996
|
if (state.phase === "starting" || state.phase === "running") {
|
|
2031
2997
|
return state.credentialsSnapshot;
|
|
@@ -2040,6 +3006,8 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
2040
3006
|
recycle,
|
|
2041
3007
|
deliverSignal,
|
|
2042
3008
|
deliverSources,
|
|
3009
|
+
deliverCredentials,
|
|
3010
|
+
reEmitParkedCorrelations,
|
|
2043
3011
|
getCredentialsSnapshot,
|
|
2044
3012
|
};
|
|
2045
3013
|
}
|
|
@@ -2097,71 +3065,6 @@ async function pumpEvents(iter, onInferenceEvent) {
|
|
|
2097
3065
|
function defaultInProcessMailAuditRef(messageId, _rawMessage) {
|
|
2098
3066
|
return { store: "in-process", path: messageId };
|
|
2099
3067
|
}
|
|
2100
|
-
/**
|
|
2101
|
-
* Derive a stable message identifier from the raw bytes the bus
|
|
2102
|
-
* delivered. The RFC 2822 `Message-ID` header (if present) is the
|
|
2103
|
-
* canonical identifier the audit log surfaces as
|
|
2104
|
-
* `RunStarted.consumedMessageId`; downstream consumers join inbound
|
|
2105
|
-
* mail to workflow-run events on this value, so the header parse must
|
|
2106
|
-
* win when the sender emitted one. A message that lacks a
|
|
2107
|
-
* `Message-ID` header falls back to a sha256 of the raw bytes so
|
|
2108
|
-
* runs originating from non-RFC 2822 transports still receive a
|
|
2109
|
-
* deterministic identifier.
|
|
2110
|
-
*
|
|
2111
|
-
* The parser walks the message until the headers/body separator
|
|
2112
|
-
* (`CRLF CRLF` per RFC 2822 §2.1, with the lone-`LF` variant tolerated
|
|
2113
|
-
* to match common in-memory senders). Header-field unfolding follows
|
|
2114
|
-
* RFC 2822 §2.2.3: a continuation line begins with whitespace and
|
|
2115
|
-
* appends to the prior line. Header-name comparison is
|
|
2116
|
-
* case-insensitive per RFC 2822 §1.2.2.
|
|
2117
|
-
*/
|
|
2118
|
-
async function deriveMessageId(rawMessage) {
|
|
2119
|
-
const messageIdFromHeader = parseMessageIdHeader(rawMessage);
|
|
2120
|
-
if (messageIdFromHeader !== null) {
|
|
2121
|
-
return messageIdFromHeader;
|
|
2122
|
-
}
|
|
2123
|
-
const digest = await crypto.subtle.digest("SHA-256",
|
|
2124
|
-
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- ArrayBuffer-backed at the call site; Web Crypto's BufferSource type rejects Uint8Array<ArrayBufferLike> under TS 5.9 (microsoft/TypeScript#62240)
|
|
2125
|
-
rawMessage);
|
|
2126
|
-
return hexEncode(new Uint8Array(digest));
|
|
2127
|
-
}
|
|
2128
|
-
function parseMessageIdHeader(rawMessage) {
|
|
2129
|
-
const text = new TextDecoder("utf-8", { fatal: false }).decode(rawMessage);
|
|
2130
|
-
// Headers end at the first blank line. RFC 2822 mandates `CRLF CRLF`
|
|
2131
|
-
// but tolerate `LF LF` for callers that normalize line endings.
|
|
2132
|
-
let headerSection = text;
|
|
2133
|
-
const crlfBoundary = text.indexOf("\r\n\r\n");
|
|
2134
|
-
const lfBoundary = text.indexOf("\n\n");
|
|
2135
|
-
if (crlfBoundary >= 0 && (lfBoundary < 0 || crlfBoundary < lfBoundary)) {
|
|
2136
|
-
headerSection = text.slice(0, crlfBoundary);
|
|
2137
|
-
}
|
|
2138
|
-
else if (lfBoundary >= 0) {
|
|
2139
|
-
headerSection = text.slice(0, lfBoundary);
|
|
2140
|
-
}
|
|
2141
|
-
// Unfold continuation lines (a line starting with WSP belongs to
|
|
2142
|
-
// the prior header field).
|
|
2143
|
-
const lines = headerSection.split(/\r?\n/);
|
|
2144
|
-
const unfolded = [];
|
|
2145
|
-
for (const line of lines) {
|
|
2146
|
-
if (line.length > 0 && (line[0] === " " || line[0] === "\t")) {
|
|
2147
|
-
if (unfolded.length === 0)
|
|
2148
|
-
continue;
|
|
2149
|
-
unfolded[unfolded.length - 1] += " " + line.trim();
|
|
2150
|
-
continue;
|
|
2151
|
-
}
|
|
2152
|
-
unfolded.push(line);
|
|
2153
|
-
}
|
|
2154
|
-
for (const line of unfolded) {
|
|
2155
|
-
const colon = line.indexOf(":");
|
|
2156
|
-
if (colon < 0)
|
|
2157
|
-
continue;
|
|
2158
|
-
const name = line.slice(0, colon).trim().toLowerCase();
|
|
2159
|
-
if (name !== "message-id")
|
|
2160
|
-
continue;
|
|
2161
|
-
return line.slice(colon + 1).trim();
|
|
2162
|
-
}
|
|
2163
|
-
return null;
|
|
2164
|
-
}
|
|
2165
3068
|
/**
|
|
2166
3069
|
* Project the wire shape of a `terminal.event` upstream control frame
|
|
2167
3070
|
* into the workflow-vocabulary `TerminalRunEvent` discriminated union
|