@intx/workflow-host 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -14
- package/dist/adapters/mail-part-store.d.ts +46 -0
- package/dist/adapters/mail-part-store.js +251 -0
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +56 -65
- package/dist/adapters/spawn-child.d.ts +109 -44
- package/dist/adapters/spawn-child.js +77 -81
- package/dist/adapters/step-invoker.d.ts +52 -2
- package/dist/adapters/step-invoker.js +284 -37
- package/dist/adapters/substrate-mailbox-store.d.ts +80 -0
- package/dist/adapters/substrate-mailbox-store.js +404 -0
- package/dist/child/child-mailbox-reader.d.ts +10 -0
- package/dist/child/child-mailbox-reader.js +23 -0
- package/dist/child/credential-cell.d.ts +8 -0
- package/dist/child/credential-cell.js +66 -0
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/from-process-env.d.ts +12 -0
- package/dist/child/from-process-env.js +6 -0
- package/dist/child/index.d.ts +6 -2
- package/dist/child/index.js +4 -1
- package/dist/child/mailbox-mutation-bridge.d.ts +61 -0
- package/dist/child/mailbox-mutation-bridge.js +101 -0
- package/dist/child/mailbox-watch-registry.d.ts +17 -0
- package/dist/child/mailbox-watch-registry.js +61 -0
- package/dist/child/outbound-mail-bridge.d.ts +3 -2
- package/dist/child/outbound-mail-bridge.js +20 -32
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/pending-request.d.ts +89 -0
- package/dist/child/pending-request.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +170 -14
- package/dist/child/run-child.js +569 -155
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/substrate-write-bridge.d.ts +3 -2
- package/dist/child/substrate-write-bridge.js +21 -38
- package/dist/child/supervisor-backed-transport.d.ts +52 -6
- package/dist/child/supervisor-backed-transport.js +205 -62
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/child/warm-agent-cache.d.ts +44 -4
- package/dist/child/warm-agent-cache.js +41 -10
- package/dist/index.d.ts +6 -4
- package/dist/index.js +6 -4
- package/dist/ipc/control-channel.d.ts +151 -2
- package/dist/ipc/control-channel.js +222 -29
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/ipc/index.d.ts +1 -1
- package/dist/ipc/index.js +1 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/run-body-then-cleanup.d.ts +17 -0
- package/dist/run-body-then-cleanup.js +38 -0
- package/dist/seams/scheduler.d.ts +16 -6
- package/dist/seams/scheduler.js +87 -97
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +4 -8
- package/dist/supervisor/credentials.d.ts +28 -15
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +10 -3
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +14 -19
- package/dist/supervisor/run-event-recovery.d.ts +34 -0
- package/dist/supervisor/run-event-recovery.js +45 -0
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +106 -26
- package/dist/supervisor/supervisor.js +1903 -414
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +130 -0
- package/dist/supervisor/types.d.ts +180 -23
- package/dist/testing/change-notifier.d.ts +12 -0
- package/dist/testing/change-notifier.js +63 -0
- package/dist/testing/index.d.ts +8 -0
- package/dist/testing/index.js +16 -0
- package/dist/testing/log-capture.d.ts +52 -0
- package/dist/testing/log-capture.js +124 -0
- package/dist/testing/mail-bus.d.ts +22 -0
- package/dist/testing/mail-bus.js +78 -0
- package/dist/testing/memory-streams.d.ts +43 -0
- package/dist/testing/memory-streams.js +211 -0
- package/dist/testing/spawn-observer.d.ts +12 -0
- package/dist/testing/spawn-observer.js +36 -0
- package/dist/testing/stub-repo-store.d.ts +10 -0
- package/dist/testing/stub-repo-store.js +39 -0
- package/dist/testing/supervisor-reaper.d.ts +24 -0
- package/dist/testing/supervisor-reaper.js +49 -0
- package/dist/testing/upstream-frames.d.ts +47 -0
- package/dist/testing/upstream-frames.js +94 -0
- package/dist/workflow-definition-loader.d.ts +187 -0
- package/dist/workflow-definition-loader.js +422 -0
- package/package.json +18 -11
|
@@ -43,49 +43,90 @@ import { type } from "arktype";
|
|
|
43
43
|
import { getLogger } from "@intx/log";
|
|
44
44
|
import { sampleStructuralCounters, forceRepack, } from "./dispatch-attribution.js";
|
|
45
45
|
import { generateKeyPair } from "@intx/crypto";
|
|
46
|
-
import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed,
|
|
47
|
-
import { base64Decode, base64Encode,
|
|
46
|
+
import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed, scanRunsForBoot, readWorkflowRunLifecycle, replayProcessingToInbox as defaultReplayProcessingToInbox, StaleInboxEnqueueError, DEFAULT_CONSUMED_RETENTION_MS, } from "@intx/hub-sessions/substrate";
|
|
47
|
+
import { base64Decode, base64Encode, deriveMessageId, deriveWorkflowRunId, signalName, } from "@intx/types";
|
|
48
48
|
import { RepoId } from "@intx/types/sidecar";
|
|
49
49
|
import { createControlChannelSender, generateChannelId, generateHmacKey, receiveControlChannel, receiveEventChannel, } from "../ipc/index.js";
|
|
50
50
|
import { assembleCredentialsSnapshot, } from "./credentials.js";
|
|
51
51
|
import { commitCancelRequested } from "./cancel-signing.js";
|
|
52
|
+
import { commitRunFailed } from "./terminal-commit.js";
|
|
52
53
|
import { buildChildSpawnEnv } from "./spawn-env.js";
|
|
53
54
|
import { compactRunEvents } from "./run-event-compaction.js";
|
|
55
|
+
import { recoverInterruptedCompactions } from "./run-event-recovery.js";
|
|
56
|
+
import { decodeMail } from "@intx/mime";
|
|
57
|
+
import { commitMail, InvalidMailError } from "../adapters/mail-part-store.js";
|
|
58
|
+
import { mergeCredentialDelivery } from "../child/credential-cell.js";
|
|
59
|
+
import { createSubstrateMailboxStore, MAILBOX_INBOX_DIR, } from "../adapters/substrate-mailbox-store.js";
|
|
54
60
|
import { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, } from "./drain-timeout.js";
|
|
55
61
|
import { createRecyclePolicy, triggerRecycle, } from "./recycle.js";
|
|
56
62
|
import { createTerminalBroadcaster, } from "./terminal-broadcaster.js";
|
|
57
63
|
import { DEFAULT_KILL_TIMEOUT_MS, DEFAULT_READY_TIMEOUT_MS, defaultClearTimer, defaultSetTimer, killChildHandle, waitDeadline, } from "./child-termination.js";
|
|
58
64
|
const logger = getLogger(["workflow-host", "supervisor"]);
|
|
65
|
+
/** IMAP system flag marking a dispatched mailbox entry as read. */
|
|
66
|
+
const MAILBOX_FLAG_SEEN = "\\Seen";
|
|
59
67
|
/**
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
* `substrate.write.response` back to the child until the dispatch
|
|
63
|
-
* loop's `markConsumed` settles for the matching terminal event; an
|
|
64
|
-
* unbounded wait would chain into a child / runtime / dispatch loop
|
|
65
|
-
* deadlock if `markConsumed` never armed (bug in the dispatch loop, a
|
|
66
|
-
* torn-down cohort, a stalled inbox primitive). 30s sits between the
|
|
67
|
-
* recycle path's `DEFAULT_KILL_TIMEOUT_MS` (5s, a hard process-level
|
|
68
|
-
* kill cap) and `DEFAULT_DRAIN_TIMEOUT_MS` (60s, the per-deployment
|
|
69
|
-
* drain budget) -- generous enough to absorb a slow legitimate
|
|
70
|
-
* markConsumed, tight enough to surface a real deadlock long before
|
|
71
|
-
* the drainTimeout would otherwise mask it.
|
|
68
|
+
* Interchange keyword flag marking a mailbox entry the supervisor has dispatched
|
|
69
|
+
* as a workflow turn (a `trigger.fire` or `signal.deliver`).
|
|
72
70
|
*/
|
|
73
|
-
|
|
71
|
+
const MAILBOX_FLAG_PROCESSED = "$Processed";
|
|
74
72
|
/**
|
|
75
|
-
*
|
|
76
|
-
* `
|
|
77
|
-
*
|
|
78
|
-
* during a recycle, or a supervisor shutdown). Callers awaiting the
|
|
79
|
-
* resolved value receive an instance of this error so the failure mode
|
|
80
|
-
* is recognisable from a generic substrate-merge or markConsumed
|
|
81
|
-
* failure.
|
|
73
|
+
* IMAP system flag marking a mailbox entry for expunge. The warm agent sets it
|
|
74
|
+
* (via `mail_flag`) to consume a processed message; a subsequent `expunge`
|
|
75
|
+
* sweeps every entry carrying it out of the live INBOX.
|
|
82
76
|
*/
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
77
|
+
const MAILBOX_FLAG_DELETED = "\\Deleted";
|
|
78
|
+
/**
|
|
79
|
+
* Default crash-loop bound: the supervisor stops respawning and latches
|
|
80
|
+
* the deployment once the workflow-process child exits unexpectedly this
|
|
81
|
+
* many times within `DEFAULT_CRASH_LOOP_WINDOW_MS`. Overridable via
|
|
82
|
+
* `WorkflowSupervisorBindings.crashLoopMaxCount`.
|
|
83
|
+
*/
|
|
84
|
+
export const DEFAULT_CRASH_LOOP_MAX_COUNT = 3;
|
|
85
|
+
/**
|
|
86
|
+
* Default sliding window (ms) over which `DEFAULT_CRASH_LOOP_MAX_COUNT`
|
|
87
|
+
* unexpected exits latch the deployment. Overridable via
|
|
88
|
+
* `WorkflowSupervisorBindings.crashLoopWindowMs`.
|
|
89
|
+
*/
|
|
90
|
+
export const DEFAULT_CRASH_LOOP_WINDOW_MS = 60_000;
|
|
91
|
+
/**
|
|
92
|
+
* Default stable-run duration (ms): once a respawned child stays up this
|
|
93
|
+
* long, the crash counter resets so flapping followed by stability does
|
|
94
|
+
* not permanently latch. Overridable via
|
|
95
|
+
* `WorkflowSupervisorBindings.crashLoopStableResetMs`.
|
|
96
|
+
*/
|
|
97
|
+
export const DEFAULT_CRASH_LOOP_STABLE_RESET_MS = 60_000;
|
|
98
|
+
/**
|
|
99
|
+
* Default initial respawn backoff (ms): the wait before the first respawn
|
|
100
|
+
* after an unexpected exit. Overridable via
|
|
101
|
+
* `WorkflowSupervisorBindings.respawnBackoffInitialMs`.
|
|
102
|
+
*/
|
|
103
|
+
export const DEFAULT_RESPAWN_BACKOFF_INITIAL_MS = 1_000;
|
|
104
|
+
/**
|
|
105
|
+
* Default cap (ms) on the exponential respawn backoff. Kept below
|
|
106
|
+
* `DEFAULT_CRASH_LOOP_WINDOW_MS` so a slow flapper's crashes still fall
|
|
107
|
+
* within the window and latch the guard. Overridable via
|
|
108
|
+
* `WorkflowSupervisorBindings.respawnBackoffMaxMs`.
|
|
109
|
+
*/
|
|
110
|
+
export const DEFAULT_RESPAWN_BACKOFF_MAX_MS = 30_000;
|
|
111
|
+
/**
|
|
112
|
+
* Default watchdog for `reEmitParkedCorrelations`' wait on the child's
|
|
113
|
+
* `parked-correlations.response`. 30s is generous enough for a healthy child
|
|
114
|
+
* to enumerate its in-flight runs and load each parked snapshot, tight
|
|
115
|
+
* enough that a wedged-but-alive child does not hang the reconnect caller
|
|
116
|
+
* until some coarser timeout intervenes.
|
|
117
|
+
*/
|
|
118
|
+
export const DEFAULT_PARKED_QUERY_WATCHDOG_MS = 30_000;
|
|
119
|
+
/**
|
|
120
|
+
* Backstop for `waitForRunTerminalOrPark`. A dispatch waits here for the child
|
|
121
|
+
* to park or terminate the run before releasing `markConsumed`; a lost park
|
|
122
|
+
* wake or a wedged child would otherwise hang the deployment's dispatch loop
|
|
123
|
+
* forever. Five minutes is far beyond any healthy per-message dispatch (which
|
|
124
|
+
* settles in well under a second), so this never fires on a legitimately long
|
|
125
|
+
* run without also being a genuine fault -- and when it does fire it is logged
|
|
126
|
+
* loudly and fails the dispatch (the mail is left reclaimable, never consumed
|
|
127
|
+
* on the assumption the run made progress), not silently swallowed.
|
|
128
|
+
*/
|
|
129
|
+
export const TERMINAL_OR_PARK_BACKSTOP_MS = 300_000;
|
|
89
130
|
/**
|
|
90
131
|
* Construct a per-deployment supervisor. All host-specific
|
|
91
132
|
* dependencies are pulled in via `bindings`; nothing in the
|
|
@@ -93,23 +134,74 @@ export class MergeAbortedError extends Error {
|
|
|
93
134
|
*/
|
|
94
135
|
export function createWorkflowSupervisor(bindings) {
|
|
95
136
|
let state = { phase: "idle" };
|
|
137
|
+
// The live credential delivery to seed the child on every spawn and every
|
|
138
|
+
// pre-trigger barrier. Initialized from the deploy-time delivery and MUTATED
|
|
139
|
+
// by `deliverCredentials` on every runtime update, so a mid-life revocation or
|
|
140
|
+
// rotation is durable: the re-assertion sites read THIS mirror, not the frozen
|
|
141
|
+
// `bindings.credentialDelivery`. Reading the frozen delivery would re-add an
|
|
142
|
+
// evicted credential on the next spawn/barrier, and the warm-recycle path
|
|
143
|
+
// (seeded only by the barrier) would lose every runtime update. Normalized on
|
|
144
|
+
// init via the same merge the child applies, so the mirror matches the child's
|
|
145
|
+
// deduped cell from step zero. Lives outside `state` because `state` is a
|
|
146
|
+
// phase union replaced on every transition; this must survive them.
|
|
147
|
+
let currentCredentialDelivery = bindings.credentialDelivery !== undefined
|
|
148
|
+
? mergeCredentialDelivery(null, bindings.credentialDelivery, undefined)
|
|
149
|
+
: null;
|
|
96
150
|
/**
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
151
|
+
* ALL runIds the current child cohort is driving, regardless of
|
|
152
|
+
* who spawned them: supervisor-dispatched + self-discovered.
|
|
153
|
+
* Populated at `trigger.fire` time and when the child reports
|
|
154
|
+
* `resumed.runs`. Removed on terminal event or cohort teardown.
|
|
155
|
+
*
|
|
156
|
+
* Used by: `drain()` to arm one drainTimeout accumulator per run.
|
|
157
|
+
*/
|
|
158
|
+
const cohortRunIds = new Set();
|
|
159
|
+
/**
|
|
160
|
+
* Runs observed terminal in this supervisor process. The terminal control
|
|
161
|
+
* frame follows the durable event commit, but retaining the observation
|
|
162
|
+
* closes the short visibility window before the working tree reflects that
|
|
163
|
+
* commit. Terminal membership is permanent for a deployment: the stable
|
|
164
|
+
* top-level run is never cleared and fired again.
|
|
165
|
+
*/
|
|
166
|
+
const terminalRunIds = new Set();
|
|
167
|
+
/**
|
|
168
|
+
* Per-run input channel cache. When a long-lived run parks on an input
|
|
169
|
+
* signal, the child sends `park.notify` with `parkKind: "input"`. The
|
|
170
|
+
* supervisor stores the `correlationId` here so that subsequent mail
|
|
171
|
+
* deliveries can fire `signal.deliver` without re-reading the substrate.
|
|
172
|
+
* Cleared on terminal event or cohort abort.
|
|
173
|
+
*/
|
|
174
|
+
const runInputChannels = new Map();
|
|
175
|
+
/**
|
|
176
|
+
* Waiters for dispatch loops blocked on a `park.notify` for a
|
|
177
|
+
* specific runId. When `park.notify` arrives, the handler resolves
|
|
178
|
+
* the waiter so the dispatch loop re-evaluates routing.
|
|
102
179
|
*/
|
|
103
|
-
const
|
|
104
|
-
|
|
180
|
+
const parkNotifyWaiters = new Map();
|
|
181
|
+
/**
|
|
182
|
+
* Monotonic per-run INPUT-park generation. Bumped on every
|
|
183
|
+
* `park.notify(input)` for a runId. `waitForRunTerminalOrPark` captures a
|
|
184
|
+
* `sinceGen` before its caller's pre-wait awaits and returns `"parked"` when
|
|
185
|
+
* the generation later exceeds it, so the wait keys on the park EDGE rather
|
|
186
|
+
* than `runInputChannels`' LEVEL state: a park that fired during the pre-wait
|
|
187
|
+
* awaits (before the waiter armed, so `resolveParkNotifyWaiter` no-op'd) is
|
|
188
|
+
* still observed, and a stale `runInputChannels` entry from a prior run or
|
|
189
|
+
* incarnation cannot false-positive because it did not bump the generation
|
|
190
|
+
* past `sinceGen`. Incarnation-scoped: cleared with `parkNotifyWaiters` on
|
|
191
|
+
* recycle/teardown so a reused runId cannot carry a stale captured generation
|
|
192
|
+
* across incarnations.
|
|
193
|
+
*/
|
|
194
|
+
const parkGenerations = new Map();
|
|
195
|
+
// D2 attribution (measurement-only): the messageId the dispatch loop is
|
|
105
196
|
// currently servicing. Set at `dispatch-start`, cleared after
|
|
106
197
|
// `reply-produced`. The dispatch loop is strictly serial (one message
|
|
107
198
|
// in flight at a time -- the sustained interactive case the bench
|
|
108
|
-
// drives), so a child-proxied
|
|
109
|
-
//
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
|
|
199
|
+
// drives), so a child-proxied `substrate.write.request` is unambiguously
|
|
200
|
+
// attributable to this message. Both the WAL leg (whose `agent-state/...`
|
|
201
|
+
// prefix names no run) and the run-event leg (whose `runs/<runId>/events/`
|
|
202
|
+
// prefix names only the stable per-deployment run id, not the message)
|
|
203
|
+
// take their per-message key from here.
|
|
204
|
+
let currentDispatchMessageId = null;
|
|
113
205
|
/**
|
|
114
206
|
* Per-run drainTimeout accumulators armed by `drain()`. Held so
|
|
115
207
|
* `shutdown()` can stop every accumulator cleanly before tearing
|
|
@@ -134,21 +226,21 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
134
226
|
}
|
|
135
227
|
});
|
|
136
228
|
const drainTimeoutMs = bindings.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
|
|
137
|
-
const
|
|
229
|
+
const parkedQueryWatchdogMs = bindings.parkedQueryWatchdogMs ?? DEFAULT_PARKED_QUERY_WATCHDOG_MS;
|
|
138
230
|
// Pure observability: invoke the dispatch-timing hook (when wired) at
|
|
139
231
|
// the two per-message boundaries the 4.7 latency gate brackets. A
|
|
140
232
|
// throwing observer is swallowed and logged so a benchmark hook bug
|
|
141
233
|
// cannot wedge the dispatch loop.
|
|
142
|
-
function emitDispatchTiming(
|
|
234
|
+
function emitDispatchTiming(messageId, marker, atMs) {
|
|
143
235
|
const observer = bindings.onDispatchTiming;
|
|
144
236
|
if (observer === undefined)
|
|
145
237
|
return;
|
|
146
238
|
try {
|
|
147
|
-
observer({ kind: "roundtrip",
|
|
239
|
+
observer({ kind: "roundtrip", messageId, marker, atMs });
|
|
148
240
|
}
|
|
149
241
|
catch (cause) {
|
|
150
242
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
151
|
-
logger.warn `onDispatchTiming observer threw for ${
|
|
243
|
+
logger.warn `onDispatchTiming observer threw for ${messageId} (${marker}): ${message}`;
|
|
152
244
|
}
|
|
153
245
|
}
|
|
154
246
|
// D2 per-leg attribution (measurement-only). Emits a paired
|
|
@@ -160,14 +252,14 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
160
252
|
// observability: a throwing observer is swallowed + logged so a
|
|
161
253
|
// benchmark hook bug cannot wedge dispatch, and no clock or directory
|
|
162
254
|
// is sampled when the observer is unwired.
|
|
163
|
-
function legMarkStart(
|
|
255
|
+
function legMarkStart(messageId, leg) {
|
|
164
256
|
if (bindings.onDispatchTiming === undefined)
|
|
165
257
|
return 0;
|
|
166
258
|
const atMs = performance.now();
|
|
167
259
|
try {
|
|
168
260
|
bindings.onDispatchTiming({
|
|
169
261
|
kind: "leg",
|
|
170
|
-
|
|
262
|
+
messageId,
|
|
171
263
|
leg,
|
|
172
264
|
phase: "start",
|
|
173
265
|
atMs,
|
|
@@ -175,11 +267,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
175
267
|
}
|
|
176
268
|
catch (cause) {
|
|
177
269
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
178
|
-
logger.warn `onDispatchTiming leg observer threw for ${
|
|
270
|
+
logger.warn `onDispatchTiming leg observer threw for ${messageId} (${leg} start): ${message}`;
|
|
179
271
|
}
|
|
180
272
|
return atMs;
|
|
181
273
|
}
|
|
182
|
-
function legMarkEnd(
|
|
274
|
+
function legMarkEnd(messageId, leg) {
|
|
183
275
|
const observer = bindings.onDispatchTiming;
|
|
184
276
|
if (observer === undefined)
|
|
185
277
|
return;
|
|
@@ -193,12 +285,12 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
193
285
|
// surface it on the log and emit the end mark without counters so
|
|
194
286
|
// the timing slope is still recoverable.
|
|
195
287
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
196
|
-
logger.warn `structural-counter sample failed for ${
|
|
288
|
+
logger.warn `structural-counter sample failed for ${messageId} (${leg}): ${message}`;
|
|
197
289
|
}
|
|
198
290
|
try {
|
|
199
291
|
observer({
|
|
200
292
|
kind: "leg",
|
|
201
|
-
|
|
293
|
+
messageId,
|
|
202
294
|
leg,
|
|
203
295
|
phase: "end",
|
|
204
296
|
atMs,
|
|
@@ -207,7 +299,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
207
299
|
}
|
|
208
300
|
catch (cause) {
|
|
209
301
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
210
|
-
logger.warn `onDispatchTiming leg observer threw for ${
|
|
302
|
+
logger.warn `onDispatchTiming leg observer threw for ${messageId} (${leg} end): ${message}`;
|
|
211
303
|
}
|
|
212
304
|
}
|
|
213
305
|
// §10c forced-repack A/B (measurement-only). Absent toggle => never
|
|
@@ -244,27 +336,26 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
244
336
|
}
|
|
245
337
|
/**
|
|
246
338
|
* Classify a child-proxied `substrate.write.request` into the D2 leg it
|
|
247
|
-
* represents, plus the
|
|
248
|
-
* `runs/<runId>/events/` is the run-event bracket commit
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
*
|
|
252
|
-
*
|
|
253
|
-
*
|
|
339
|
+
* represents, plus the messageId the per-message OLS fit groups on.
|
|
340
|
+
* `runs/<runId>/events/` is the run-event bracket commit and `agent-state/...`
|
|
341
|
+
* is the D1 conversation WAL append; neither prefix carries the message
|
|
342
|
+
* identity (the run-event prefix names only the stable per-deployment run
|
|
343
|
+
* id), so both are attributed to the dispatch loop's current serial
|
|
344
|
+
* message. Any other prefix is an unmarked proxied write. Returns `null`
|
|
345
|
+
* when no observer is wired (so the supervisor samples nothing), when the
|
|
346
|
+
* prefix is not an attributed leg, or when no message is in flight to
|
|
347
|
+
* attribute it to.
|
|
254
348
|
*/
|
|
255
349
|
function classifyProxiedWriteLeg(preservePrefix) {
|
|
256
350
|
if (bindings.onDispatchTiming === undefined)
|
|
257
351
|
return null;
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
return { leg: "runevent", runId };
|
|
352
|
+
if (currentDispatchMessageId === null)
|
|
353
|
+
return null;
|
|
354
|
+
if (/^runs\/[^/]+\/events\/$/.test(preservePrefix)) {
|
|
355
|
+
return { leg: "runevent", messageId: currentDispatchMessageId };
|
|
263
356
|
}
|
|
264
357
|
if (preservePrefix.startsWith("agent-state/")) {
|
|
265
|
-
|
|
266
|
-
return { leg: "wal", runId: currentDispatchRunId };
|
|
267
|
-
}
|
|
358
|
+
return { leg: "wal", messageId: currentDispatchMessageId };
|
|
268
359
|
}
|
|
269
360
|
return null;
|
|
270
361
|
}
|
|
@@ -286,7 +377,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
286
377
|
const deriveMailAuditRef = bindings.deriveMailAuditRef ?? defaultInProcessMailAuditRef;
|
|
287
378
|
const defaultInboxWritePrincipal = {
|
|
288
379
|
kind: "supervisor",
|
|
289
|
-
|
|
380
|
+
anchorRunId: bindings.anchorRunId,
|
|
290
381
|
};
|
|
291
382
|
const inboxWritePrincipal = bindings.inboxWritePrincipal ?? defaultInboxWritePrincipal;
|
|
292
383
|
// Resolve the consumed-dedup retention horizon once at the bindings
|
|
@@ -302,6 +393,17 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
302
393
|
const readyTimeoutMs = bindings.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS;
|
|
303
394
|
const readySetTimer = bindings.setTimer ?? defaultSetTimer;
|
|
304
395
|
const readyClearTimer = bindings.clearTimer ?? defaultClearTimer;
|
|
396
|
+
// Resolve the crash-loop guard bounds once at the bindings edge (the
|
|
397
|
+
// layer that owns the operator config). The stable-run reset timer
|
|
398
|
+
// reuses the same injectable `setTimer`/`clearTimer` pair as the ready
|
|
399
|
+
// handshake, and the wall clock reuses the recycle-policy `now` seam, so
|
|
400
|
+
// tests drive the whole guard deterministically through existing knobs.
|
|
401
|
+
const crashLoopMaxCount = bindings.crashLoopMaxCount ?? DEFAULT_CRASH_LOOP_MAX_COUNT;
|
|
402
|
+
const crashLoopWindowMs = bindings.crashLoopWindowMs ?? DEFAULT_CRASH_LOOP_WINDOW_MS;
|
|
403
|
+
const crashLoopStableResetMs = bindings.crashLoopStableResetMs ?? DEFAULT_CRASH_LOOP_STABLE_RESET_MS;
|
|
404
|
+
const respawnBackoffInitialMs = bindings.respawnBackoffInitialMs ?? DEFAULT_RESPAWN_BACKOFF_INITIAL_MS;
|
|
405
|
+
const respawnBackoffMaxMs = bindings.respawnBackoffMaxMs ?? DEFAULT_RESPAWN_BACKOFF_MAX_MS;
|
|
406
|
+
const crashNow = bindings.recyclePolicyNow ?? defaultNow;
|
|
305
407
|
/**
|
|
306
408
|
* Resolved on every successful `enqueueInbox`; the dispatch loop
|
|
307
409
|
* awaits this promise after a null dequeue so it returns to
|
|
@@ -332,12 +434,482 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
332
434
|
*/
|
|
333
435
|
let spawnContext = null;
|
|
334
436
|
let recyclePolicy = null;
|
|
335
|
-
|
|
437
|
+
// Mutual-exclusion latch shared by every respawn path (operator/policy/
|
|
438
|
+
// self recycle and crash-respawn). `runRespawn` owns the set/clear; each
|
|
439
|
+
// caller owns the contention read because the two paths diverge on
|
|
440
|
+
// contention (recycle throws, crash-respawn declines silently).
|
|
441
|
+
let respawnInProgress = false;
|
|
442
|
+
// Monotonic child-cohort generation, bumped atomically with each
|
|
443
|
+
// transition to `running` (initial spawn and every respawn's
|
|
444
|
+
// `installNewChild`). The exit-watcher captures the generation it was
|
|
445
|
+
// armed for; a watcher whose generation is no longer current is stale
|
|
446
|
+
// (a newer child already superseded it) and is ignored. Generation plus
|
|
447
|
+
// the phase guard plus `respawnInProgress` together classify every child
|
|
448
|
+
// exit as planned or unexpected without a separate per-handle marker.
|
|
449
|
+
let childGeneration = 0;
|
|
450
|
+
// An unexpected child exit that could not be handled the instant it was
|
|
451
|
+
// observed (a respawn was already in flight). Recorded generation-tagged
|
|
452
|
+
// and drained by `maybeHandleChildExit` once `runRespawn` clears the
|
|
453
|
+
// latch; a pending exit for a superseded generation is dropped as stale.
|
|
454
|
+
let pendingChildExit = null;
|
|
455
|
+
// Monotonic timestamps (ms, from `crashNow`) of recent unexpected exits,
|
|
456
|
+
// pruned to `crashLoopWindowMs`. The crash-loop guard latches when the
|
|
457
|
+
// count within the window reaches `crashLoopMaxCount`.
|
|
458
|
+
const crashTimestamps = [];
|
|
459
|
+
// Handle for the armed stable-run reset timer (or null). Cleared and
|
|
460
|
+
// re-armed on every transition to `running`; cleared on teardown.
|
|
461
|
+
let stableRunResetTimer = null;
|
|
462
|
+
// Current exponential respawn backoff (ms). Advances by doubling on each
|
|
463
|
+
// respawn (capped at `respawnBackoffMaxMs`) and resets to the initial
|
|
464
|
+
// value when a respawned child runs stably. See `waitRespawnBackoff`.
|
|
465
|
+
let respawnBackoffMs = respawnBackoffInitialMs;
|
|
466
|
+
// Every armed backoff wait: the injected timer handle plus the promise
|
|
467
|
+
// resolver, so a shutdown can cancel all of them and unblock the parked
|
|
468
|
+
// respawn coroutines (each then bails at its phase/generation re-check).
|
|
469
|
+
// A SET, not a single slot: more than one coroutine can be parked at once
|
|
470
|
+
// -- a recycle can install a fresh, live child DURING one crash's backoff
|
|
471
|
+
// wait, and that child crashing parks a second coroutine. A single slot
|
|
472
|
+
// would drop the earlier one's timer and leak it past shutdown.
|
|
473
|
+
const respawnBackoffWaits = new Set();
|
|
474
|
+
// =====================================================================
|
|
475
|
+
// CRASH-RESPAWN -- auto-recovery from an unexpected workflow-process exit
|
|
476
|
+
// =====================================================================
|
|
477
|
+
//
|
|
478
|
+
// An UNEXPECTED child exit (crash, OOM, panic, signal -- not a
|
|
479
|
+
// supervisor-initiated shutdown, drain, or recycle) is detected by
|
|
480
|
+
// watching `handle.exited`, the only universal death signal: a clean
|
|
481
|
+
// process death ends the IPC channel readers without a protocol-level
|
|
482
|
+
// crash callback (see `onChildCrash`). `armChildForRunning` arms that
|
|
483
|
+
// watcher and bumps `childGeneration` on every transition to `running`;
|
|
484
|
+
// an exit is classified as unexpected iff its generation is still current
|
|
485
|
+
// AND the phase is still `running` (a planned kill is observed in a
|
|
486
|
+
// non-running phase, or against a superseded generation).
|
|
487
|
+
//
|
|
488
|
+
// On an unexpected exit `handleUnexpectedChildExit` runs: it records the
|
|
489
|
+
// crash against the crash-loop guard and either
|
|
490
|
+
// - RESPAWNS -- after an exponential backoff (`waitRespawnBackoff`,
|
|
491
|
+
// 1s doubling to a 30s cap), it reuses the recycle path's
|
|
492
|
+
// `runRespawn` with a no-op drain (the child is already dead) so the
|
|
493
|
+
// stranded-mail replay and fresh-child spawn happen exactly as a
|
|
494
|
+
// recycle's do; or
|
|
495
|
+
// - LATCHES -- once the guard trips (`crashLoopMaxCount` exits within
|
|
496
|
+
// `crashLoopWindowMs`), it tears down to the terminal `crash-looping`
|
|
497
|
+
// phase and commits a `RunFailed` for the deployment's run so the
|
|
498
|
+
// crash-loop is durably observable as a failed run status.
|
|
499
|
+
//
|
|
500
|
+
// A respawned child that survives `crashLoopStableResetMs`
|
|
501
|
+
// (`armStableRunResetTimer`) resets both the crash counter and the
|
|
502
|
+
// backoff. The backoff wait sits OUTSIDE the `respawnInProgress` latch,
|
|
503
|
+
// so the generation re-check after the wait is what stops a recycle that
|
|
504
|
+
// installed a fresh cohort mid-wait from being respawned a second time.
|
|
505
|
+
// Full policy: `packages/workflow-host/README.md` "Respawn policy".
|
|
506
|
+
// A protocol violation on a live cohort's control or event channel. The
|
|
507
|
+
// channel receiver ends its iterator and invokes this; a clean process
|
|
508
|
+
// death does NOT (it just ends the reader with no crash callback), so
|
|
509
|
+
// the exit-watcher on `handle.exited` is the universal death signal and
|
|
510
|
+
// this path only covers the frame-level violations the receiver detects.
|
|
511
|
+
// On the live (`running`) cohort, force the child down so its `exited`
|
|
512
|
+
// fires the exit-watcher and the crash flows through the SAME
|
|
513
|
+
// respawn/crash-loop path a clean death takes -- a violation that emits a
|
|
514
|
+
// garbage frame just before dying must not divert to `stopped` while a
|
|
515
|
+
// clean SIGKILL respawns. In any other phase the owning lifecycle path
|
|
516
|
+
// (spawn handshake, recycle reap, shutdown) owns teardown.
|
|
336
517
|
function onChildCrash(reason) {
|
|
337
|
-
|
|
338
|
-
|
|
518
|
+
if (state.phase === "running") {
|
|
519
|
+
logger.error `workflow-process channel crash on live cohort; forcing child down to respawn: ${reason}`;
|
|
520
|
+
state.handle.kill();
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
logger.error `workflow-process channel crash: ${reason}`;
|
|
524
|
+
// Only a live, registered supervisor driven down by a channel crash is a
|
|
525
|
+
// self-termination the host must reclaim, and that is `recycling`:
|
|
526
|
+
// `running` took the kill branch above (its exit reaches the crash-loop
|
|
527
|
+
// latch, which carries its own flag), `starting` is the pre-registration
|
|
528
|
+
// initial spawn handshake whose failure the deploy unwind owns, and
|
|
529
|
+
// `stopping` is a teardown already in flight (a host `shutdown()`, or a
|
|
530
|
+
// self-termination already firing). An allowlist, not a denylist, so a
|
|
531
|
+
// future phase defaults to no self-terminate rather than a spurious one.
|
|
532
|
+
const selfTerminated = state.phase === "recycling";
|
|
533
|
+
void shutdownInternal({ reason, selfTerminated });
|
|
534
|
+
}
|
|
535
|
+
// Prune crash timestamps older than the sliding window relative to `nowMs`.
|
|
536
|
+
function pruneCrashTimestamps(nowMs) {
|
|
537
|
+
const cutoff = nowMs - crashLoopWindowMs;
|
|
538
|
+
while (true) {
|
|
539
|
+
const oldest = crashTimestamps[0];
|
|
540
|
+
if (oldest === undefined || oldest > cutoff)
|
|
541
|
+
break;
|
|
542
|
+
crashTimestamps.shift();
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
function clearStableRunResetTimer() {
|
|
546
|
+
if (stableRunResetTimer !== null) {
|
|
547
|
+
readyClearTimer(stableRunResetTimer);
|
|
548
|
+
stableRunResetTimer = null;
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
// Arm (or re-arm) the stable-run reset for the cohort that just reached
|
|
552
|
+
// `running`. If the child is still that same live cohort when the timer
|
|
553
|
+
// fires, the run has been stable for `crashLoopStableResetMs` and the
|
|
554
|
+
// crash counter is cleared so a flap-then-stabilize sequence does not
|
|
555
|
+
// latch. A crash before the timer fires re-arms it for the next cohort;
|
|
556
|
+
// teardown clears it.
|
|
557
|
+
function armStableRunResetTimer(generation) {
|
|
558
|
+
clearStableRunResetTimer();
|
|
559
|
+
stableRunResetTimer = readySetTimer(() => {
|
|
560
|
+
stableRunResetTimer = null;
|
|
561
|
+
if (generation === childGeneration && state.phase === "running") {
|
|
562
|
+
// The cohort ran stably: reset both the crash counter and the
|
|
563
|
+
// exponential backoff so a flap followed by stability starts over.
|
|
564
|
+
crashTimestamps.length = 0;
|
|
565
|
+
respawnBackoffMs = respawnBackoffInitialMs;
|
|
566
|
+
}
|
|
567
|
+
}, crashLoopStableResetMs);
|
|
568
|
+
}
|
|
569
|
+
// Wait the current respawn backoff before a respawn. The wait uses the
|
|
570
|
+
// injected timer so tests drive it deterministically, and is cancellable:
|
|
571
|
+
// `cancelRespawnBackoffWaits` (called on shutdown) clears the timer and
|
|
572
|
+
// resolves the promise early so the parked respawn coroutine unblocks and
|
|
573
|
+
// bails at its phase/generation re-check rather than sleeping out a full
|
|
574
|
+
// 30s backoff against a torn-down supervisor.
|
|
575
|
+
function waitRespawnBackoff(ms) {
|
|
576
|
+
return new Promise((resolve) => {
|
|
577
|
+
const entry = {
|
|
578
|
+
timer: null,
|
|
579
|
+
resolve,
|
|
580
|
+
};
|
|
581
|
+
entry.timer = readySetTimer(() => {
|
|
582
|
+
respawnBackoffWaits.delete(entry);
|
|
583
|
+
resolve();
|
|
584
|
+
}, ms);
|
|
585
|
+
respawnBackoffWaits.add(entry);
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
// Cancel every armed backoff wait. Idempotent: an empty set is a no-op,
|
|
589
|
+
// and each entry's own fire path has already removed it.
|
|
590
|
+
function cancelRespawnBackoffWaits() {
|
|
591
|
+
for (const entry of respawnBackoffWaits) {
|
|
592
|
+
readyClearTimer(entry.timer);
|
|
593
|
+
entry.resolve();
|
|
594
|
+
}
|
|
595
|
+
respawnBackoffWaits.clear();
|
|
596
|
+
}
|
|
597
|
+
// Bump the generation and arm the `handle.exited` watcher for a child
|
|
598
|
+
// that just became the `running` cohort. Called atomically with the
|
|
599
|
+
// state swap to `running` (no await between the swap and this call).
|
|
600
|
+
// `exited` resolving OR rejecting both mean the process is gone. The
|
|
601
|
+
// stable-run reset timer is NOT armed here: on the pristine first spawn
|
|
602
|
+
// there is no crash counter to reset, so it is armed only after a
|
|
603
|
+
// respawn (see `handleUnexpectedChildExit`).
|
|
604
|
+
function armChildForRunning(handle) {
|
|
605
|
+
childGeneration += 1;
|
|
606
|
+
const generation = childGeneration;
|
|
607
|
+
void handle.exited
|
|
608
|
+
.then(() => {
|
|
609
|
+
onChildExited(generation, "workflow-process child exited");
|
|
610
|
+
})
|
|
611
|
+
.catch(() => {
|
|
612
|
+
onChildExited(generation, "workflow-process child exited (exit promise rejected)");
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
// Record a child exit and try to act on it. Stale exits (a newer cohort
|
|
616
|
+
// already installed) are dropped.
|
|
617
|
+
function onChildExited(generation, reason) {
|
|
618
|
+
if (generation !== childGeneration)
|
|
619
|
+
return;
|
|
620
|
+
pendingChildExit = { generation, reason };
|
|
621
|
+
maybeHandleChildExit();
|
|
339
622
|
}
|
|
340
|
-
|
|
623
|
+
// Drain a recorded child exit if the supervisor is in a state to act on
|
|
624
|
+
// it. Declines (leaving the exit pending) while a respawn is in flight;
|
|
625
|
+
// `runRespawn`'s `finally` re-invokes this after clearing the latch so
|
|
626
|
+
// an exit observed during the respawn is not lost. A pending exit for a
|
|
627
|
+
// superseded generation, or one observed after the deployment left the
|
|
628
|
+
// `running` phase (shutdown/recycle owns that teardown), is dropped.
|
|
629
|
+
function maybeHandleChildExit() {
|
|
630
|
+
if (respawnInProgress)
|
|
631
|
+
return;
|
|
632
|
+
const pending = pendingChildExit;
|
|
633
|
+
if (pending === null)
|
|
634
|
+
return;
|
|
635
|
+
if (pending.generation !== childGeneration) {
|
|
636
|
+
pendingChildExit = null;
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
if (state.phase !== "running")
|
|
640
|
+
return;
|
|
641
|
+
pendingChildExit = null;
|
|
642
|
+
void handleUnexpectedChildExit(pending.reason).catch((cause) => {
|
|
643
|
+
// Fire-and-forget context (the exit-watcher has no caller to catch
|
|
644
|
+
// this). `runRespawn` already ran its own failure teardown to a
|
|
645
|
+
// terminal state before rethrowing, so the deployment is not wedged;
|
|
646
|
+
// surface the failure and stop.
|
|
647
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
648
|
+
logger.error `crash-respawn handling failed; deployment torn down: ${message}`;
|
|
649
|
+
});
|
|
650
|
+
}
|
|
651
|
+
// Handle one unexpected child exit: record it against the crash-loop
|
|
652
|
+
// guard and either latch the deployment (guard tripped) or respawn.
|
|
653
|
+
async function handleUnexpectedChildExit(reason) {
|
|
654
|
+
if (state.phase !== "running" || spawnContext === null) {
|
|
655
|
+
// Raced a shutdown/recycle between the drain check and here; the
|
|
656
|
+
// owning lifecycle path handles teardown.
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
// A cohort that just crashed did not earn a stability reward: disarm its
|
|
660
|
+
// stable-run reset timer up front so it cannot fire during the backoff
|
|
661
|
+
// wait below (where the phase is still `running` and the generation is
|
|
662
|
+
// still this dead cohort's -- exactly the state the timer's own guard
|
|
663
|
+
// reads as "alive and stable") and wrongly clear the crash counter.
|
|
664
|
+
clearStableRunResetTimer();
|
|
665
|
+
// Capture the crashing cohort's generation. If a recycle or policy
|
|
666
|
+
// respawn installs a fresh cohort while the backoff wait below is
|
|
667
|
+
// parked, the generation advances and this handler must NOT respawn
|
|
668
|
+
// that healthy cohort -- the post-wait guard bails on the mismatch.
|
|
669
|
+
const armedGeneration = childGeneration;
|
|
670
|
+
const nowMs = crashNow();
|
|
671
|
+
crashTimestamps.push(nowMs);
|
|
672
|
+
pruneCrashTimestamps(nowMs);
|
|
673
|
+
if (crashTimestamps.length >= crashLoopMaxCount) {
|
|
674
|
+
// Crash-loop latch. The deployment stops respawning and tears down
|
|
675
|
+
// to the terminal `crash-looping` state so a rapidly-flapping child
|
|
676
|
+
// cannot saturate the host.
|
|
677
|
+
const crashCount = crashTimestamps.length;
|
|
678
|
+
logger.error `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms; stopping the deployment (${reason})`;
|
|
679
|
+
await shutdownInternal({
|
|
680
|
+
reason: `crash-loop: ${reason}`,
|
|
681
|
+
terminalPhase: "crash-looping",
|
|
682
|
+
selfTerminated: true,
|
|
683
|
+
});
|
|
684
|
+
// Commit the RunFailed tombstone AFTER teardown: shutdownInternal has
|
|
685
|
+
// quiesced the drain accumulators (stop + await disposed), so the
|
|
686
|
+
// run-event tree is settled and no escalation commit races this write.
|
|
687
|
+
// This RunFailed is the SOLE durable, externally-queryable signal of
|
|
688
|
+
// the crash-loop (the `crash-looping` phase is in-memory only), so a
|
|
689
|
+
// failure to write it is logged loudly rather than swallowed. Best-
|
|
690
|
+
// effort: the deployment is already terminal, so the write not landing
|
|
691
|
+
// costs observability, not correctness.
|
|
692
|
+
try {
|
|
693
|
+
// `anchorRunId` and the tombstone's `runId` are DISTINCT ids and must
|
|
694
|
+
// not be conflated. `bindings.anchorRunId` is the workflow-run repo
|
|
695
|
+
// slug (`deriveWorkflowRunRepoId`), which the supervisor principal's
|
|
696
|
+
// authz check keys on (`repoId.id === anchorRunId`). The RunFailed must
|
|
697
|
+
// land on the deployment's ONE top-level run, whose id is the local
|
|
698
|
+
// part of the deployment's mail address (`deriveWorkflowRunId`) -- the
|
|
699
|
+
// same id the dispatch loop writes every run event under. For a domain
|
|
700
|
+
// like `integration.interchange` the two ids differ (the repo slug
|
|
701
|
+
// carries a domain suffix), so writing the tombstone under the repo
|
|
702
|
+
// slug would strand it in a run subtree no reader consults.
|
|
703
|
+
await commitRunFailed({
|
|
704
|
+
substrate: bindings.repoStore,
|
|
705
|
+
repoId: bindings.workflowRunRepoId,
|
|
706
|
+
ref: bindings.workflowRunRef,
|
|
707
|
+
anchorRunId: bindings.anchorRunId,
|
|
708
|
+
runId: deriveWorkflowRunId(bindings.deploymentMailAddress),
|
|
709
|
+
at: new Date(nowMs).toISOString(),
|
|
710
|
+
message: `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms`,
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
catch (cause) {
|
|
714
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
715
|
+
logger.error `crash-loop RunFailed commit failed; deployment has no durable failure tombstone: ${message}`;
|
|
716
|
+
}
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
const thisBackoffMs = respawnBackoffMs;
|
|
720
|
+
logger.warn `workflow-process exited unexpectedly; respawning after ${String(thisBackoffMs)}ms backoff (${reason})`;
|
|
721
|
+
await waitRespawnBackoff(thisBackoffMs);
|
|
722
|
+
// A recycle/policy respawn, or a shutdown, may have run during the
|
|
723
|
+
// backoff wait. Bail unless THIS dead cohort is still the current
|
|
724
|
+
// running one: the generation guard prevents respawning a healthy
|
|
725
|
+
// cohort a recycle just installed, and the phase guard prevents acting
|
|
726
|
+
// after teardown. No await separates this re-check from `runRespawn`'s
|
|
727
|
+
// synchronous `respawnInProgress` set, so nothing can interleave.
|
|
728
|
+
if (childGeneration !== armedGeneration ||
|
|
729
|
+
state.phase !== "running" ||
|
|
730
|
+
spawnContext === null) {
|
|
731
|
+
// The backoff above announced a respawn before parking. Report the bail
|
|
732
|
+
// too, so a deployment that stops respawning does not leave an operator
|
|
733
|
+
// with a promised respawn and no record of what happened to it.
|
|
734
|
+
logger.info `respawn backoff elapsed but the crashed cohort is no longer the running one; skipping respawn (phase=${state.phase}, armed generation ${String(armedGeneration)}, current generation ${String(childGeneration)}): ${reason}`;
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
const priorRunning = state;
|
|
738
|
+
const priorContext = spawnContext;
|
|
739
|
+
// Advance the backoff only now that a respawn is actually happening, so
|
|
740
|
+
// a bail above does not inflate the wait for a respawn that never ran.
|
|
741
|
+
respawnBackoffMs = Math.min(respawnBackoffMs * 2, respawnBackoffMaxMs);
|
|
742
|
+
await runRespawn({
|
|
743
|
+
origin: "crash",
|
|
744
|
+
reason,
|
|
745
|
+
prior: priorRunning,
|
|
746
|
+
priorContext,
|
|
747
|
+
// The child is already dead: there is nothing to drain, and
|
|
748
|
+
// `runRespawn`'s kill step is a no-op on a dead handle. The replay
|
|
749
|
+
// step still runs, moving any mail stranded mid-flight back to the
|
|
750
|
+
// inbox tail before dispatch resumes.
|
|
751
|
+
drain: async () => undefined,
|
|
752
|
+
});
|
|
753
|
+
// The respawned child is now the running cohort. Arm the stable-run
|
|
754
|
+
// reset against its generation: if it survives `crashLoopStableResetMs`
|
|
755
|
+
// the crash counter and backoff reset, so a flap followed by stability
|
|
756
|
+
// does not latch on a later, unrelated crash.
|
|
757
|
+
armStableRunResetTimer(childGeneration);
|
|
758
|
+
}
|
|
759
|
+
// Eager per-run mailbox (§3b inbound). On arrival the supervisor commits each
|
|
760
|
+
// fresh inbound message into the deployment's substrate-backed INBOX and fires
|
|
761
|
+
// a one-way `mailbox.notify` to the child, so the warm agent's `watch` /
|
|
762
|
+
// `mail_wait` observes the arrival mid-turn -- decoupled from the FIFO claim-
|
|
763
|
+
// check dispatch that resolves a run's step input. The supervisor is the sole
|
|
764
|
+
// mailbox writer; the store is its long-lived in-memory mirror over the
|
|
765
|
+
// committed `mailbox/INBOX/` subtree, constructed lazily on the first arrival.
|
|
766
|
+
const mailboxWritePrincipal = {
|
|
767
|
+
kind: "supervisor",
|
|
768
|
+
anchorRunId: bindings.anchorRunId,
|
|
769
|
+
};
|
|
770
|
+
let mailboxStore = null;
|
|
771
|
+
// Claim-check messageId -> assigned mailbox uid, so a message dispatched as a
|
|
772
|
+
// turn can be flagged \Seen/$Processed by uid. In-memory only: a missing entry
|
|
773
|
+
// (a restart, an arrival whose eager commit failed, or an already-processed
|
|
774
|
+
// message whose entry was pruned) skips the flag mark, which is a cosmetic
|
|
775
|
+
// IMAP flag, never a delivery guarantee. `markMailboxProcessed` prunes an
|
|
776
|
+
// entry once its mark completes, so the map holds only messages awaiting the
|
|
777
|
+
// flag mark rather than growing for the deployment's life.
|
|
778
|
+
const mailboxUidByMessageId = new Map();
|
|
779
|
+
// Serializes every mailbox mutation (lazy construction, the arrival
|
|
780
|
+
// append+flush, the dispatch flag mark) so concurrent arrivals and a fire-and-
|
|
781
|
+
// forget flag mark never interleave against the shared in-memory mirror.
|
|
782
|
+
let mailboxTail = Promise.resolve();
|
|
783
|
+
function runMailboxExclusive(fn) {
|
|
784
|
+
const run = mailboxTail.then(fn, fn);
|
|
785
|
+
mailboxTail = run.then(() => undefined, () => undefined);
|
|
786
|
+
return run;
|
|
787
|
+
}
|
|
788
|
+
async function getMailboxStore() {
|
|
789
|
+
if (mailboxStore === null) {
|
|
790
|
+
mailboxStore = await createSubstrateMailboxStore({
|
|
791
|
+
substrate: bindings.repoStore,
|
|
792
|
+
repoId: bindings.workflowRunRepoId,
|
|
793
|
+
principal: mailboxWritePrincipal,
|
|
794
|
+
ref: bindings.workflowRunRef,
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
return mailboxStore;
|
|
798
|
+
}
|
|
799
|
+
function storedEnvelopeFromHeaders(headers, receivedAt) {
|
|
800
|
+
// The Date header is unvalidated external input; fall back to the arrival
|
|
801
|
+
// time when it is absent or unparseable so the store's `toISOString`
|
|
802
|
+
// serialization cannot throw on an Invalid Date.
|
|
803
|
+
const parsed = new Date(headers.date);
|
|
804
|
+
const date = Number.isNaN(parsed.getTime()) ? new Date(receivedAt) : parsed;
|
|
805
|
+
return {
|
|
806
|
+
messageId: headers.messageId,
|
|
807
|
+
from: headers.from,
|
|
808
|
+
to: headers.to,
|
|
809
|
+
subject: headers.subject ?? "",
|
|
810
|
+
date,
|
|
811
|
+
inReplyTo: headers.inReplyTo,
|
|
812
|
+
references: headers.references ?? [],
|
|
813
|
+
interchangeType: headers.interchangeType,
|
|
814
|
+
interchangeCorrelationId: headers.interchangeCorrelationId,
|
|
815
|
+
};
|
|
816
|
+
}
|
|
817
|
+
/**
|
|
818
|
+
* Eager-commit one freshly-arrived inbound message into the deployment's
|
|
819
|
+
* substrate mailbox, then notify the child. Runs on the mail-arrival path,
|
|
820
|
+
* before and independent of FIFO dispatch, so the warm agent's `mail_wait`
|
|
821
|
+
* observes the message mid-turn. Best-effort: the claim-check inbox is the
|
|
822
|
+
* durable delivery contract, so a decode or substrate fault here is logged
|
|
823
|
+
* loudly and never withholds the mail's ack -- the message still reaches the
|
|
824
|
+
* agent as its turn's step input via `trigger.fire`. The `mailbox.notify` is
|
|
825
|
+
* sent only AFTER the append is flushed, so the child reads committed state.
|
|
826
|
+
*/
|
|
827
|
+
async function commitInboundToMailbox(messageId, rawMessage, receivedAt) {
|
|
828
|
+
try {
|
|
829
|
+
await runMailboxExclusive(async () => {
|
|
830
|
+
// The caller gates on a fresh `enqueued` outcome, so a redelivery never
|
|
831
|
+
// reaches here; this guard is belt-and-suspenders against a double
|
|
832
|
+
// append of the same messageId.
|
|
833
|
+
if (mailboxUidByMessageId.has(messageId))
|
|
834
|
+
return;
|
|
835
|
+
let decoded;
|
|
836
|
+
try {
|
|
837
|
+
decoded = decodeMail(rawMessage);
|
|
838
|
+
}
|
|
839
|
+
catch (cause) {
|
|
840
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
841
|
+
logger.error `eager mailbox commit: dropping undecodable inbound mail ${messageId}: ${message}`;
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
844
|
+
const store = await getMailboxStore();
|
|
845
|
+
const uid = store.append(rawMessage, storedEnvelopeFromHeaders(decoded.headers, receivedAt), []);
|
|
846
|
+
mailboxUidByMessageId.set(messageId, uid);
|
|
847
|
+
await store.flush();
|
|
848
|
+
const commit = await bindings.repoStore.resolveRef(mailboxWritePrincipal, bindings.workflowRunRepoId, bindings.workflowRunRef);
|
|
849
|
+
if (commit === null) {
|
|
850
|
+
logger.error `eager mailbox commit: ${bindings.workflowRunRef} did not resolve after flush; skipping mailbox.notify for ${messageId}`;
|
|
851
|
+
return;
|
|
852
|
+
}
|
|
853
|
+
const sender = activeControlSender();
|
|
854
|
+
if (sender === null) {
|
|
855
|
+
logger.info `eager mailbox commit: no active control sender; committed ${messageId} as uid ${String(uid)} without mailbox.notify`;
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
await sender.send({
|
|
859
|
+
type: "mailbox.notify",
|
|
860
|
+
data: {
|
|
861
|
+
runId: deriveWorkflowRunId(bindings.deploymentMailAddress),
|
|
862
|
+
mailbox: MAILBOX_INBOX_DIR,
|
|
863
|
+
uid,
|
|
864
|
+
headers: decoded.headers,
|
|
865
|
+
},
|
|
866
|
+
});
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
catch (cause) {
|
|
870
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
871
|
+
logger.error `eager mailbox commit failed for ${messageId}; mail still delivered via claim-check dispatch: ${message}`;
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
/**
|
|
875
|
+
* Flag a dispatched message's mailbox entry \Seen/$Processed. Fire-and-forget
|
|
876
|
+
* off the dispatch critical path: the flag is a cosmetic IMAP marker, so a
|
|
877
|
+
* missing uid (no eager mailbox entry) or a substrate fault is logged and
|
|
878
|
+
* dropped, never failing the turn.
|
|
879
|
+
*/
|
|
880
|
+
function markMailboxProcessed(messageId) {
|
|
881
|
+
const uid = mailboxUidByMessageId.get(messageId);
|
|
882
|
+
if (uid === undefined)
|
|
883
|
+
return;
|
|
884
|
+
void runMailboxExclusive(async () => {
|
|
885
|
+
try {
|
|
886
|
+
const store = await getMailboxStore();
|
|
887
|
+
if (store.find(uid) === undefined)
|
|
888
|
+
return;
|
|
889
|
+
store.addFlags(uid, [MAILBOX_FLAG_SEEN, MAILBOX_FLAG_PROCESSED]);
|
|
890
|
+
await store.flush();
|
|
891
|
+
}
|
|
892
|
+
finally {
|
|
893
|
+
// The id->uid mapping exists only to flag this message once. After the
|
|
894
|
+
// mark runs (or the message is already gone), the entry is dead weight,
|
|
895
|
+
// so drop it to bound the map over a long-lived conversational mailbox.
|
|
896
|
+
// Redelivery dedup is owned by the durable inbox index, not this map.
|
|
897
|
+
// The delete runs inside the exclusive section so it never interleaves
|
|
898
|
+
// with the arrival path's `has(messageId)` check.
|
|
899
|
+
mailboxUidByMessageId.delete(messageId);
|
|
900
|
+
}
|
|
901
|
+
}).catch((cause) => {
|
|
902
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
903
|
+
logger.warn `mailbox flag mark failed for ${messageId} (uid ${String(uid)}): ${message}`;
|
|
904
|
+
});
|
|
905
|
+
}
|
|
906
|
+
// Resolves once the inbound mail is durably accepted (its inbox write landed
|
|
907
|
+
// or the message was already durably present); rejects when it was not (a
|
|
908
|
+
// phase where the deployment is not accepting mail, a transient enqueue
|
|
909
|
+
// failure, or a stale refusal). The host propagates that settlement to the
|
|
910
|
+
// wire, so resolution is the durable-receipt ACK signal and rejection is the
|
|
911
|
+
// WITHHOLD signal -- a withheld message is redelivered by the hub.
|
|
912
|
+
async function onMailMessage(rawMessage) {
|
|
341
913
|
// Every inbound mail flows through the FIFO inbox claim-check
|
|
342
914
|
// queue, regardless of the supervisor's current phase. The
|
|
343
915
|
// dispatch loop (started by `spawn()` and restarted by the
|
|
@@ -349,16 +921,33 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
349
921
|
// the envelope's `receivedAt` prefix on the inbox filename.
|
|
350
922
|
if (state.phase === "idle" ||
|
|
351
923
|
state.phase === "stopping" ||
|
|
352
|
-
state.phase === "stopped"
|
|
353
|
-
|
|
354
|
-
//
|
|
355
|
-
//
|
|
356
|
-
|
|
924
|
+
state.phase === "stopped" ||
|
|
925
|
+
state.phase === "crash-looping") {
|
|
926
|
+
// The host's higher-level lifecycle is already tearing the deployment
|
|
927
|
+
// down; nothing is enqueued. Reject rather than silently drop so the
|
|
928
|
+
// ack is WITHHELD and the hub redelivers -- a later generation (or a
|
|
929
|
+
// recycle-installed dispatch loop) may accept it, and a permanently
|
|
930
|
+
// torn-down address exhausts the hub's bounded retry budget instead of
|
|
931
|
+
// losing a message a transiently-idle deployment would have taken.
|
|
932
|
+
throw new Error(`inbound mail not accepted: supervisor phase is "${state.phase}"`);
|
|
933
|
+
}
|
|
934
|
+
try {
|
|
935
|
+
await enqueueInboundMail(rawMessage);
|
|
936
|
+
}
|
|
937
|
+
catch (cause) {
|
|
938
|
+
// Both branches WITHHOLD (rethrow); the split only sets log severity so
|
|
939
|
+
// a stale refusal surfaces as its own loud signal rather than blending
|
|
940
|
+
// into ordinary enqueue-failure noise. The ack/withhold decision is the
|
|
941
|
+
// rethrow itself, never this classification.
|
|
942
|
+
if (cause instanceof StaleInboxEnqueueError) {
|
|
943
|
+
logger.error `inbound mail refused as stale, withholding ack (hub will redeliver): ${cause.message}`;
|
|
944
|
+
}
|
|
945
|
+
else {
|
|
946
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
947
|
+
logger.error `enqueueInbox failed, withholding ack (hub will redeliver): ${message}`;
|
|
948
|
+
}
|
|
949
|
+
throw cause;
|
|
357
950
|
}
|
|
358
|
-
void enqueueInboundMail(rawMessage).catch((cause) => {
|
|
359
|
-
const message = cause instanceof Error ? cause.message : String(cause);
|
|
360
|
-
logger.error `enqueueInbox failed: ${message}`;
|
|
361
|
-
});
|
|
362
951
|
}
|
|
363
952
|
async function enqueueInboundMail(rawMessage) {
|
|
364
953
|
const messageId = await deriveMessageId(rawMessage);
|
|
@@ -373,11 +962,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
373
962
|
const rawMessageBase64 = base64Encode(rawMessage);
|
|
374
963
|
// D2 leg: `enqueueInbox` runs in `onMailMessage` BEFORE dispatch, so
|
|
375
964
|
// it is paid OUTSIDE the dispatch-start..reply-produced window -- its
|
|
376
|
-
// growth is invisible to the 4.7 bracket.
|
|
377
|
-
//
|
|
378
|
-
//
|
|
965
|
+
// growth is invisible to the 4.7 bracket. This leg mark is keyed by the
|
|
966
|
+
// messageId, the same per-message key every in-window leg uses, so the
|
|
967
|
+
// D2 per-message OLS fit groups the enqueue leg with the rest.
|
|
379
968
|
legMarkStart(messageId, "enqueue");
|
|
380
|
-
await inboxPrimitives.enqueueInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
|
|
969
|
+
const outcome = await inboxPrimitives.enqueueInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
|
|
381
970
|
address: bindings.deploymentMailAddress,
|
|
382
971
|
messageId,
|
|
383
972
|
receivedAt,
|
|
@@ -385,7 +974,25 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
385
974
|
rawMessage: rawMessageBase64,
|
|
386
975
|
});
|
|
387
976
|
legMarkEnd(messageId, "enqueue");
|
|
388
|
-
|
|
977
|
+
// Only a fresh enqueue added a new inbox entry; wake the dispatch loop for
|
|
978
|
+
// it alone. An `already-present` outcome landed nothing new -- returning
|
|
979
|
+
// (which acks) without waking is correct, since the earlier delivery of
|
|
980
|
+
// the same messageId already drives dispatch. This resolves for both
|
|
981
|
+
// outcomes: both mean the bytes are durably accounted for, so both ack.
|
|
982
|
+
if (outcome.outcome === "enqueued") {
|
|
983
|
+
// Eager-commit the fresh message into the per-run mailbox and notify the
|
|
984
|
+
// child BEFORE waking dispatch, so the warm agent's mail_wait can observe
|
|
985
|
+
// it committed. Non-fatal by contract: the enqueue above already secured
|
|
986
|
+
// the durable delivery, so this never withholds the ack.
|
|
987
|
+
await commitInboundToMailbox(messageId, rawMessage, receivedAt);
|
|
988
|
+
wakeDispatch();
|
|
989
|
+
}
|
|
990
|
+
else {
|
|
991
|
+
// A redelivery of a message already durably present: the ack still
|
|
992
|
+
// fires (it is on disk), but no new run is dispatched. Surface it so an
|
|
993
|
+
// at-least-once redelivery being made effectively-once is observable.
|
|
994
|
+
logger.info `inbound mail ${messageId} already durably present (${outcome.reason}); acknowledging without re-dispatch`;
|
|
995
|
+
}
|
|
389
996
|
}
|
|
390
997
|
/**
|
|
391
998
|
* Pump child-initiated upstream control frames after `ready` has
|
|
@@ -448,6 +1055,18 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
448
1055
|
});
|
|
449
1056
|
continue;
|
|
450
1057
|
}
|
|
1058
|
+
if (payload.type === "mailbox.mutate.request") {
|
|
1059
|
+
// INBOUND half of mailbox ownership (§3b). The child asked the
|
|
1060
|
+
// supervisor -- the sole mailbox writer -- to apply a flag write or
|
|
1061
|
+
// expunge. Run it off the iterator's loop so the iterator keeps
|
|
1062
|
+
// draining while the store flushes; the handler owns the
|
|
1063
|
+
// `mailbox.mutate.response` reply that resolves the child's awaiter.
|
|
1064
|
+
void handleMailboxMutation(payload.data).catch((cause) => {
|
|
1065
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1066
|
+
logger.error `mailbox.mutate.request handler crashed: ${message}`;
|
|
1067
|
+
});
|
|
1068
|
+
continue;
|
|
1069
|
+
}
|
|
451
1070
|
if (payload.type === "terminal.event") {
|
|
452
1071
|
// The workflow-process child mirrors every terminal-run commit
|
|
453
1072
|
// over the control IPC. Fan it out to the COHORT'S broadcaster
|
|
@@ -456,16 +1075,98 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
456
1075
|
// to-one with its cohort's `controlIncoming` iterator: a
|
|
457
1076
|
// buffered `terminal.event` the OLD child emitted before kill
|
|
458
1077
|
// landed must NEVER route to the NEW cohort's broadcaster.
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
// (the
|
|
462
|
-
//
|
|
1078
|
+
// Every run shares the stable runId (the deployment mail
|
|
1079
|
+
// address), so an OLD-cohort frame and a NEW-cohort run collide
|
|
1080
|
+
// on that id by construction (the recycle/replay case); without
|
|
1081
|
+
// this binding the stale frame would falsely settle the NEW
|
|
1082
|
+
// cohort's `waitForRunTerminalOrPark` and commit `markConsumed`
|
|
463
1083
|
// on a run still in flight. The broadcaster's own `dispose()`
|
|
464
1084
|
// on cohort teardown turns post-dispose notify into a no-op,
|
|
465
1085
|
// so a stale frame dequeued after the cohort was torn down
|
|
466
1086
|
// drops cleanly without leaking into any successor cohort.
|
|
467
1087
|
const event = terminalEventFromPayload(payload.data);
|
|
468
1088
|
cohortBroadcaster.notify(payload.data.runId, event);
|
|
1089
|
+
terminalRunIds.add(payload.data.runId);
|
|
1090
|
+
// Clean up cohort tracking for the terminated run. Self-discovered
|
|
1091
|
+
// runs have no dispatch-loop entry, so their cleanup happens here.
|
|
1092
|
+
cohortRunIds.delete(payload.data.runId);
|
|
1093
|
+
runInputChannels.delete(payload.data.runId);
|
|
1094
|
+
continue;
|
|
1095
|
+
}
|
|
1096
|
+
if (payload.type === "park.notify") {
|
|
1097
|
+
// The workflow-process child reported a control-plane suspension: an
|
|
1098
|
+
// agent step parked on a reserved `signalName(correlationId)` channel.
|
|
1099
|
+
if (payload.data.parkKind === "input") {
|
|
1100
|
+
// Input parks are owned by the supervisor, not the hub. Register
|
|
1101
|
+
// cohort membership BEFORE caching the correlationId so a run is
|
|
1102
|
+
// never a channel-without-cohort entry (the routing-hygiene
|
|
1103
|
+
// invariant); a live run that parked is always in cohortRunIds
|
|
1104
|
+
// already, so this is idempotent belt-and-suspenders. Cache the
|
|
1105
|
+
// correlationId so the dispatch loop can fire signal.deliver on
|
|
1106
|
+
// subsequent mail without a substrate round-trip.
|
|
1107
|
+
cohortRunIds.add(payload.data.runId);
|
|
1108
|
+
runInputChannels.set(payload.data.runId, {
|
|
1109
|
+
correlationId: payload.data.correlationId,
|
|
1110
|
+
parkKind: "input",
|
|
1111
|
+
});
|
|
1112
|
+
// Stop any drain accumulator for a run that has parked. Input parks
|
|
1113
|
+
// own this because a drain arms no accumulator for a run that already
|
|
1114
|
+
// holds an input channel; an approval park never carries one, so its
|
|
1115
|
+
// drain interaction is a separate concern and stays out of this arm.
|
|
1116
|
+
const accumulator = drainAccumulators.get(payload.data.runId);
|
|
1117
|
+
if (accumulator !== undefined) {
|
|
1118
|
+
accumulator.stop();
|
|
1119
|
+
drainAccumulators.delete(payload.data.runId);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
else if (payload.data.parkKind === "approval") {
|
|
1123
|
+
// Approval parks are hub-registered through the shared
|
|
1124
|
+
// `registerSuspension` transform.
|
|
1125
|
+
registerSuspension({
|
|
1126
|
+
runId: payload.data.runId,
|
|
1127
|
+
correlationId: payload.data.correlationId,
|
|
1128
|
+
parkKind: "approval",
|
|
1129
|
+
...(payload.data.snapshot !== undefined
|
|
1130
|
+
? { snapshot: payload.data.snapshot }
|
|
1131
|
+
: {}),
|
|
1132
|
+
});
|
|
1133
|
+
}
|
|
1134
|
+
else {
|
|
1135
|
+
// A `signal-relay` park is relayed down into the body child by the
|
|
1136
|
+
// section runtime and is never hub-registered, so it does not ride
|
|
1137
|
+
// `park.notify`. One arriving here is a protocol violation; log and
|
|
1138
|
+
// drop rather than mis-registering it as an approval.
|
|
1139
|
+
logger.error `park.notify for run ${payload.data.runId} carried parkKind=${payload.data.parkKind}, which is not a hub-registered kind; dropping`;
|
|
1140
|
+
}
|
|
1141
|
+
// A park of ANY kind suspends the run, so a dispatch loop waiting on
|
|
1142
|
+
// `waitForRunTerminalOrPark` after firing the trigger (or delivering
|
|
1143
|
+
// the last signal) must be released here regardless of park kind. An
|
|
1144
|
+
// approval park that only registered its suspension would leave that
|
|
1145
|
+
// loop hanging to the terminal-or-park backstop. Bump the park
|
|
1146
|
+
// generation BEFORE resolving the waiter: a dispatch loop that captured
|
|
1147
|
+
// `sinceGen` for this run must see the newer generation both when its
|
|
1148
|
+
// armed waiter fires here and when it re-reads the generation after
|
|
1149
|
+
// arming (the check-after-register).
|
|
1150
|
+
parkGenerations.set(payload.data.runId, (parkGenerations.get(payload.data.runId) ?? 0) + 1);
|
|
1151
|
+
// Wake any dispatch loop waiting for this run to park.
|
|
1152
|
+
resolveParkNotifyWaiter(payload.data.runId);
|
|
1153
|
+
continue;
|
|
1154
|
+
}
|
|
1155
|
+
if (payload.type === "parked-correlations.response") {
|
|
1156
|
+
// The child answered a `reEmitParkedCorrelations` query. Resolve the
|
|
1157
|
+
// awaiting driver; a response with no pending entry (the query already
|
|
1158
|
+
// timed out and dropped it) is logged and dropped, never thrown, so it
|
|
1159
|
+
// cannot tear the pump down.
|
|
1160
|
+
resolveParkedResponse(payload.data);
|
|
1161
|
+
continue;
|
|
1162
|
+
}
|
|
1163
|
+
if (payload.type === "resumed.runs") {
|
|
1164
|
+
// The child self-discovered runs from the substrate after reconnect
|
|
1165
|
+
// or recycle. Seed cohort tracking so drain accumulators and dispatch
|
|
1166
|
+
// routing account for runs the supervisor did not personally fire.
|
|
1167
|
+
for (const runId of payload.data.runIds) {
|
|
1168
|
+
cohortRunIds.add(runId);
|
|
1169
|
+
}
|
|
469
1170
|
continue;
|
|
470
1171
|
}
|
|
471
1172
|
logger.warn `workflow-process upstream control payload ignored: type=${payload.type}`;
|
|
@@ -473,22 +1174,26 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
473
1174
|
}
|
|
474
1175
|
const pendingMerges = new Map();
|
|
475
1176
|
/**
|
|
476
|
-
* Reject every pending merge round-trip and every
|
|
477
|
-
*
|
|
478
|
-
*
|
|
479
|
-
*
|
|
480
|
-
*
|
|
481
|
-
*
|
|
482
|
-
*
|
|
1177
|
+
* Reject every pending merge round-trip and every park-notify
|
|
1178
|
+
* waiter. Invoked on cohort transitions (shutdown, recycle's
|
|
1179
|
+
* `installNewChild`) so closures awaiting these promises do not
|
|
1180
|
+
* outlive the cohort that armed them. Without this, a
|
|
1181
|
+
* `handleSubstrateWriteRequest` mid-merge or a dispatch loop
|
|
1182
|
+
* waiting for park would sit on a resolver that the dying control
|
|
1183
|
+
* channel will never invoke.
|
|
483
1184
|
*/
|
|
484
1185
|
function rejectCohortAwaiters(reason) {
|
|
485
1186
|
for (const [requestId, entry] of pendingMerges) {
|
|
486
1187
|
pendingMerges.delete(requestId);
|
|
487
1188
|
entry.resolve({ ok: false, reason: `cohort aborted: ${reason}` });
|
|
488
1189
|
}
|
|
489
|
-
for (const [
|
|
490
|
-
|
|
491
|
-
|
|
1190
|
+
for (const [requestId, entry] of pendingParkedQueries) {
|
|
1191
|
+
pendingParkedQueries.delete(requestId);
|
|
1192
|
+
entry.settle(null);
|
|
1193
|
+
}
|
|
1194
|
+
for (const [runId, resolve] of parkNotifyWaiters.entries()) {
|
|
1195
|
+
parkNotifyWaiters.delete(runId);
|
|
1196
|
+
resolve();
|
|
492
1197
|
}
|
|
493
1198
|
}
|
|
494
1199
|
function resolveMergeResponse(data) {
|
|
@@ -525,6 +1230,125 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
525
1230
|
}
|
|
526
1231
|
entry.resolve({ ok: false, reason: data.result.reason });
|
|
527
1232
|
}
|
|
1233
|
+
// Stamp the deployment identity the supervisor owns onto a child-supplied
|
|
1234
|
+
// park and hand it to the host's suspension-register sink. Shared by the
|
|
1235
|
+
// `park.notify` arm (the happy-path emit) and `reEmitParkedCorrelations`
|
|
1236
|
+
// (the re-establishment re-emit). Best-effort: a throwing sink is logged,
|
|
1237
|
+
// not rethrown, so it cannot tear the upstream pump down or abort a re-emit
|
|
1238
|
+
// partway through the parked set. The sink (production: the sidecar) turns
|
|
1239
|
+
// the stamped registration into a `signal.correlation.register` frame the
|
|
1240
|
+
// hub co-writes the run's routing + approval rows from.
|
|
1241
|
+
function registerSuspension(park) {
|
|
1242
|
+
if (bindings.onSuspensionRegister === undefined) {
|
|
1243
|
+
logger.warn `suspension register for runId=${park.runId} but no onSuspensionRegister sink is wired; correlation ${park.correlationId} not registered`;
|
|
1244
|
+
return;
|
|
1245
|
+
}
|
|
1246
|
+
try {
|
|
1247
|
+
bindings.onSuspensionRegister({
|
|
1248
|
+
runId: park.runId,
|
|
1249
|
+
correlationId: park.correlationId,
|
|
1250
|
+
kind: park.parkKind,
|
|
1251
|
+
anchorRunId: bindings.anchorRunId,
|
|
1252
|
+
agentAddress: bindings.deploymentMailAddress,
|
|
1253
|
+
...(park.snapshot !== undefined
|
|
1254
|
+
? { approvalSnapshot: park.snapshot }
|
|
1255
|
+
: {}),
|
|
1256
|
+
});
|
|
1257
|
+
}
|
|
1258
|
+
catch (cause) {
|
|
1259
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1260
|
+
logger.error `onSuspensionRegister sink threw for runId=${park.runId} correlationId=${park.correlationId}: ${message}`;
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
const pendingParkedQueries = new Map();
|
|
1264
|
+
let parkedQuerySeq = 0;
|
|
1265
|
+
function resolveParkedResponse(data) {
|
|
1266
|
+
const entry = pendingParkedQueries.get(data.requestId);
|
|
1267
|
+
if (entry === undefined) {
|
|
1268
|
+
logger.warn `parked-correlations.response landed with no pending entry; requestId=${data.requestId} dropped`;
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
pendingParkedQueries.delete(data.requestId);
|
|
1272
|
+
entry.settle(data.parked);
|
|
1273
|
+
}
|
|
1274
|
+
async function reEmitParkedCorrelations() {
|
|
1275
|
+
// No-op (NOT throw) when the child is not addressable. `deliverSignal`
|
|
1276
|
+
// throws on non-running/starting (including recycling) because it points a
|
|
1277
|
+
// write at the dying child's closing pipe and wants the caller to retry;
|
|
1278
|
+
// this driver's contract is the opposite -- the next re-establishment
|
|
1279
|
+
// re-drives it -- so skipping recycling and letting the next spawn's
|
|
1280
|
+
// re-emit cover it is correct here, not a missed guard.
|
|
1281
|
+
if (state.phase !== "running" && state.phase !== "starting") {
|
|
1282
|
+
logger.info `reEmitParkedCorrelations: child not addressable (phase=${state.phase}); skipping`;
|
|
1283
|
+
return;
|
|
1284
|
+
}
|
|
1285
|
+
const controlSender = state.controlSender;
|
|
1286
|
+
const requestId = `pc-${String((parkedQuerySeq += 1))}`;
|
|
1287
|
+
const responded = new Promise((resolve) => {
|
|
1288
|
+
pendingParkedQueries.set(requestId, { settle: resolve });
|
|
1289
|
+
});
|
|
1290
|
+
// Watchdog: a wedged-but-alive child never tears its cohort down, so the
|
|
1291
|
+
// cohort-abort settle would never fire and this await would hang the
|
|
1292
|
+
// re-establishment caller. On expiry, drop the pending entry and return;
|
|
1293
|
+
// the next re-establishment re-drives (the hub co-write is idempotent).
|
|
1294
|
+
let timeoutHandle = null;
|
|
1295
|
+
const watchdog = new Promise((resolve) => {
|
|
1296
|
+
timeoutHandle = setTimeout(() => {
|
|
1297
|
+
timeoutHandle = null;
|
|
1298
|
+
if (pendingParkedQueries.delete(requestId)) {
|
|
1299
|
+
logger.warn `reEmitParkedCorrelations: requestId=${requestId} did not respond within ${String(parkedQueryWatchdogMs)}ms; re-registration retries on the next re-establishment`;
|
|
1300
|
+
}
|
|
1301
|
+
resolve("timeout");
|
|
1302
|
+
}, parkedQueryWatchdogMs);
|
|
1303
|
+
});
|
|
1304
|
+
let outcome;
|
|
1305
|
+
try {
|
|
1306
|
+
await controlSender.send({
|
|
1307
|
+
type: "parked-correlations.request",
|
|
1308
|
+
data: { requestId },
|
|
1309
|
+
});
|
|
1310
|
+
outcome = await Promise.race([responded, watchdog]);
|
|
1311
|
+
}
|
|
1312
|
+
catch (cause) {
|
|
1313
|
+
// The downstream send failed (a closing pipe). Best-effort: drop the
|
|
1314
|
+
// pending entry, log, and return; the next re-establishment re-drives.
|
|
1315
|
+
pendingParkedQueries.delete(requestId);
|
|
1316
|
+
if (timeoutHandle !== null)
|
|
1317
|
+
clearTimeout(timeoutHandle);
|
|
1318
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1319
|
+
logger.warn `reEmitParkedCorrelations query failed: ${message}; re-registration retries on the next re-establishment`;
|
|
1320
|
+
return;
|
|
1321
|
+
}
|
|
1322
|
+
if (timeoutHandle !== null)
|
|
1323
|
+
clearTimeout(timeoutHandle);
|
|
1324
|
+
// `timeout` (watchdog fired) or `null` (cohort torn down before the child
|
|
1325
|
+
// answered): nothing to re-emit; the next re-establishment re-drives.
|
|
1326
|
+
if (outcome === "timeout" || outcome === null)
|
|
1327
|
+
return;
|
|
1328
|
+
for (const parked of outcome) {
|
|
1329
|
+
if (parked.parkKind === "input") {
|
|
1330
|
+
// Register cohort membership BEFORE the input channel so the run is
|
|
1331
|
+
// never a channel-without-cohort entry: the dispatch loop's routing
|
|
1332
|
+
// hygiene drops exactly such entries, and a live resumed run must not
|
|
1333
|
+
// be mistaken for a dead one and have its channel deleted.
|
|
1334
|
+
cohortRunIds.add(parked.runId);
|
|
1335
|
+
runInputChannels.set(parked.runId, {
|
|
1336
|
+
correlationId: parked.correlationId,
|
|
1337
|
+
parkKind: "input",
|
|
1338
|
+
});
|
|
1339
|
+
}
|
|
1340
|
+
else {
|
|
1341
|
+
registerSuspension({ ...parked, parkKind: "approval" });
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
function resolveParkNotifyWaiter(runId) {
|
|
1346
|
+
const resolve = parkNotifyWaiters.get(runId);
|
|
1347
|
+
if (resolve === undefined)
|
|
1348
|
+
return;
|
|
1349
|
+
parkNotifyWaiters.delete(runId);
|
|
1350
|
+
resolve();
|
|
1351
|
+
}
|
|
528
1352
|
/**
|
|
529
1353
|
* OUTBOUND half of mailbox ownership (§3a). The workflow-process child
|
|
530
1354
|
* never holds the agent's signing key; it forwards the structured
|
|
@@ -577,6 +1401,111 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
577
1401
|
});
|
|
578
1402
|
}
|
|
579
1403
|
}
|
|
1404
|
+
/**
|
|
1405
|
+
* Apply a child-requested mailbox mutation to the owned store (INBOUND
|
|
1406
|
+
* half of mailbox ownership, §3b). The supervisor is the sole writer to
|
|
1407
|
+
* the workflow-run mailbox; the child never flushes it. A flag write
|
|
1408
|
+
* (`addFlags` / `removeFlags`) targets one uid; an `expunge` sweeps every
|
|
1409
|
+
* `\Deleted` message out of the live INBOX. The mutation is applied under
|
|
1410
|
+
* `runMailboxExclusive` and flushed before the reply, so the child's next
|
|
1411
|
+
* committed read observes it -- the flush-before-signal ordering
|
|
1412
|
+
* `commitInboundToMailbox` uses. A failure (unknown uid, wrong mailbox,
|
|
1413
|
+
* substrate fault) surfaces back as a structured `{ ok: false, reason }`
|
|
1414
|
+
* so the agent's mail-tool call fails loudly rather than dropping the
|
|
1415
|
+
* mutation silently.
|
|
1416
|
+
*/
|
|
1417
|
+
async function handleMailboxMutation(data) {
|
|
1418
|
+
// Capture the sender once. Re-fetching after the flush could return a
|
|
1419
|
+
// successor cohort's sender and misroute the reply to the wrong child
|
|
1420
|
+
// (see the substrate-write handler's note). A null sender means the
|
|
1421
|
+
// supervisor is mid-recycle or tearing down: there is nothing to reply
|
|
1422
|
+
// on, so drop and warn -- the child's read end is closing alongside, so
|
|
1423
|
+
// its pending awaiter is rejected by the control loop's `cancelAll`.
|
|
1424
|
+
const controlSender = activeControlSender();
|
|
1425
|
+
if (controlSender === null) {
|
|
1426
|
+
logger.warn `mailbox.mutate.request received outside running phase; requestId=${data.requestId} dropped (child awaiter will fail on pipe close)`;
|
|
1427
|
+
return;
|
|
1428
|
+
}
|
|
1429
|
+
// The supervisor owns exactly one mailbox, the substrate INBOX. Reject a
|
|
1430
|
+
// request for any other name rather than silently mutate INBOX under it,
|
|
1431
|
+
// which would be a wrong-target durable write reported as success. The
|
|
1432
|
+
// frame carries an unconstrained mailbox string, so this is validated
|
|
1433
|
+
// here at the owning layer, not trusted from the child transport.
|
|
1434
|
+
if (data.mailbox !== MAILBOX_INBOX_DIR) {
|
|
1435
|
+
await controlSender.send({
|
|
1436
|
+
type: "mailbox.mutate.response",
|
|
1437
|
+
data: {
|
|
1438
|
+
requestId: data.requestId,
|
|
1439
|
+
result: {
|
|
1440
|
+
ok: false,
|
|
1441
|
+
reason: `unknown mailbox "${data.mailbox}"; only ${MAILBOX_INBOX_DIR} is writable`,
|
|
1442
|
+
},
|
|
1443
|
+
},
|
|
1444
|
+
});
|
|
1445
|
+
return;
|
|
1446
|
+
}
|
|
1447
|
+
try {
|
|
1448
|
+
const expungedUids = await runMailboxExclusive(async () => {
|
|
1449
|
+
const store = await getMailboxStore();
|
|
1450
|
+
if (data.op === "expunge") {
|
|
1451
|
+
// Snapshot the \Deleted uids before removing: `store.messages` is
|
|
1452
|
+
// the live array, so `.filter().map()` materializes the targets
|
|
1453
|
+
// before any `remove` splices it. The whole sweep runs
|
|
1454
|
+
// synchronously under the lock, so no snapshotted uid can vanish
|
|
1455
|
+
// before its `remove`.
|
|
1456
|
+
const uids = store.messages
|
|
1457
|
+
.filter((m) => m.flags.has(MAILBOX_FLAG_DELETED))
|
|
1458
|
+
.map((m) => m.uid);
|
|
1459
|
+
for (const uid of uids) {
|
|
1460
|
+
store.remove(uid);
|
|
1461
|
+
// Bound the id->uid map: drop any entry now pointing at a removed
|
|
1462
|
+
// uid. Not load-bearing -- `markMailboxProcessed` guards with
|
|
1463
|
+
// `find` -- but keeps the map from retaining dead uids.
|
|
1464
|
+
for (const [messageId, mappedUid] of mailboxUidByMessageId) {
|
|
1465
|
+
if (mappedUid === uid)
|
|
1466
|
+
mailboxUidByMessageId.delete(messageId);
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
await store.flush();
|
|
1470
|
+
return uids;
|
|
1471
|
+
}
|
|
1472
|
+
if (data.op === "addFlags") {
|
|
1473
|
+
store.addFlags(data.uid, data.flags);
|
|
1474
|
+
}
|
|
1475
|
+
else {
|
|
1476
|
+
store.removeFlags(data.uid, data.flags);
|
|
1477
|
+
}
|
|
1478
|
+
await store.flush();
|
|
1479
|
+
return undefined;
|
|
1480
|
+
});
|
|
1481
|
+
await controlSender.send({
|
|
1482
|
+
type: "mailbox.mutate.response",
|
|
1483
|
+
data: {
|
|
1484
|
+
requestId: data.requestId,
|
|
1485
|
+
result: expungedUids === undefined
|
|
1486
|
+
? { ok: true }
|
|
1487
|
+
: { ok: true, expungedUids },
|
|
1488
|
+
},
|
|
1489
|
+
});
|
|
1490
|
+
}
|
|
1491
|
+
catch (cause) {
|
|
1492
|
+
// Reply on the same captured sender. If this send itself throws (a
|
|
1493
|
+
// broken pipe), it propagates to the pump's `.catch`, and the child's
|
|
1494
|
+
// awaiter is rejected by the control loop's `cancelAll` -- the backstop
|
|
1495
|
+
// `handleOutboundMessage` also relies on. Accepted window: a mutation
|
|
1496
|
+
// can flush durably while its reply is undeliverable, so the agent tool
|
|
1497
|
+
// errors on a mutation that landed. This is inherent to apply-then-reply
|
|
1498
|
+
// across a teardown boundary and identical to `handleOutboundMessage`.
|
|
1499
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
1500
|
+
await controlSender.send({
|
|
1501
|
+
type: "mailbox.mutate.response",
|
|
1502
|
+
data: {
|
|
1503
|
+
requestId: data.requestId,
|
|
1504
|
+
result: { ok: false, reason },
|
|
1505
|
+
},
|
|
1506
|
+
});
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
580
1509
|
async function handleSubstrateWriteRequest(data) {
|
|
581
1510
|
const controlSender = activeControlSender();
|
|
582
1511
|
if (controlSender === null) {
|
|
@@ -628,7 +1557,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
628
1557
|
// is which process owns the substrate write contract.
|
|
629
1558
|
const writePrincipal = {
|
|
630
1559
|
kind: "workflow-process",
|
|
631
|
-
|
|
1560
|
+
anchorRunId: bindings.anchorRunId,
|
|
632
1561
|
};
|
|
633
1562
|
// The commit's terminal detection comes from the kind handler's
|
|
634
1563
|
// typed `newlyTerminalRuns` signal (returned below), not a sniff of
|
|
@@ -656,7 +1585,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
656
1585
|
// benchmark's per-message OLS fit groups on.
|
|
657
1586
|
const legClassification = classifyProxiedWriteLeg(data.preservePrefix);
|
|
658
1587
|
if (legClassification !== null) {
|
|
659
|
-
legMarkStart(legClassification.
|
|
1588
|
+
legMarkStart(legClassification.messageId, legClassification.leg);
|
|
660
1589
|
}
|
|
661
1590
|
try {
|
|
662
1591
|
const { commitSha, newlyTerminalRuns } = await bindings.repoStore.writeTreePreservingPrefix(writePrincipal, validatedRepoId, data.ref, {
|
|
@@ -701,22 +1630,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
701
1630
|
});
|
|
702
1631
|
// D2 leg end: the substrate commit (hash objects, write tree,
|
|
703
1632
|
// advance ref under the per-repo lock) just resolved. Stamped here,
|
|
704
|
-
// before the
|
|
705
|
-
//
|
|
706
|
-
//
|
|
1633
|
+
// before the response, so the run-event/wal leg measures only its own
|
|
1634
|
+
// commit and not the dispatch loop's markConsumed (which the
|
|
1635
|
+
// `markconsumed` leg owns).
|
|
707
1636
|
if (legClassification !== null) {
|
|
708
|
-
legMarkEnd(legClassification.
|
|
709
|
-
}
|
|
710
|
-
const watchdog = await synchronouslyDispatchTerminalWrite(newlyTerminalRuns);
|
|
711
|
-
if (!watchdog.ok) {
|
|
712
|
-
await controlSender.send({
|
|
713
|
-
type: "substrate.write.response",
|
|
714
|
-
data: {
|
|
715
|
-
requestId: data.requestId,
|
|
716
|
-
result: { ok: false, reason: watchdog.reason },
|
|
717
|
-
},
|
|
718
|
-
});
|
|
719
|
-
return;
|
|
1637
|
+
legMarkEnd(legClassification.messageId, legClassification.leg);
|
|
720
1638
|
}
|
|
721
1639
|
await controlSender.send({
|
|
722
1640
|
type: "substrate.write.response",
|
|
@@ -740,7 +1658,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
740
1658
|
substrate: bindings.repoStore,
|
|
741
1659
|
repoId: validatedRepoId,
|
|
742
1660
|
ref: data.ref,
|
|
743
|
-
|
|
1661
|
+
anchorRunId: bindings.anchorRunId,
|
|
744
1662
|
runId,
|
|
745
1663
|
}).catch((cause) => {
|
|
746
1664
|
logger.warn `compaction of run ${runId} failed: ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
@@ -765,129 +1683,6 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
765
1683
|
});
|
|
766
1684
|
}
|
|
767
1685
|
}
|
|
768
|
-
// Per-runId synchronization between the substrate-write handler
|
|
769
|
-
// and the dispatch loop's `markConsumed`. The handler arms a
|
|
770
|
-
// waiter when it commits a terminal-event blob and waits for the
|
|
771
|
-
// dispatch loop to fire `resolveMarkConsumedWaiter(runId)` before
|
|
772
|
-
// sending the substrate.write.response back to the child.
|
|
773
|
-
const markConsumedCompletionWaiters = new Map();
|
|
774
|
-
function resolveMarkConsumedWaiter(runId) {
|
|
775
|
-
const waiter = markConsumedCompletionWaiters.get(runId);
|
|
776
|
-
if (waiter === undefined)
|
|
777
|
-
return;
|
|
778
|
-
markConsumedCompletionWaiters.delete(runId);
|
|
779
|
-
waiter.resolve();
|
|
780
|
-
}
|
|
781
|
-
/**
|
|
782
|
-
* Hold the substrate.write.response until the dispatch loop's
|
|
783
|
-
* markConsumed settles for each run the kind handler reports as newly
|
|
784
|
-
* terminal in this commit. Terminal-ness comes from the handler's typed
|
|
785
|
-
* `newlyTerminalRuns` signal -- determined authoritatively during
|
|
786
|
-
* validation -- not re-derived from the committed path shape, so it
|
|
787
|
-
* survives the run-event layout changing (e.g. compaction folding a
|
|
788
|
-
* run's per-event files into one combined file). The wait is per-runId
|
|
789
|
-
* so multiple runs can proceed concurrently if a future dispatch loop
|
|
790
|
-
* ever processes more than one mail in parallel.
|
|
791
|
-
*
|
|
792
|
-
* A watchdog timeout (`terminalWriteWatchdogMs`) caps each wait so a
|
|
793
|
-
* never-arming markConsumed (a bug in the dispatch loop, a torn-down
|
|
794
|
-
* cohort, a stalled inbox primitive) does not deadlock the child's
|
|
795
|
-
* write -- and therefore the runtime body, and therefore the dispatch
|
|
796
|
-
* loop. On expiry the waiter is force-released and a structured failure
|
|
797
|
-
* propagates back to the child as
|
|
798
|
-
* `{ ok: false, reason: "terminal-write watchdog timeout: ..." }`.
|
|
799
|
-
*/
|
|
800
|
-
async function synchronouslyDispatchTerminalWrite(newlyTerminalRuns) {
|
|
801
|
-
const holds = [];
|
|
802
|
-
for (const { runId, terminalEventJson } of newlyTerminalRuns) {
|
|
803
|
-
if (!inFlightRuns.has(runId))
|
|
804
|
-
continue;
|
|
805
|
-
holds.push(holdResponseForMarkConsumed(runId, terminalEventJson));
|
|
806
|
-
}
|
|
807
|
-
if (holds.length === 0)
|
|
808
|
-
return { ok: true };
|
|
809
|
-
const results = await Promise.all(holds);
|
|
810
|
-
return results.find((r) => !r.ok) ?? { ok: true };
|
|
811
|
-
}
|
|
812
|
-
async function holdResponseForMarkConsumed(runId, terminalEventJson) {
|
|
813
|
-
const completed = new Promise((resolve, reject) => {
|
|
814
|
-
markConsumedCompletionWaiters.set(runId, { resolve, reject });
|
|
815
|
-
});
|
|
816
|
-
const broadcaster = activeTerminalBroadcaster();
|
|
817
|
-
if (broadcaster !== null) {
|
|
818
|
-
const synthetic = synthesizeTerminalEvent(terminalEventJson);
|
|
819
|
-
if (synthetic !== null) {
|
|
820
|
-
broadcaster.notify(runId, synthetic);
|
|
821
|
-
}
|
|
822
|
-
}
|
|
823
|
-
let timeoutHandle = null;
|
|
824
|
-
const watchdog = new Promise((resolve) => {
|
|
825
|
-
timeoutHandle = setTimeout(() => {
|
|
826
|
-
timeoutHandle = null;
|
|
827
|
-
// Force-release the waiter so the dispatch loop's eventual
|
|
828
|
-
// resolve does not strand a dangling map entry, then surface
|
|
829
|
-
// the structured failure to the caller. The reason text is
|
|
830
|
-
// logged through the package logger so the watchdog is not
|
|
831
|
-
// silent on the host side.
|
|
832
|
-
const stillPending = markConsumedCompletionWaiters.get(runId) !== undefined;
|
|
833
|
-
if (stillPending) {
|
|
834
|
-
markConsumedCompletionWaiters.delete(runId);
|
|
835
|
-
}
|
|
836
|
-
const reason = `terminal-write watchdog timeout: markConsumed for runId=${runId} did not settle within ${String(terminalWriteWatchdogMs)}ms`;
|
|
837
|
-
logger.error `${reason}`;
|
|
838
|
-
resolve({ ok: false, reason });
|
|
839
|
-
}, terminalWriteWatchdogMs);
|
|
840
|
-
});
|
|
841
|
-
const result = await Promise.race([
|
|
842
|
-
completed.then(() => ({ ok: true })),
|
|
843
|
-
watchdog,
|
|
844
|
-
]);
|
|
845
|
-
if (timeoutHandle !== null) {
|
|
846
|
-
clearTimeout(timeoutHandle);
|
|
847
|
-
}
|
|
848
|
-
return result;
|
|
849
|
-
}
|
|
850
|
-
function synthesizeTerminalEvent(terminalEventJson) {
|
|
851
|
-
let parsed;
|
|
852
|
-
try {
|
|
853
|
-
parsed = JSON.parse(terminalEventJson);
|
|
854
|
-
}
|
|
855
|
-
catch {
|
|
856
|
-
return null;
|
|
857
|
-
}
|
|
858
|
-
if (typeof parsed !== "object" ||
|
|
859
|
-
parsed === null ||
|
|
860
|
-
!("type" in parsed) ||
|
|
861
|
-
!("seq" in parsed)) {
|
|
862
|
-
return null;
|
|
863
|
-
}
|
|
864
|
-
const body = parsed;
|
|
865
|
-
if (typeof body.seq !== "number")
|
|
866
|
-
return null;
|
|
867
|
-
const at = typeof body.at === "string" ? body.at : new Date().toISOString();
|
|
868
|
-
if (body.type === "RunCompleted") {
|
|
869
|
-
return { kind: "RunCompleted", seq: body.seq, at };
|
|
870
|
-
}
|
|
871
|
-
if (body.type === "RunCancelled") {
|
|
872
|
-
return { kind: "RunCancelled", seq: body.seq, at };
|
|
873
|
-
}
|
|
874
|
-
if (body.type === "RunFailed") {
|
|
875
|
-
// The wire schema makes `error.message` required when the event
|
|
876
|
-
// type is `RunFailed`. An event that doesn't carry one is a
|
|
877
|
-
// contract violation upstream of the supervisor; coercing it to an
|
|
878
|
-
// empty string would silently hide the producer bug.
|
|
879
|
-
if (typeof body.error?.message !== "string") {
|
|
880
|
-
throw new Error(`synthesizeTerminalEvent: RunFailed event missing required error.message`);
|
|
881
|
-
}
|
|
882
|
-
return {
|
|
883
|
-
kind: "RunFailed",
|
|
884
|
-
seq: body.seq,
|
|
885
|
-
at,
|
|
886
|
-
error: { message: body.error.message },
|
|
887
|
-
};
|
|
888
|
-
}
|
|
889
|
-
return null;
|
|
890
|
-
}
|
|
891
1686
|
function activeControlSender() {
|
|
892
1687
|
if (state.phase === "starting" ||
|
|
893
1688
|
state.phase === "running" ||
|
|
@@ -896,14 +1691,6 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
896
1691
|
}
|
|
897
1692
|
return null;
|
|
898
1693
|
}
|
|
899
|
-
function activeTerminalBroadcaster() {
|
|
900
|
-
if (state.phase === "starting" ||
|
|
901
|
-
state.phase === "running" ||
|
|
902
|
-
state.phase === "recycling") {
|
|
903
|
-
return state.terminalBroadcaster;
|
|
904
|
-
}
|
|
905
|
-
return null;
|
|
906
|
-
}
|
|
907
1694
|
async function wireChild(args) {
|
|
908
1695
|
const controlSender = createControlChannelSender({
|
|
909
1696
|
privateKeySeed: args.ipcKeypair.privateKey,
|
|
@@ -921,10 +1708,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
921
1708
|
hmacKey: args.hmacKey,
|
|
922
1709
|
channelId: args.channelId,
|
|
923
1710
|
reader: args.handle.eventReader,
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1711
|
+
// Route event-channel crashes through the same funnel as
|
|
1712
|
+
// control-channel crashes so both drive the respawn/crash-loop path
|
|
1713
|
+
// uniformly on the live cohort (and defer to the owning lifecycle
|
|
1714
|
+
// path in every other phase).
|
|
1715
|
+
onCrash: onChildCrash,
|
|
928
1716
|
});
|
|
929
1717
|
const eventPump = pumpEvents(eventIter, args.onInferenceEvent);
|
|
930
1718
|
return {
|
|
@@ -951,7 +1739,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
951
1739
|
channelId,
|
|
952
1740
|
hmacKey,
|
|
953
1741
|
hostPublicKey: ipcKeypair.publicKey,
|
|
954
|
-
|
|
1742
|
+
anchorRunId: bindings.anchorRunId,
|
|
955
1743
|
deploymentMailAddress: bindings.deploymentMailAddress,
|
|
956
1744
|
stepCount: bindings.stepCount,
|
|
957
1745
|
definitionHash: opts.definitionHash,
|
|
@@ -1016,6 +1804,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1016
1804
|
terminalBroadcaster: createTerminalBroadcaster(),
|
|
1017
1805
|
dispatchLoop: null,
|
|
1018
1806
|
replayDone: null,
|
|
1807
|
+
sweepDone: null,
|
|
1019
1808
|
};
|
|
1020
1809
|
// Everything from here to the successful `return` runs with the state
|
|
1021
1810
|
// record in "starting" (then "running"). A throw at any of these
|
|
@@ -1029,7 +1818,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1029
1818
|
repoStore: bindings.repoStore,
|
|
1030
1819
|
principal: bindings.readPrincipal,
|
|
1031
1820
|
stepOrder: opts.stepOrder,
|
|
1032
|
-
|
|
1821
|
+
anchorRunId: bindings.anchorRunId,
|
|
1033
1822
|
deriveStepAddress: bindings.deriveStepAddress,
|
|
1034
1823
|
...(bindings.deriveStepRepoId !== undefined
|
|
1035
1824
|
? { deriveStepRepoId: bindings.deriveStepRepoId }
|
|
@@ -1048,8 +1837,13 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1048
1837
|
// first `dequeueToProcessing` so a fresh inbound mail that lands
|
|
1049
1838
|
// during the replay window cannot ship ahead of the orphan once
|
|
1050
1839
|
// the replay completes.
|
|
1051
|
-
|
|
1052
|
-
|
|
1840
|
+
// One scan of `runs/` feeds both spawn-time recovery consumers: the
|
|
1841
|
+
// orphan replay (which gates dispatch) and the compaction sweep (which
|
|
1842
|
+
// does not). Sharing the walk keeps recovery off a second O(total-runs)
|
|
1843
|
+
// scan.
|
|
1844
|
+
const scanDone = scanRunsForBoot(bindings.repoStore, bindings.workflowRunRepoId);
|
|
1845
|
+
const replayDone = scanDone
|
|
1846
|
+
.then(({ ownedMessageIds }) => inboxPrimitives.replayProcessingToInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress, { ownedMessageIds }))
|
|
1053
1847
|
.then(() => {
|
|
1054
1848
|
wakeDispatch();
|
|
1055
1849
|
})
|
|
@@ -1067,7 +1861,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1067
1861
|
// sweep that picks up parked orphans. Left as logged
|
|
1068
1862
|
// best-effort until that lands.
|
|
1069
1863
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1070
|
-
logger.warn `
|
|
1864
|
+
logger.warn `boot recovery scan or processing replay failed on spawn: ${message}`;
|
|
1071
1865
|
});
|
|
1072
1866
|
// Hold the replay promise on the active-state record so
|
|
1073
1867
|
// `shutdownInternal` awaits its settlement before tearing the
|
|
@@ -1075,6 +1869,36 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1075
1869
|
// flight would otherwise leave the substrate write pending past
|
|
1076
1870
|
// the supervisor's exit.
|
|
1077
1871
|
state.replayDone = replayDone;
|
|
1872
|
+
// Re-seal runs a crash left terminal-but-per-event when their
|
|
1873
|
+
// fire-and-forget fold never ran. Unlike the replay above, this must
|
|
1874
|
+
// NOT gate dispatch: reclaiming leaked per-event files is housekeeping
|
|
1875
|
+
// and cannot be allowed to delay the first dequeue. Best-effort, held
|
|
1876
|
+
// on the active-state record so shutdown awaits its settlement (see the
|
|
1877
|
+
// `sweepDone` field docstring for the teardown-latency tradeoff).
|
|
1878
|
+
const sweepDone = scanDone
|
|
1879
|
+
.then(({ pendingSealRunIds }) => recoverInterruptedCompactions({
|
|
1880
|
+
substrate: bindings.repoStore,
|
|
1881
|
+
repoId: bindings.workflowRunRepoId,
|
|
1882
|
+
ref: bindings.workflowRunRef,
|
|
1883
|
+
anchorRunId: bindings.anchorRunId,
|
|
1884
|
+
pendingSealRunIds,
|
|
1885
|
+
}))
|
|
1886
|
+
.then(({ sealed, failed }) => {
|
|
1887
|
+
if (sealed > 0) {
|
|
1888
|
+
logger.info `recovery sweep sealed ${String(sealed)} interrupted run(s)`;
|
|
1889
|
+
}
|
|
1890
|
+
if (failed.length > 0) {
|
|
1891
|
+
const detail = failed
|
|
1892
|
+
.map((f) => `${f.runId} (${f.message})`)
|
|
1893
|
+
.join("; ");
|
|
1894
|
+
logger.warn `recovery sweep left ${String(failed.length)} run(s) unsealed: ${detail}`;
|
|
1895
|
+
}
|
|
1896
|
+
})
|
|
1897
|
+
.catch((cause) => {
|
|
1898
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1899
|
+
logger.warn `boot recovery scan or compaction sweep failed on spawn: ${message}`;
|
|
1900
|
+
});
|
|
1901
|
+
state.sweepDone = sweepDone;
|
|
1078
1902
|
bindings.mailBus.registerAddress(bindings.deploymentMailAddress);
|
|
1079
1903
|
const mailUnsubscribe = bindings.mailBus.subscribeMailForAddress(bindings.deploymentMailAddress, onMailMessage);
|
|
1080
1904
|
state.mailUnsubscribe = mailUnsubscribe;
|
|
@@ -1124,19 +1948,47 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1124
1948
|
// same control channel `trigger.fire` uses, so the ordering
|
|
1125
1949
|
// guarantee (`grants-updated` lands before `trigger.fire`) holds
|
|
1126
1950
|
// for buffered and post-ready inbound mail alike.
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1951
|
+
//
|
|
1952
|
+
// Suppressed when `onRunStart` is wired: that binding makes the
|
|
1953
|
+
// dispatch loop push a per-run snapshot before each `trigger.fire`,
|
|
1954
|
+
// so the spawn-time push would only mask a broken per-run barrier
|
|
1955
|
+
// (the child would already hold grants and never hit its throw-on-
|
|
1956
|
+
// null guard). The per-run push is then the sole grants source.
|
|
1957
|
+
if (bindings.onRunStart === undefined) {
|
|
1958
|
+
await wired.wiring.controlSender.send({
|
|
1959
|
+
type: "grants-updated",
|
|
1960
|
+
data: {
|
|
1961
|
+
snapshot: {
|
|
1962
|
+
steps: credentialsSnapshot.steps.map((s) => ({
|
|
1963
|
+
stepId: s.stepId,
|
|
1964
|
+
address: s.address,
|
|
1965
|
+
grants: [...s.grants],
|
|
1966
|
+
contentHash: s.contentHash,
|
|
1967
|
+
})),
|
|
1968
|
+
},
|
|
1137
1969
|
},
|
|
1138
|
-
}
|
|
1139
|
-
}
|
|
1970
|
+
});
|
|
1971
|
+
}
|
|
1972
|
+
// Deliver the run's credential material to the child on EVERY spawn, not
|
|
1973
|
+
// only through the per-trigger `onRunStart` barrier. A restored run resumes
|
|
1974
|
+
// from its parked state without a fresh `trigger.fire` (a signal wakes it),
|
|
1975
|
+
// so the barrier would never re-deliver the cell and the resumed run's
|
|
1976
|
+
// inference would fail closed. Seeding the live cell here at spawn lets an
|
|
1977
|
+
// offline restart resolve each source's credential from the persisted
|
|
1978
|
+
// (unsealed) delivery without waiting on a hub reconnect. Unlike the grants
|
|
1979
|
+
// push above this is NOT suppressed when `onRunStart` is wired: the barrier
|
|
1980
|
+
// fires per trigger, but a resume has no trigger, so the spawn push is the
|
|
1981
|
+
// only credential source on the resume path. Idempotent with the barrier's
|
|
1982
|
+
// own push on a fresh run: the child merges both, and both carry the same
|
|
1983
|
+
// mirror. Absent when the deployment binds no credentials. Reads the live
|
|
1984
|
+
// mirror, not the frozen deploy delivery, so a credential revoked earlier
|
|
1985
|
+
// this process stays evicted.
|
|
1986
|
+
if (currentCredentialDelivery !== null) {
|
|
1987
|
+
await wired.wiring.controlSender.send({
|
|
1988
|
+
type: "credentials-updated",
|
|
1989
|
+
data: { delivery: currentCredentialDelivery },
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
1140
1992
|
// Transition to running. The dispatch loop (started below)
|
|
1141
1993
|
// picks up any pre-ready buffered mail through the FIFO inbox
|
|
1142
1994
|
// queue rather than through an in-memory buffer; arrival order
|
|
@@ -1169,7 +2021,13 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1169
2021
|
terminalBroadcaster: startingPhaseBroadcaster,
|
|
1170
2022
|
dispatchLoop,
|
|
1171
2023
|
replayDone,
|
|
2024
|
+
sweepDone,
|
|
1172
2025
|
};
|
|
2026
|
+
// Bump the generation and arm the exit-watcher atomically with the
|
|
2027
|
+
// running transition (no await between the swap above and this call)
|
|
2028
|
+
// so an unexpected exit of this child is classified against the
|
|
2029
|
+
// right generation.
|
|
2030
|
+
armChildForRunning(handle);
|
|
1173
2031
|
// Kick the dispatch loop in case mail landed in the inbox
|
|
1174
2032
|
// before the loop's first `await dispatchWake`. A wake against a
|
|
1175
2033
|
// freshly-minted promise is a no-op; the dispatch loop's first
|
|
@@ -1200,6 +2058,18 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1200
2058
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1201
2059
|
logger.error `upstream control pump failed: ${message}`;
|
|
1202
2060
|
});
|
|
2061
|
+
// Trigger A: re-register every correlation the freshly-ready child is
|
|
2062
|
+
// parked on. A `park.notify` register can be lost while the hub is down
|
|
2063
|
+
// at the original suspend; a child that resumes such a parked run (a
|
|
2064
|
+
// sidecar restart re-spawning this deployment, or a recycle -- see the
|
|
2065
|
+
// matching call in `installNewChild`) re-parks without re-emitting, so
|
|
2066
|
+
// the supervisor re-drives it from every re-establishment. Fire-and-
|
|
2067
|
+
// forget after the pump is armed to route the response: best-effort,
|
|
2068
|
+
// watchdog-bounded, and an empty round-trip when nothing is parked.
|
|
2069
|
+
void reEmitParkedCorrelations().catch((cause) => {
|
|
2070
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
2071
|
+
logger.warn `re-emit of parked correlations on re-establishment failed: ${message}`;
|
|
2072
|
+
});
|
|
1203
2073
|
// Arm the recycle policy. The policy is a no-op when all bounds
|
|
1204
2074
|
// are `undefined`; bounds resolution lives inside `createRecyclePolicy`.
|
|
1205
2075
|
if (bindings.recyclePolicy !== undefined) {
|
|
@@ -1318,60 +2188,200 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1318
2188
|
}
|
|
1319
2189
|
/**
|
|
1320
2190
|
* Forward one dequeued inbox entry to the child as `trigger.fire`
|
|
1321
|
-
* and record its runId as in-flight. The runId is the
|
|
1322
|
-
*
|
|
1323
|
-
*
|
|
1324
|
-
* `
|
|
2191
|
+
* and record its runId as in-flight. The runId is the local part of the
|
|
2192
|
+
* deployment's mail address (see `deriveWorkflowRunId`), identifying its one
|
|
2193
|
+
* top-level run. The resolved `Mail` (headers plus committed part references)
|
|
2194
|
+
* rides in the frame as the run's trigger payload; the `messageId`
|
|
2195
|
+
* accompanies it for correlation and audit. The runId is the same value the
|
|
2196
|
+
* dispatch loop waits on via `terminalEventSource`.
|
|
1325
2197
|
*/
|
|
1326
|
-
async function forwardDispatchedEntry(sender, messageId, receivedAt) {
|
|
2198
|
+
async function forwardDispatchedEntry(sender, messageId, receivedAt, runId, payload) {
|
|
1327
2199
|
await sender.send({
|
|
1328
2200
|
type: "trigger.fire",
|
|
1329
2201
|
data: {
|
|
1330
|
-
runId
|
|
2202
|
+
runId,
|
|
1331
2203
|
messageId,
|
|
1332
2204
|
receivedAt,
|
|
2205
|
+
payload,
|
|
1333
2206
|
},
|
|
1334
2207
|
});
|
|
1335
|
-
|
|
1336
|
-
return
|
|
2208
|
+
cohortRunIds.add(runId);
|
|
2209
|
+
return runId;
|
|
2210
|
+
}
|
|
2211
|
+
/**
|
|
2212
|
+
* Resolve a dequeued inbound mail to the run's input: a decoded `Mail`
|
|
2213
|
+
* (headers plus part descriptors that reference the part bytes committed to
|
|
2214
|
+
* the workflow-run substrate). The supervisor is the sole mail owner and
|
|
2215
|
+
* commits the parts here (a direct workflow-run write; the workflow child's
|
|
2216
|
+
* control loop cannot do a synchronous proxied write without deadlock), so
|
|
2217
|
+
* both turns share this one preparation site.
|
|
2218
|
+
*
|
|
2219
|
+
* The two failure modes are deliberately distinct:
|
|
2220
|
+
* - A DETERMINISTIC input rejection -- missing bytes, unparseable MIME, or
|
|
2221
|
+
* a messageId that cannot form a path segment -- returns `{ ok: false }`
|
|
2222
|
+
* so the caller drops the mail. Replaying it would fail identically.
|
|
2223
|
+
* - A TRANSIENT substrate write failure propagates (thrown), so the caller
|
|
2224
|
+
* treats it as a dispatch fault and leaves the mail reclaimable rather
|
|
2225
|
+
* than silently discarding it on an infrastructure hiccup.
|
|
2226
|
+
*/
|
|
2227
|
+
async function prepareMail(envelope, runId) {
|
|
2228
|
+
if (envelope.rawMessage === undefined) {
|
|
2229
|
+
return {
|
|
2230
|
+
ok: false,
|
|
2231
|
+
rejection: {
|
|
2232
|
+
code: "malformed_mail",
|
|
2233
|
+
message: `inbound mail ${envelope.messageId} carries no rawMessage bytes`,
|
|
2234
|
+
},
|
|
2235
|
+
};
|
|
2236
|
+
}
|
|
2237
|
+
let decoded;
|
|
2238
|
+
try {
|
|
2239
|
+
decoded = decodeMail(base64Decode(envelope.rawMessage));
|
|
2240
|
+
}
|
|
2241
|
+
catch (cause) {
|
|
2242
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
2243
|
+
return {
|
|
2244
|
+
ok: false,
|
|
2245
|
+
rejection: {
|
|
2246
|
+
code: "malformed_mail",
|
|
2247
|
+
message: `inbound mail ${envelope.messageId} could not be decoded: ${message}`,
|
|
2248
|
+
},
|
|
2249
|
+
};
|
|
2250
|
+
}
|
|
2251
|
+
const writePrincipal = {
|
|
2252
|
+
kind: "supervisor",
|
|
2253
|
+
anchorRunId: bindings.anchorRunId,
|
|
2254
|
+
};
|
|
2255
|
+
try {
|
|
2256
|
+
const mail = await commitMail({
|
|
2257
|
+
substrate: bindings.repoStore,
|
|
2258
|
+
repoId: bindings.workflowRunRepoId,
|
|
2259
|
+
principal: writePrincipal,
|
|
2260
|
+
runId,
|
|
2261
|
+
ref: bindings.workflowRunRef,
|
|
2262
|
+
}, envelope.messageId, decoded);
|
|
2263
|
+
return { ok: true, mail };
|
|
2264
|
+
}
|
|
2265
|
+
catch (cause) {
|
|
2266
|
+
if (cause instanceof InvalidMailError) {
|
|
2267
|
+
return {
|
|
2268
|
+
ok: false,
|
|
2269
|
+
rejection: { code: "malformed_mail", message: cause.message },
|
|
2270
|
+
};
|
|
2271
|
+
}
|
|
2272
|
+
throw cause;
|
|
2273
|
+
}
|
|
2274
|
+
}
|
|
2275
|
+
/**
|
|
2276
|
+
* Push the run's grants snapshot to the child ahead of its
|
|
2277
|
+
* `trigger.fire`. Returns `true` if the barrier FAILED (the caller must
|
|
2278
|
+
* skip the fire; the run has already been settled as `RunFailed`) and
|
|
2279
|
+
* `false` if the barrier passed or is not armed (`onRunStart` unwired,
|
|
2280
|
+
* where `spawn` supplied the snapshot instead).
|
|
2281
|
+
*
|
|
2282
|
+
* The sink is a request/response contract: the supervisor awaits the
|
|
2283
|
+
* returned snapshot and awaits the `grants-updated` send so both land on
|
|
2284
|
+
* the child's control channel before the trigger. A throw from either --
|
|
2285
|
+
* the sink itself or the control send -- is surfaced as a synthesized
|
|
2286
|
+
* `RunFailed` fanned out to this run's broadcaster watcher, never
|
|
2287
|
+
* swallowed, so the run fails deterministically instead of the child
|
|
2288
|
+
* authorizing against a stale or absent snapshot.
|
|
2289
|
+
*/
|
|
2290
|
+
async function pushRunGrants(sender, runId, broadcaster) {
|
|
2291
|
+
if (bindings.onRunStart === undefined)
|
|
2292
|
+
return false;
|
|
2293
|
+
try {
|
|
2294
|
+
const snapshot = await bindings.onRunStart({
|
|
2295
|
+
runId,
|
|
2296
|
+
anchorRunId: bindings.anchorRunId,
|
|
2297
|
+
});
|
|
2298
|
+
await sender.send({
|
|
2299
|
+
type: "grants-updated",
|
|
2300
|
+
data: {
|
|
2301
|
+
snapshot: {
|
|
2302
|
+
steps: snapshot.steps.map((s) => ({
|
|
2303
|
+
stepId: s.stepId,
|
|
2304
|
+
address: s.address,
|
|
2305
|
+
grants: [...s.grants],
|
|
2306
|
+
contentHash: s.contentHash,
|
|
2307
|
+
})),
|
|
2308
|
+
},
|
|
2309
|
+
},
|
|
2310
|
+
});
|
|
2311
|
+
// Deliver the deployment's credential material on the same pre-trigger
|
|
2312
|
+
// barrier, so a tool that resolves a credential on the first step already
|
|
2313
|
+
// has it in the child's cell. Reads the live mirror, not the frozen deploy
|
|
2314
|
+
// delivery: a rotation or revocation delivered earlier via
|
|
2315
|
+
// `deliverCredentials` is reflected here, and a recycled child (seeded only
|
|
2316
|
+
// by this barrier) inherits the current set instead of the deploy-time one.
|
|
2317
|
+
// Absent when the deployment binds none.
|
|
2318
|
+
if (currentCredentialDelivery !== null) {
|
|
2319
|
+
await sender.send({
|
|
2320
|
+
type: "credentials-updated",
|
|
2321
|
+
data: { delivery: currentCredentialDelivery },
|
|
2322
|
+
});
|
|
2323
|
+
}
|
|
2324
|
+
return false;
|
|
2325
|
+
}
|
|
2326
|
+
catch (cause) {
|
|
2327
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
2328
|
+
logger.error `onRunStart grants barrier failed for run ${runId}; failing the run: ${message}`;
|
|
2329
|
+
broadcaster.notify(runId, {
|
|
2330
|
+
kind: "RunFailed",
|
|
2331
|
+
seq: 0,
|
|
2332
|
+
at: new Date().toISOString(),
|
|
2333
|
+
error: {
|
|
2334
|
+
message: `workflow-host supervisor: run ${runId} not authorized; grants barrier failed before trigger.fire: ${message}`,
|
|
2335
|
+
},
|
|
2336
|
+
});
|
|
2337
|
+
return true;
|
|
2338
|
+
}
|
|
1337
2339
|
}
|
|
1338
2340
|
/**
|
|
1339
2341
|
* One iteration of the dispatch loop: dequeue the FIFO-first inbox
|
|
1340
|
-
* entry,
|
|
1341
|
-
*
|
|
1342
|
-
*
|
|
1343
|
-
*
|
|
1344
|
-
*
|
|
2342
|
+
* entry, decide whether to `signal.deliver` or `trigger.fire` (or wait
|
|
2343
|
+
* if the run is in-flight but not yet parked), then `markConsumed`
|
|
2344
|
+
* once the child has taken up the message -- for a `trigger.fire` that
|
|
2345
|
+
* means after the run reaches a terminal event or parks, so the
|
|
2346
|
+
* claim-check entry the child still needs to read is not deleted out
|
|
2347
|
+
* from under it. Returns `true` if a dispatch landed (caller should
|
|
2348
|
+
* loop immediately) and `false` if the inbox was empty (caller should
|
|
2349
|
+
* await the next wake).
|
|
1345
2350
|
*/
|
|
1346
2351
|
async function dispatchOne(sender, cohortAbort, broadcaster) {
|
|
1347
2352
|
if (cohortAbort.signal.aborted)
|
|
1348
2353
|
return false;
|
|
1349
|
-
// Subscribe to the terminal broadcaster BEFORE forwarding the
|
|
1350
|
-
// trigger.fire so a terminal event the child notifies between
|
|
1351
|
-
// forward and subscribe cannot be missed. The broadcaster fires
|
|
1352
|
-
// its listeners synchronously inside `notify`; with the subscribe
|
|
1353
|
-
// ordered first the listener buffers the event until the
|
|
1354
|
-
// dispatch loop's `iter.next()` consumes it.
|
|
1355
2354
|
const beforeDequeueMs = dispatchTimingEnabled() ? performance.now() : 0;
|
|
1356
2355
|
const dequeued = await inboxPrimitives.dequeueToProcessing(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress);
|
|
1357
2356
|
if (dequeued === null)
|
|
1358
2357
|
return false;
|
|
1359
2358
|
const envelope = dequeued.envelope;
|
|
1360
|
-
const runId =
|
|
1361
|
-
|
|
1362
|
-
|
|
2359
|
+
const runId = deriveWorkflowRunId(bindings.deploymentMailAddress);
|
|
2360
|
+
const messageId = envelope.messageId;
|
|
2361
|
+
let rejection;
|
|
2362
|
+
const rejectTerminalRun = () => {
|
|
2363
|
+
if (rejection !== undefined)
|
|
2364
|
+
return;
|
|
2365
|
+
rejection = {
|
|
2366
|
+
code: "workflow_run_terminal",
|
|
2367
|
+
message: `Workflow run ${runId} is terminal and cannot be fired again`,
|
|
2368
|
+
};
|
|
2369
|
+
logger.warn `rejecting inbound mail ${messageId}: workflow run ${runId} is terminal`;
|
|
2370
|
+
};
|
|
2371
|
+
currentDispatchMessageId = messageId;
|
|
2372
|
+
emitDispatchTiming(messageId, "dispatch-start", beforeDequeueMs);
|
|
1363
2373
|
// D2 leg: the claim-check dequeue READ. `dispatch-start` is sampled
|
|
1364
2374
|
// BEFORE the dequeue (so the roundtrip bracket includes the read);
|
|
1365
2375
|
// the dequeue leg's own start mark is that same pre-dequeue sample
|
|
1366
2376
|
// re-stamped under the leg channel, and its end is now (the read just
|
|
1367
2377
|
// completed). Emitting the start retroactively here -- rather than
|
|
1368
|
-
// before the await -- keeps the leg keyed by the
|
|
1369
|
-
// known after the dequeue resolves.
|
|
2378
|
+
// before the await -- keeps the leg keyed by the messageId, which is
|
|
2379
|
+
// only known after the dequeue resolves.
|
|
1370
2380
|
if (bindings.onDispatchTiming !== undefined) {
|
|
1371
2381
|
try {
|
|
1372
2382
|
bindings.onDispatchTiming({
|
|
1373
2383
|
kind: "leg",
|
|
1374
|
-
|
|
2384
|
+
messageId,
|
|
1375
2385
|
leg: "dequeue",
|
|
1376
2386
|
phase: "start",
|
|
1377
2387
|
atMs: beforeDequeueMs,
|
|
@@ -1379,28 +2389,233 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1379
2389
|
}
|
|
1380
2390
|
catch (cause) {
|
|
1381
2391
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1382
|
-
logger.warn `onDispatchTiming leg observer threw for ${
|
|
2392
|
+
logger.warn `onDispatchTiming leg observer threw for ${messageId} (dequeue start): ${message}`;
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2395
|
+
legMarkEnd(messageId, "dequeue");
|
|
2396
|
+
// In-memory cohort membership is deliberately not the lifecycle
|
|
2397
|
+
// authority: it is empty for a new run, after a terminal frame, and while
|
|
2398
|
+
// a child is rediscovering a live run after restart. Consult the durable
|
|
2399
|
+
// log before deciding that "no cohort" means "fire". A live log is added
|
|
2400
|
+
// back to cohort tracking so the existing terminal/park wait handles the
|
|
2401
|
+
// recovery window; a terminal log is rejected permanently.
|
|
2402
|
+
if (!cohortRunIds.has(runId)) {
|
|
2403
|
+
const lifecycle = terminalRunIds.has(runId)
|
|
2404
|
+
? "terminal"
|
|
2405
|
+
: await readWorkflowRunLifecycle(bindings.repoStore, bindings.workflowRunRepoId, runId);
|
|
2406
|
+
if (lifecycle === "terminal") {
|
|
2407
|
+
rejectTerminalRun();
|
|
2408
|
+
}
|
|
2409
|
+
else if (lifecycle === "live") {
|
|
2410
|
+
cohortRunIds.add(runId);
|
|
2411
|
+
}
|
|
2412
|
+
}
|
|
2413
|
+
if (rejection === undefined) {
|
|
2414
|
+
// Subscribe to the terminal broadcaster BEFORE the grants barrier so
|
|
2415
|
+
// a synthetic `RunFailed` from a barrier failure can be captured.
|
|
2416
|
+
const preIter = broadcaster.source(runId)[Symbol.asyncIterator]();
|
|
2417
|
+
// Per-run grants barrier. When `onRunStart` is wired, push this run's
|
|
2418
|
+
// grants snapshot BEFORE the trigger/signal so the child's authorize
|
|
2419
|
+
// closure binds to it rather than throwing on a null snapshot. The push
|
|
2420
|
+
// and the fire share the child's control channel, so a `grants-updated`
|
|
2421
|
+
// awaited here is observed by the child ahead of the trigger. A barrier
|
|
2422
|
+
// failure (the sink throws, or the push fails) fails the run loudly --
|
|
2423
|
+
// a synthesized `RunFailed` fanned out to this run's watcher -- and the
|
|
2424
|
+
// trigger is NOT fired, so no step ever runs against absent grants.
|
|
2425
|
+
const barrierFailed = await pushRunGrants(sender, runId, broadcaster);
|
|
2426
|
+
if (barrierFailed) {
|
|
2427
|
+
// Wait for the synthetic RunFailed before consuming the message.
|
|
2428
|
+
await waitForRunTerminal(preIter, cohortAbort.signal);
|
|
2429
|
+
// Clean up for synthetic barrier failure (real terminal events are
|
|
2430
|
+
// cleaned up by pumpUpstreamControl, but synthetic ones are not).
|
|
2431
|
+
cohortRunIds.delete(runId);
|
|
2432
|
+
runInputChannels.delete(runId);
|
|
2433
|
+
}
|
|
2434
|
+
else {
|
|
2435
|
+
// Dispose the pre-created iterator; the normal path creates fresh
|
|
2436
|
+
// iterators inside the dispatch-decision loop when waiting.
|
|
2437
|
+
if (typeof preIter.return === "function") {
|
|
2438
|
+
await preIter.return();
|
|
2439
|
+
}
|
|
2440
|
+
// Unified dispatch: park → signal.deliver; no live run → trigger.fire;
|
|
2441
|
+
// in-flight but undecided → wait for terminal or park, then re-evaluate.
|
|
2442
|
+
while (!cohortAbort.signal.aborted) {
|
|
2443
|
+
if (terminalRunIds.has(runId)) {
|
|
2444
|
+
rejectTerminalRun();
|
|
2445
|
+
break;
|
|
2446
|
+
}
|
|
2447
|
+
// Capture the park generation BEFORE this iteration's pre-wait awaits
|
|
2448
|
+
// so `waitForRunTerminalOrPark` can accept a strictly-newer park that
|
|
2449
|
+
// fires during them (see the latch there). Re-captured each iteration
|
|
2450
|
+
// so a wait that returned "parked" is not re-counted next time around.
|
|
2451
|
+
const sinceGen = parkGenerations.get(runId) ?? 0;
|
|
2452
|
+
// Routing hygiene (defense-in-depth the latch never depends on): a
|
|
2453
|
+
// runInputChannels entry with no live run in this cohort is a stale
|
|
2454
|
+
// routing hazard left by a dead incarnation. Drop it BEFORE the signal
|
|
2455
|
+
// branch so a fresh mail cannot be routed onto a dead run's
|
|
2456
|
+
// correlation. The resumed.runs invariant (cohortRunIds registered
|
|
2457
|
+
// before its input channel) keeps a LIVE resumed run out of this
|
|
2458
|
+
// branch, so this only ever drops genuinely-dead entries.
|
|
2459
|
+
if (!cohortRunIds.has(runId) && runInputChannels.has(runId)) {
|
|
2460
|
+
runInputChannels.delete(runId);
|
|
2461
|
+
}
|
|
2462
|
+
const inputChannel = runInputChannels.get(runId);
|
|
2463
|
+
if (inputChannel !== undefined) {
|
|
2464
|
+
// Resolve the inbound mail to the run's input HERE, the single site
|
|
2465
|
+
// that knows this payload's provenance is mail, applying the SAME
|
|
2466
|
+
// preparation the turn-1 trigger does. The signal.deliver frame's
|
|
2467
|
+
// payload is the resume decision in FINAL form -- a Mail (headers plus committed part references); deliverSignal's structured signals ship their own
|
|
2468
|
+
// payload unchanged. Done BEFORE minting the terminal watcher so a
|
|
2469
|
+
// failure here cannot leak an un-finalized iterator.
|
|
2470
|
+
const prepared = await prepareMail(envelope, runId);
|
|
2471
|
+
if (!prepared.ok) {
|
|
2472
|
+
// A DETERMINISTICALLY malformed turn-2 mail cannot resume the
|
|
2473
|
+
// parked agent. DROP it: log loudly and consume it (break to the
|
|
2474
|
+
// post-loop markConsumed) rather than throwing -- replay would
|
|
2475
|
+
// re-deliver the same poison mail forever. The run stays parked
|
|
2476
|
+
// on its current correlation, ready for the next valid mail; one
|
|
2477
|
+
// bad mail must not tear down a long-lived conversation. A
|
|
2478
|
+
// TRANSIENT write failure is NOT caught here: `prepareMail`
|
|
2479
|
+
// throws it, so it propagates as a dispatch fault and the mail
|
|
2480
|
+
// stays reclaimable for retry.
|
|
2481
|
+
logger.error `signal.deliver for run ${runId}: dropping malformed inbound mail ${envelope.messageId}: ${prepared.rejection.message}`;
|
|
2482
|
+
break;
|
|
2483
|
+
}
|
|
2484
|
+
// Mint the terminal watcher only now, after the payload resolved, so
|
|
2485
|
+
// a terminal the resumed run reaches right after applying the signal
|
|
2486
|
+
// is not missed; the park watcher is armed inside
|
|
2487
|
+
// waitForRunTerminalOrPark.
|
|
2488
|
+
const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
|
|
2489
|
+
let waitEntered = false;
|
|
2490
|
+
try {
|
|
2491
|
+
await sender.send({
|
|
2492
|
+
type: "signal.deliver",
|
|
2493
|
+
data: {
|
|
2494
|
+
runId,
|
|
2495
|
+
signalName: signalName(inputChannel.correlationId),
|
|
2496
|
+
signalId: envelope.messageId,
|
|
2497
|
+
payload: prepared.mail,
|
|
2498
|
+
},
|
|
2499
|
+
});
|
|
2500
|
+
// Invalidate the cached input channel: its correlation is now
|
|
2501
|
+
// consumed by this delivery, so the NEXT mail must not reuse it.
|
|
2502
|
+
// The resumed run re-parks on a FRESH correlation (a new
|
|
2503
|
+
// park.notify repopulates runInputChannels); a mail arriving before
|
|
2504
|
+
// that re-park waits via the in-flight branch rather than
|
|
2505
|
+
// delivering onto the stale channel. Routing hygiene only -- the
|
|
2506
|
+
// wait keys on the park-generation edge, not this level state.
|
|
2507
|
+
runInputChannels.delete(runId);
|
|
2508
|
+
// The message was dispatched as a turn: mark its eager mailbox
|
|
2509
|
+
// entry \Seen/$Processed. Fire-and-forget off the dispatch path.
|
|
2510
|
+
markMailboxProcessed(envelope.messageId);
|
|
2511
|
+
// Durable-consume contract, mirroring the trigger.fire path: hold
|
|
2512
|
+
// markConsumed until the child has durably taken up the signal --
|
|
2513
|
+
// the resumed run re-parks or reaches a terminal event. That gate
|
|
2514
|
+
// is downstream of durability DESPITE the child's fire-and-forget
|
|
2515
|
+
// SignalReceived writer: the runtime reaches re-park/terminal only
|
|
2516
|
+
// by resuming from the COMMITTED SignalReceived, which its per-run
|
|
2517
|
+
// subscribeKind substrate subscription surfaces only after the
|
|
2518
|
+
// commit lands -- so the substrate subscription IS the ack, and a
|
|
2519
|
+
// failed deliver commit is observed by nothing, never re-parks, and
|
|
2520
|
+
// never releases markConsumed (the mail stays reclaimable). A crash
|
|
2521
|
+
// before the re-park/terminal leaves the claim-check entry in
|
|
2522
|
+
// processing/, so replayProcessingToInbox re-delivers the signal on
|
|
2523
|
+
// restart. On cohort abort the wait returns and the post-loop guard
|
|
2524
|
+
// skips markConsumed.
|
|
2525
|
+
waitEntered = true;
|
|
2526
|
+
await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
|
|
2527
|
+
}
|
|
2528
|
+
finally {
|
|
2529
|
+
// waitForRunTerminalOrPark finalizes the iterator it consumes; the
|
|
2530
|
+
// only leak is when `sender.send` throws before the wait is
|
|
2531
|
+
// entered, so finalize only in that case.
|
|
2532
|
+
if (!waitEntered && typeof iter.return === "function") {
|
|
2533
|
+
await iter.return(undefined).catch(() => {
|
|
2534
|
+
/* best-effort finalisation of the watcher iterator. */
|
|
2535
|
+
});
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2538
|
+
break;
|
|
2539
|
+
}
|
|
2540
|
+
if (!cohortRunIds.has(runId)) {
|
|
2541
|
+
// Resolve the inbound mail to the run's input before firing. A
|
|
2542
|
+
// DETERMINISTICALLY malformed first trigger cannot start the run:
|
|
2543
|
+
// record the rejection on the consumed entry and drop it (break to
|
|
2544
|
+
// the post-loop markConsumed), since replay would fail identically.
|
|
2545
|
+
// A TRANSIENT write failure instead propagates from
|
|
2546
|
+
// `prepareMail` as a dispatch fault, leaving the mail
|
|
2547
|
+
// reclaimable. Unlike a turn-2 parse failure (which leaves a live
|
|
2548
|
+
// run parked), a malformed first trigger produces no run at all --
|
|
2549
|
+
// the rejection surfaces on the consumed entry, not as a RunFailed
|
|
2550
|
+
// terminal event.
|
|
2551
|
+
const prepared = await prepareMail(envelope, runId);
|
|
2552
|
+
if (!prepared.ok) {
|
|
2553
|
+
if (rejection === undefined)
|
|
2554
|
+
rejection = prepared.rejection;
|
|
2555
|
+
logger.error `trigger.fire for run ${runId}: rejecting malformed inbound mail ${envelope.messageId}: ${prepared.rejection.message}`;
|
|
2556
|
+
break;
|
|
2557
|
+
}
|
|
2558
|
+
// Subscribe the terminal watcher BEFORE the trigger fires. The
|
|
2559
|
+
// broadcaster drops a notify that has no listener (its subscribe-
|
|
2560
|
+
// before-fire contract), so a terminal that lands while
|
|
2561
|
+
// forwardDispatchedEntry is in flight would be lost and the wait
|
|
2562
|
+
// would hang to the backstop.
|
|
2563
|
+
const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
|
|
2564
|
+
let waitEntered = false;
|
|
2565
|
+
try {
|
|
2566
|
+
await forwardDispatchedEntry(sender, envelope.messageId, envelope.receivedAt, runId, prepared.mail);
|
|
2567
|
+
// The message was dispatched as a turn: mark its eager mailbox
|
|
2568
|
+
// entry \Seen/$Processed. Fire-and-forget off the dispatch path.
|
|
2569
|
+
markMailboxProcessed(envelope.messageId);
|
|
2570
|
+
// Wait for the child to durably take up this trigger (RunStarted
|
|
2571
|
+
// committed, then the run parks or terminates) before allowing
|
|
2572
|
+
// `markConsumed` to move the claim-check entry out of
|
|
2573
|
+
// `processing/`. The payload now rides the frame, so the child no
|
|
2574
|
+
// longer reads it from the entry -- but the durable-consume
|
|
2575
|
+
// contract still holds markConsumed until the run's uptake is
|
|
2576
|
+
// committed, so a crash before RunStarted leaves the entry in
|
|
2577
|
+
// processing/ for replayProcessingToInbox to re-deliver. On cohort
|
|
2578
|
+
// abort the wait returns and the post-loop guard skips
|
|
2579
|
+
// markConsumed.
|
|
2580
|
+
waitEntered = true;
|
|
2581
|
+
await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
|
|
2582
|
+
}
|
|
2583
|
+
finally {
|
|
2584
|
+
// waitForRunTerminalOrPark finalizes the iterator it consumes; the
|
|
2585
|
+
// only leak is when forward throws before the wait is
|
|
2586
|
+
// entered, so finalize only in that case.
|
|
2587
|
+
if (!waitEntered && typeof iter.return === "function") {
|
|
2588
|
+
await iter.return(undefined).catch(() => {
|
|
2589
|
+
/* best-effort finalisation of the watcher iterator. */
|
|
2590
|
+
});
|
|
2591
|
+
}
|
|
2592
|
+
}
|
|
2593
|
+
break;
|
|
2594
|
+
}
|
|
2595
|
+
const iter = broadcaster.source(runId)[Symbol.asyncIterator]();
|
|
2596
|
+
const outcome = await waitForRunTerminalOrPark(iter, cohortAbort.signal, runId, sinceGen);
|
|
2597
|
+
if (outcome === "aborted")
|
|
2598
|
+
break;
|
|
2599
|
+
if (outcome === "terminal") {
|
|
2600
|
+
// This message was waiting for an already-live run to expose its
|
|
2601
|
+
// next input correlation. The run terminated first, so the mail
|
|
2602
|
+
// was never delivered and must not fall through to trigger.fire.
|
|
2603
|
+
rejectTerminalRun();
|
|
2604
|
+
break;
|
|
2605
|
+
}
|
|
2606
|
+
// Continue loop: re-evaluate runInputChannels / cohortRunIds
|
|
2607
|
+
}
|
|
1383
2608
|
}
|
|
1384
2609
|
}
|
|
1385
|
-
legMarkEnd(runId, "dequeue");
|
|
1386
|
-
const iterable = broadcaster.source(runId);
|
|
1387
|
-
const iter = iterable[Symbol.asyncIterator]();
|
|
1388
|
-
await forwardDispatchedEntry(sender, envelope.messageId, envelope.receivedAt);
|
|
1389
|
-
await waitForRunTerminal(iter, cohortAbort.signal);
|
|
1390
|
-
emitDispatchTiming(runId, "reply-produced", performance.now());
|
|
1391
|
-
inFlightRuns.delete(runId);
|
|
1392
2610
|
if (cohortAbort.signal.aborted) {
|
|
1393
|
-
|
|
1394
|
-
// alongside it). Skip `markConsumed` so the recycle path's
|
|
1395
|
-
// drain-side replay can reclaim the processing entry.
|
|
1396
|
-
currentDispatchRunId = null;
|
|
1397
|
-
resolveMarkConsumedWaiter(runId);
|
|
2611
|
+
currentDispatchMessageId = null;
|
|
1398
2612
|
return false;
|
|
1399
2613
|
}
|
|
1400
|
-
|
|
1401
|
-
//
|
|
1402
|
-
// the leg mark makes the
|
|
1403
|
-
|
|
2614
|
+
emitDispatchTiming(messageId, "reply-produced", performance.now());
|
|
2615
|
+
// D2 leg: `markConsumed` is paid AFTER `reply-produced`, so its growth
|
|
2616
|
+
// is invisible to the 4.7 round-trip bracket -- the leg mark makes the
|
|
2617
|
+
// out-of-window cost visible.
|
|
2618
|
+
legMarkStart(messageId, "markconsumed");
|
|
1404
2619
|
try {
|
|
1405
2620
|
await inboxPrimitives.markConsumed(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, {
|
|
1406
2621
|
address: bindings.deploymentMailAddress,
|
|
@@ -1408,17 +2623,21 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1408
2623
|
runId,
|
|
1409
2624
|
consumedAt: Date.now(),
|
|
1410
2625
|
retentionHorizonMs: consumedRetentionMs,
|
|
2626
|
+
...(rejection !== undefined ? { rejection } : {}),
|
|
1411
2627
|
});
|
|
1412
2628
|
}
|
|
1413
2629
|
catch (cause) {
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
2630
|
+
// A markConsumed failure is fatal:
|
|
2631
|
+
// swallowing it treats the dispatch as complete while the mail is NOT
|
|
2632
|
+
// durably recorded consumed, hiding the failure and leaving a mail that
|
|
2633
|
+
// is neither cleanly consumed nor visibly failed. Propagate into the
|
|
2634
|
+
// dispatch fault handler so the failure surfaces and the mail stays
|
|
2635
|
+
// reclaimable.
|
|
2636
|
+
throw new Error(`failed to markConsumed for run ${runId}`, { cause });
|
|
2637
|
+
}
|
|
2638
|
+
legMarkEnd(messageId, "markconsumed");
|
|
1420
2639
|
maybeRepack(runId);
|
|
1421
|
-
|
|
2640
|
+
currentDispatchMessageId = null;
|
|
1422
2641
|
return true;
|
|
1423
2642
|
}
|
|
1424
2643
|
/**
|
|
@@ -1460,6 +2679,94 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1460
2679
|
}
|
|
1461
2680
|
}
|
|
1462
2681
|
}
|
|
2682
|
+
/**
|
|
2683
|
+
* Wait until the run's terminal event lands on the cohort
|
|
2684
|
+
* broadcaster's iterator, the child parks the run (bumping the park
|
|
2685
|
+
* generation past `sinceGen`), or the cohort aborts. Returns `"terminal"`
|
|
2686
|
+
* when a terminal event arrived, `"parked"` when the run parked,
|
|
2687
|
+
* and `"aborted"` when the cohort tore down. Throws when the backstop
|
|
2688
|
+
* fires (see `TERMINAL_OR_PARK_BACKSTOP_MS`).
|
|
2689
|
+
*
|
|
2690
|
+
* `sinceGen` is the park generation the CALLER captured before its pre-wait
|
|
2691
|
+
* awaits; the wait accepts only a STRICTLY NEWER park (`generation >
|
|
2692
|
+
* sinceGen`). Keying on that edge -- not `runInputChannels`' level state --
|
|
2693
|
+
* makes the wait's correctness local: a park during the pre-wait awaits is
|
|
2694
|
+
* observed even though its `resolveParkNotifyWaiter` no-op'd, and a stale
|
|
2695
|
+
* channel entry from a prior run or incarnation cannot false-positive.
|
|
2696
|
+
*/
|
|
2697
|
+
async function waitForRunTerminalOrPark(iter, abortSignal, runId, sinceGen) {
|
|
2698
|
+
let onAbort = null;
|
|
2699
|
+
const abortPromise = new Promise((resolve) => {
|
|
2700
|
+
if (abortSignal.aborted) {
|
|
2701
|
+
resolve({ source: "abort" });
|
|
2702
|
+
return;
|
|
2703
|
+
}
|
|
2704
|
+
onAbort = () => resolve({ source: "abort" });
|
|
2705
|
+
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
2706
|
+
});
|
|
2707
|
+
let parkResolve = null;
|
|
2708
|
+
const parkPromise = new Promise((resolve) => {
|
|
2709
|
+
parkResolve = () => resolve({ source: "park" });
|
|
2710
|
+
parkNotifyWaiters.set(runId, parkResolve);
|
|
2711
|
+
});
|
|
2712
|
+
let timeoutHandle;
|
|
2713
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
2714
|
+
timeoutHandle = setTimeout(() => resolve({ source: "timeout" }), TERMINAL_OR_PARK_BACKSTOP_MS);
|
|
2715
|
+
});
|
|
2716
|
+
try {
|
|
2717
|
+
if (abortSignal.aborted)
|
|
2718
|
+
return "aborted";
|
|
2719
|
+
// Check-after-register: read the generation now that the waiter above is
|
|
2720
|
+
// armed, SYNCHRONOUSLY (no await between arming and this read, so no
|
|
2721
|
+
// `park.notify` can interleave). A generation past `sinceGen` means the
|
|
2722
|
+
// run already parked -- during the caller's pre-wait awaits, before the
|
|
2723
|
+
// waiter armed, so `resolveParkNotifyWaiter` no-op'd and the armed
|
|
2724
|
+
// parkPromise would never fire -- and this catches it rather than hanging
|
|
2725
|
+
// to the backstop.
|
|
2726
|
+
if ((parkGenerations.get(runId) ?? 0) > sinceGen)
|
|
2727
|
+
return "parked";
|
|
2728
|
+
const result = await Promise.race([
|
|
2729
|
+
iter.next().then((r) => ({ source: "iter", r })),
|
|
2730
|
+
abortPromise,
|
|
2731
|
+
parkPromise,
|
|
2732
|
+
timeoutPromise,
|
|
2733
|
+
]);
|
|
2734
|
+
if (result.source === "abort")
|
|
2735
|
+
return "aborted";
|
|
2736
|
+
if (result.source === "park")
|
|
2737
|
+
return "parked";
|
|
2738
|
+
if (result.source === "timeout") {
|
|
2739
|
+
// Backstop against a lost wake or a wedged child: the run neither
|
|
2740
|
+
// parked, terminated, nor aborted within a generous window. Surface it
|
|
2741
|
+
// LOUDLY and throw so the dispatch fails -- the caller does not
|
|
2742
|
+
// markConsumed on a throw, so the mail stays reclaimable in
|
|
2743
|
+
// processing/ and is never consumed on the assumption the run
|
|
2744
|
+
// progressed.
|
|
2745
|
+
logger.error `waitForRunTerminalOrPark backstop fired for run ${runId} after ${TERMINAL_OR_PARK_BACKSTOP_MS}ms; failing the dispatch so the mail stays reclaimable`;
|
|
2746
|
+
throw new Error(`waitForRunTerminalOrPark backstop: run ${runId} did not park or terminate within ${TERMINAL_OR_PARK_BACKSTOP_MS}ms`);
|
|
2747
|
+
}
|
|
2748
|
+
if (result.r.done === true)
|
|
2749
|
+
return "aborted";
|
|
2750
|
+
// A terminal event for this runId arrived; stop waiting.
|
|
2751
|
+
return "terminal";
|
|
2752
|
+
}
|
|
2753
|
+
finally {
|
|
2754
|
+
if (timeoutHandle !== undefined) {
|
|
2755
|
+
clearTimeout(timeoutHandle);
|
|
2756
|
+
}
|
|
2757
|
+
if (parkResolve !== null) {
|
|
2758
|
+
parkNotifyWaiters.delete(runId);
|
|
2759
|
+
}
|
|
2760
|
+
if (onAbort !== null) {
|
|
2761
|
+
abortSignal.removeEventListener("abort", onAbort);
|
|
2762
|
+
}
|
|
2763
|
+
if (typeof iter.return === "function") {
|
|
2764
|
+
await iter.return(undefined).catch(() => {
|
|
2765
|
+
/* swallowed: best-effort finalisation of the watcher iterator. */
|
|
2766
|
+
});
|
|
2767
|
+
}
|
|
2768
|
+
}
|
|
2769
|
+
}
|
|
1463
2770
|
/**
|
|
1464
2771
|
* The dispatch loop body. Runs until the cohort aborts; each
|
|
1465
2772
|
* iteration drains one inbox entry through the FIFO claim-check
|
|
@@ -1482,6 +2789,13 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1482
2789
|
return;
|
|
1483
2790
|
}
|
|
1484
2791
|
while (!cohortAbort.signal.aborted) {
|
|
2792
|
+
// Capture the wake BEFORE the dispatch iteration (capture-before-check,
|
|
2793
|
+
// same discipline as the park-generation latch). `wakeDispatch` resolves
|
|
2794
|
+
// the CURRENT promise and swaps in a fresh one, so a mail that enqueues
|
|
2795
|
+
// DURING dispatchOne resolves THIS captured promise; capturing it after
|
|
2796
|
+
// dispatchOne would await the fresh, unresolved promise and strand that
|
|
2797
|
+
// mail until some later wake.
|
|
2798
|
+
const wake = dispatchWake.promise;
|
|
1485
2799
|
let dispatched;
|
|
1486
2800
|
try {
|
|
1487
2801
|
dispatched = await dispatchOne(sender, cohortAbort, broadcaster);
|
|
@@ -1499,7 +2813,6 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1499
2813
|
continue;
|
|
1500
2814
|
if (cohortAbort.signal.aborted)
|
|
1501
2815
|
return;
|
|
1502
|
-
const wake = dispatchWake.promise;
|
|
1503
2816
|
const abortPromise = new Promise((resolve) => {
|
|
1504
2817
|
if (cohortAbort.signal.aborted) {
|
|
1505
2818
|
resolve();
|
|
@@ -1517,7 +2830,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1517
2830
|
substrate: bindings.repoStore,
|
|
1518
2831
|
repoId: bindings.workflowRunRepoId,
|
|
1519
2832
|
ref: bindings.workflowRunRef,
|
|
1520
|
-
|
|
2833
|
+
anchorRunId: bindings.anchorRunId,
|
|
1521
2834
|
runId: opts.runId,
|
|
1522
2835
|
origin: opts.origin,
|
|
1523
2836
|
reason: opts.reason,
|
|
@@ -1530,7 +2843,9 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1530
2843
|
await shutdownInternal({ reason: "shutdown requested" });
|
|
1531
2844
|
}
|
|
1532
2845
|
async function shutdownInternal(opts) {
|
|
1533
|
-
if (state.phase === "idle" ||
|
|
2846
|
+
if (state.phase === "idle" ||
|
|
2847
|
+
state.phase === "stopped" ||
|
|
2848
|
+
state.phase === "crash-looping")
|
|
1534
2849
|
return;
|
|
1535
2850
|
const prior = state;
|
|
1536
2851
|
state = { phase: "stopping" };
|
|
@@ -1566,11 +2881,15 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1566
2881
|
}
|
|
1567
2882
|
}
|
|
1568
2883
|
drainAccumulators.clear();
|
|
2884
|
+
cohortRunIds.clear();
|
|
2885
|
+
runInputChannels.clear();
|
|
2886
|
+
parkNotifyWaiters.clear();
|
|
2887
|
+
parkGenerations.clear();
|
|
1569
2888
|
if (prior.phase === "starting" ||
|
|
1570
2889
|
prior.phase === "running" ||
|
|
1571
2890
|
prior.phase === "recycling") {
|
|
1572
2891
|
prior.terminalCohortAbort.abort();
|
|
1573
|
-
// Reject every pending merge round-trip and
|
|
2892
|
+
// Reject every pending merge round-trip and park-notify waiter
|
|
1574
2893
|
// so handler closures awaiting them (including fire-and-forget
|
|
1575
2894
|
// `handleSubstrateWriteRequest` instances) cannot outlive the
|
|
1576
2895
|
// dying cohort. Without this, the `await new Promise` inside
|
|
@@ -1618,6 +2937,21 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1618
2937
|
path only waits for the substrate write to settle. */
|
|
1619
2938
|
});
|
|
1620
2939
|
}
|
|
2940
|
+
if ((prior.phase === "starting" ||
|
|
2941
|
+
prior.phase === "running" ||
|
|
2942
|
+
prior.phase === "recycling") &&
|
|
2943
|
+
prior.sweepDone !== null) {
|
|
2944
|
+
// Await the spawn-time compaction sweep before teardown so an
|
|
2945
|
+
// in-flight fold's substrate commit does not outlive the supervisor
|
|
2946
|
+
// and interleave with the next incarnation's boot. Teardown latency
|
|
2947
|
+
// is bounded by the recovery backlog (see the `sweepDone` field
|
|
2948
|
+
// docstring); a normal boot has zero or one pending fold.
|
|
2949
|
+
await prior.sweepDone.catch(() => {
|
|
2950
|
+
/* swallowed: the sweep's own catch already surfaces failures to
|
|
2951
|
+
the supervisor's warn channel; the shutdown path only waits for
|
|
2952
|
+
the in-flight fold's substrate commit to settle. */
|
|
2953
|
+
});
|
|
2954
|
+
}
|
|
1621
2955
|
if (recyclePolicy !== null) {
|
|
1622
2956
|
try {
|
|
1623
2957
|
recyclePolicy.stop();
|
|
@@ -1628,6 +2962,24 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1628
2962
|
}
|
|
1629
2963
|
recyclePolicy = null;
|
|
1630
2964
|
}
|
|
2965
|
+
// Disarm the crash-loop stable-run reset timer so it cannot fire
|
|
2966
|
+
// against a torn-down supervisor. Also drop any pending child exit
|
|
2967
|
+
// that `maybeHandleChildExit` recorded but declined to act on -- it
|
|
2968
|
+
// leaves one pending when a respawn was in flight, or when the phase
|
|
2969
|
+
// had already left `running`. (The crash-loop latch path does NOT
|
|
2970
|
+
// leave one pending: `maybeHandleChildExit` nulls `pendingChildExit`
|
|
2971
|
+
// before invoking the handler that latches.) A shutdown-initiated
|
|
2972
|
+
// kill of a live child in the `finally` below resolves its
|
|
2973
|
+
// `handle.exited`, so the watcher may re-record a pending exit AFTER
|
|
2974
|
+
// this clear -- harmless: the phase is terminal, so
|
|
2975
|
+
// `maybeHandleChildExit` no-ops on it, and `spawn()` requires `idle`,
|
|
2976
|
+
// so the stale slot is never re-examined.
|
|
2977
|
+
clearStableRunResetTimer();
|
|
2978
|
+
// Cancel every armed respawn backoff wait. The phase was flipped to
|
|
2979
|
+
// `stopping` synchronously above, so each parked respawn coroutine
|
|
2980
|
+
// this unblocks re-checks the phase and bails without respawning.
|
|
2981
|
+
cancelRespawnBackoffWaits();
|
|
2982
|
+
pendingChildExit = null;
|
|
1631
2983
|
spawnContext = null;
|
|
1632
2984
|
if (prior.phase === "starting" ||
|
|
1633
2985
|
prior.phase === "running" ||
|
|
@@ -1675,7 +3027,27 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1675
3027
|
/* swallowed for the same reason as above. */
|
|
1676
3028
|
});
|
|
1677
3029
|
}
|
|
1678
|
-
state = { phase: "stopped" };
|
|
3030
|
+
state = { phase: opts.terminalPhase ?? "stopped" };
|
|
3031
|
+
}
|
|
3032
|
+
// Surface a self-termination to the host after the terminal transition is
|
|
3033
|
+
// committed. The already-terminal early-return at the top dedups the common
|
|
3034
|
+
// case, but it does NOT cover the `stopping` window, so two self-terminating
|
|
3035
|
+
// callers interleaving through teardown can each fire (e.g. an onChildCrash
|
|
3036
|
+
// during `recycling` plus the recycle-failure catch). The sink is therefore
|
|
3037
|
+
// idempotent-required, not exactly-once; the reclaim it drives absorbs a
|
|
3038
|
+
// repeat by design. Wrapped so a throwing sink cannot re-escape here and
|
|
3039
|
+
// break the documented shutdown totality.
|
|
3040
|
+
if (opts.selfTerminated === true) {
|
|
3041
|
+
try {
|
|
3042
|
+
bindings.onSelfTerminate?.({
|
|
3043
|
+
phase: opts.terminalPhase ?? "stopped",
|
|
3044
|
+
reason: opts.reason,
|
|
3045
|
+
});
|
|
3046
|
+
}
|
|
3047
|
+
catch (cause) {
|
|
3048
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
3049
|
+
logger.warn `onSelfTerminate sink threw: ${message}`;
|
|
3050
|
+
}
|
|
1679
3051
|
}
|
|
1680
3052
|
logger.info `supervisor shutdown complete (${opts.reason})`;
|
|
1681
3053
|
}
|
|
@@ -1702,11 +3074,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1702
3074
|
*/
|
|
1703
3075
|
async function drainImpl(opts, ctx) {
|
|
1704
3076
|
// Drain is meaningful only when a workflow-process child is up;
|
|
1705
|
-
// calling it from `idle`/`stopping`/`stopped
|
|
1706
|
-
// higher-level host shutdown
|
|
1707
|
-
// unconditionally without sniffing the phase.
|
|
1708
|
-
// calls drain via `drainImpl({}, { fromRecycle: true })`
|
|
1709
|
-
// admits `recycling` because the drain step runs against a
|
|
3077
|
+
// calling it from any non-active phase (`idle`/`stopping`/`stopped`/
|
|
3078
|
+
// `crash-looping`) is a no-op so the higher-level host shutdown
|
|
3079
|
+
// sequence can call drain unconditionally without sniffing the phase.
|
|
3080
|
+
// The recycle path calls drain via `drainImpl({}, { fromRecycle: true })`
|
|
3081
|
+
// and admits `recycling` because the drain step runs against a
|
|
1710
3082
|
// still-live controlSender before the kill lands.
|
|
1711
3083
|
if (state.phase !== "running" &&
|
|
1712
3084
|
state.phase !== "starting" &&
|
|
@@ -1730,15 +3102,22 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1730
3102
|
// body's cancellation cascade tears the run down without the
|
|
1731
3103
|
// supervisor having to thread any per-run wiring beyond what the
|
|
1732
3104
|
// accumulator already encapsulates.
|
|
3105
|
+
//
|
|
3106
|
+
// Runs that are already parked do not need drain escalation; the
|
|
3107
|
+
// runtime parks the run and the supervisor simply stops delivering
|
|
3108
|
+
// new mail. The cohort abort on shutdown/recycle will eventually
|
|
3109
|
+
// tear the run down.
|
|
1733
3110
|
const cohortSource = perCohortTerminalSource(state.terminalCohortAbort, state.terminalBroadcaster);
|
|
1734
|
-
for (const runId of
|
|
3111
|
+
for (const runId of cohortRunIds) {
|
|
1735
3112
|
if (drainAccumulators.has(runId))
|
|
1736
3113
|
continue;
|
|
3114
|
+
if (runInputChannels.has(runId))
|
|
3115
|
+
continue;
|
|
1737
3116
|
const accumulator = accumulatorFactory({
|
|
1738
3117
|
substrate: bindings.repoStore,
|
|
1739
3118
|
repoId: bindings.workflowRunRepoId,
|
|
1740
3119
|
ref: bindings.workflowRunRef,
|
|
1741
|
-
|
|
3120
|
+
anchorRunId: bindings.anchorRunId,
|
|
1742
3121
|
runId,
|
|
1743
3122
|
signAsPrincipal: bindings.signAsPrincipal,
|
|
1744
3123
|
drainTimeoutMs,
|
|
@@ -1752,7 +3131,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1752
3131
|
}
|
|
1753
3132
|
}
|
|
1754
3133
|
async function recycle(opts) {
|
|
1755
|
-
if (
|
|
3134
|
+
if (respawnInProgress) {
|
|
1756
3135
|
throw new Error("supervisor: recycle already in progress");
|
|
1757
3136
|
}
|
|
1758
3137
|
if (state.phase !== "running") {
|
|
@@ -1761,10 +3140,43 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1761
3140
|
if (spawnContext === null) {
|
|
1762
3141
|
throw new Error("supervisor: recycle called without a spawn context; spawn() must complete first");
|
|
1763
3142
|
}
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
3143
|
+
// The contention read (`respawnInProgress`) stays here in the caller's
|
|
3144
|
+
// precondition zone rather than inside `runRespawn`: an operator double-
|
|
3145
|
+
// recycle is an error and must throw. `runRespawn` sets the latch
|
|
3146
|
+
// synchronously at entry, so this read and that set are never separated
|
|
3147
|
+
// by an await, and no second respawn can interleave between them.
|
|
3148
|
+
return runRespawn({
|
|
3149
|
+
origin: opts.origin ?? "operator",
|
|
3150
|
+
reason: opts.reason,
|
|
3151
|
+
prior: state,
|
|
3152
|
+
priorContext: spawnContext,
|
|
3153
|
+
drain: async (deadlineMs) => {
|
|
3154
|
+
// The recycle path's drain step shares the drain primitive but
|
|
3155
|
+
// bypasses the public surface's `recycling` silent-no-op so the
|
|
3156
|
+
// still-live controlSender (this step runs BEFORE abortPriorCohort
|
|
3157
|
+
// + kill) receives the frame. The public `drain()` silently no-ops
|
|
3158
|
+
// on `recycling` for external callers because the kill/respawn gap
|
|
3159
|
+
// can leave the controlSender dying.
|
|
3160
|
+
await drainImpl({ deadlineMs }, { fromRecycle: true });
|
|
3161
|
+
},
|
|
3162
|
+
});
|
|
3163
|
+
}
|
|
3164
|
+
/**
|
|
3165
|
+
* Shared kill/replay/respawn/install driver. The caller has already
|
|
3166
|
+
* verified the supervisor is `running` with a live spawn context and
|
|
3167
|
+
* snapshotted both as `prior`/`priorContext`; this function transitions
|
|
3168
|
+
* to `recycling`, runs the six-step `triggerRecycle` sequence with the
|
|
3169
|
+
* caller-supplied `drain` step, and swaps in the new cohort via the
|
|
3170
|
+
* inline `installNewChild` callback. The operator/policy/self recycle
|
|
3171
|
+
* path is the sole caller today; the crash-respawn path will call it with
|
|
3172
|
+
* a no-op drain (its child is already dead).
|
|
3173
|
+
*/
|
|
3174
|
+
async function runRespawn(args) {
|
|
3175
|
+
// Set synchronously at entry (before any await) so the caller's
|
|
3176
|
+
// contention read and this set cannot be separated by an event-loop
|
|
3177
|
+
// turn; two respawns can never interleave.
|
|
3178
|
+
respawnInProgress = true;
|
|
3179
|
+
const { origin, reason, prior, priorContext, drain } = args;
|
|
1768
3180
|
// The cohort abort no longer fires up-front. triggerRecycle drives
|
|
1769
3181
|
// the drain and replay steps against a LIVE cohort first, then
|
|
1770
3182
|
// invokes `abortPriorCohort` (the callback below) between replay
|
|
@@ -1793,6 +3205,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1793
3205
|
terminalBroadcaster: prior.terminalBroadcaster,
|
|
1794
3206
|
dispatchLoop: null,
|
|
1795
3207
|
replayDone: null,
|
|
3208
|
+
sweepDone: prior.sweepDone,
|
|
1796
3209
|
};
|
|
1797
3210
|
let attempt;
|
|
1798
3211
|
try {
|
|
@@ -1808,16 +3221,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1808
3221
|
channelId: prior.channelId,
|
|
1809
3222
|
eventPump: prior.eventPump,
|
|
1810
3223
|
},
|
|
1811
|
-
drain
|
|
1812
|
-
// The recycle path's drain step shares the drain
|
|
1813
|
-
// primitive but bypasses the public surface's `recycling`
|
|
1814
|
-
// silent-no-op so the still-live controlSender (this
|
|
1815
|
-
// step runs BEFORE abortPriorCohort + kill) receives the
|
|
1816
|
-
// frame. The public `drain()` silently no-ops on
|
|
1817
|
-
// `recycling` for external callers because the
|
|
1818
|
-
// kill/respawn gap can leave the controlSender dying.
|
|
1819
|
-
await drainImpl({ deadlineMs }, { fromRecycle: true });
|
|
1820
|
-
},
|
|
3224
|
+
drain,
|
|
1821
3225
|
replayProcessingToInbox: async () => {
|
|
1822
3226
|
await inboxPrimitives.replayProcessingToInbox(bindings.repoStore, inboxWritePrincipal, bindings.workflowRunRepoId, bindings.deploymentMailAddress);
|
|
1823
3227
|
},
|
|
@@ -1828,6 +3232,13 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1828
3232
|
prior.terminalCohortAbort.abort();
|
|
1829
3233
|
wakeDispatch();
|
|
1830
3234
|
},
|
|
3235
|
+
// Kept inline rather than extracted: this cohort-swap closes over
|
|
3236
|
+
// the supervisor's full mutable cohort state (drainAccumulators,
|
|
3237
|
+
// cohortRunIds, runInputChannels, parkNotifyWaiters, parkGenerations,
|
|
3238
|
+
// rejectCohortAwaiters, spawnContext, plus `prior`/`priorContext`).
|
|
3239
|
+
// A standalone helper would take all of it as parameters for zero
|
|
3240
|
+
// reuse -- the crash-respawn path reaches this callback transitively
|
|
3241
|
+
// through `runRespawn`, so it needs no separate extraction.
|
|
1831
3242
|
installNewChild: ({ wiring, credentialsSnapshot, controlIncoming, }) => {
|
|
1832
3243
|
// Phase guard: a `shutdown()` that landed during the
|
|
1833
3244
|
// kill/respawn gap (between `subprocessSpawner` and this
|
|
@@ -1871,7 +3282,11 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1871
3282
|
accumulator.stop();
|
|
1872
3283
|
}
|
|
1873
3284
|
drainAccumulators.clear();
|
|
1874
|
-
|
|
3285
|
+
cohortRunIds.clear();
|
|
3286
|
+
runInputChannels.clear();
|
|
3287
|
+
parkNotifyWaiters.clear();
|
|
3288
|
+
parkGenerations.clear();
|
|
3289
|
+
// Reject every pending merge round-trip and park-notify
|
|
1875
3290
|
// waiter registered against the dying cohort so handler
|
|
1876
3291
|
// closures cannot survive the kill/respawn gap. The new
|
|
1877
3292
|
// child will re-issue substrate writes through fresh
|
|
@@ -1906,7 +3321,14 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1906
3321
|
terminalBroadcaster: newBroadcaster,
|
|
1907
3322
|
dispatchLoop: newDispatchLoop,
|
|
1908
3323
|
replayDone: null,
|
|
3324
|
+
sweepDone: prior.sweepDone,
|
|
1909
3325
|
};
|
|
3326
|
+
// Bump the generation and arm the exit-watcher for the
|
|
3327
|
+
// respawned child atomically with this running transition, so
|
|
3328
|
+
// the predecessor's watcher (already stale by generation) never
|
|
3329
|
+
// drives a spurious respawn and a crash of THIS child is
|
|
3330
|
+
// classified against the new generation.
|
|
3331
|
+
armChildForRunning(wiring.handle);
|
|
1910
3332
|
// Cache fresh spawn context with the updated spawnedAt
|
|
1911
3333
|
// so the policy timer's uptime check resets on recycle.
|
|
1912
3334
|
const now = bindings.recyclePolicyNow ?? defaultNow;
|
|
@@ -1934,6 +3356,16 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1934
3356
|
// entries the previous cohort's replayProcessingToInbox
|
|
1935
3357
|
// just moved back.
|
|
1936
3358
|
wakeDispatch();
|
|
3359
|
+
// Trigger A on the recycle seam: the respawned child re-parks any
|
|
3360
|
+
// surviving parked run without re-emitting, and a recycle leaves
|
|
3361
|
+
// the hub link untouched so the reconnect trigger never fires --
|
|
3362
|
+
// so re-drive the re-registration here too. Same fire-and-forget
|
|
3363
|
+
// contract as the spawn seam; the fresh cohort's controlSender is
|
|
3364
|
+
// in `state` now, and its pump (armed above) routes the response.
|
|
3365
|
+
void reEmitParkedCorrelations().catch((cause) => {
|
|
3366
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
3367
|
+
logger.warn `re-emit of parked correlations on re-establishment failed: ${message}`;
|
|
3368
|
+
});
|
|
1937
3369
|
},
|
|
1938
3370
|
onCrash: onChildCrash,
|
|
1939
3371
|
// Edge-resolved once at the supervisor factory; recycle bounds
|
|
@@ -1945,7 +3377,7 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1945
3377
|
...(bindings.recyclePolicyClearTimer !== undefined
|
|
1946
3378
|
? { clearTimer: bindings.recyclePolicyClearTimer }
|
|
1947
3379
|
: {}),
|
|
1948
|
-
}, { origin, reason
|
|
3380
|
+
}, { origin, reason });
|
|
1949
3381
|
// After the recycle, await the previous cohort's dispatch
|
|
1950
3382
|
// loop so a teardown coroutine cannot survive past the
|
|
1951
3383
|
// recycle's return point.
|
|
@@ -1963,11 +3395,23 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1963
3395
|
// level shutdown. Tear the prior cohort down through the same
|
|
1964
3396
|
// path a real shutdown uses so the supervisor reaches a clean
|
|
1965
3397
|
// `stopped` state, then re-throw so the operator sees the
|
|
1966
|
-
//
|
|
3398
|
+
// failure and can redeploy.
|
|
3399
|
+
//
|
|
3400
|
+
// This teardown-to-`stopped` is shared by both callers, and that is
|
|
3401
|
+
// deliberate. A crash-origin respawn whose spawn/wire/handshake fails
|
|
3402
|
+
// is a broken deploy (a bad binary, unreadable credentials), NOT a
|
|
3403
|
+
// flapping child, so it does NOT feed the crash-loop exit counter and
|
|
3404
|
+
// does NOT reach `crash-looping`; conflating a mechanism failure with
|
|
3405
|
+
// a flap would muddy what that counter means. It reaches `stopped`,
|
|
3406
|
+
// the same terminal a failed operator recycle reaches. (Consequence:
|
|
3407
|
+
// this path leaves no `RunFailed` tombstone, unlike the exit-count
|
|
3408
|
+
// latch; the crash-respawn happy path -- a clean child death whose
|
|
3409
|
+
// respawn succeeds -- is what the crash-loop guard bounds.)
|
|
1967
3410
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1968
3411
|
logger.error `recycle failed; tearing supervisor down: ${message}`;
|
|
1969
3412
|
await shutdownInternal({
|
|
1970
3413
|
reason: `recycle failed: ${message}`,
|
|
3414
|
+
selfTerminated: true,
|
|
1971
3415
|
}).catch((shutdownCause) => {
|
|
1972
3416
|
const inner = shutdownCause instanceof Error
|
|
1973
3417
|
? shutdownCause.message
|
|
@@ -1977,7 +3421,12 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1977
3421
|
throw cause;
|
|
1978
3422
|
}
|
|
1979
3423
|
finally {
|
|
1980
|
-
|
|
3424
|
+
respawnInProgress = false;
|
|
3425
|
+
// Drain any child exit observed during the respawn. A crash of the
|
|
3426
|
+
// freshly-installed child that raced this respawn's completion was
|
|
3427
|
+
// deferred (respawnInProgress was set); handle it now that the latch
|
|
3428
|
+
// is clear. A pending exit for a superseded generation drops as stale.
|
|
3429
|
+
maybeHandleChildExit();
|
|
1981
3430
|
}
|
|
1982
3431
|
return attempt;
|
|
1983
3432
|
}
|
|
@@ -1998,6 +3447,27 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
1998
3447
|
if (state.phase !== "running" && state.phase !== "starting") {
|
|
1999
3448
|
throw new Error(`supervisor: deliverSignal called in phase ${state.phase}; expected starting/running`);
|
|
2000
3449
|
}
|
|
3450
|
+
// Refresh the run's grant floor on the SAME control channel immediately
|
|
3451
|
+
// before the signal, so a standing ("always") approval resolved for a
|
|
3452
|
+
// parked run lowers the floor for the resumed run's later calls. Ordering
|
|
3453
|
+
// is structural: both frames ride this single seq-ordered FIFO, so the
|
|
3454
|
+
// `grants-updated` is observed by the child ahead of the `signal.deliver`
|
|
3455
|
+
// -- no dependence on hub-side dispatch timing. Best-effort by design; a
|
|
3456
|
+
// failed refresh is non-fatal (the durable file still governs the next
|
|
3457
|
+
// barrier), and it only re-reads that file, so a signal with no standing
|
|
3458
|
+
// approval just re-pushes the unchanged floor.
|
|
3459
|
+
await deliverGrants(opts.runId);
|
|
3460
|
+
// `deliverGrants` awaits a substrate read, yielding the event loop. A
|
|
3461
|
+
// crash/recycle can land in that window and swap `state` (its
|
|
3462
|
+
// `controlSender` then points at the dying child). Re-assert the phase the
|
|
3463
|
+
// pre-await guard checked, so the signal is never written into a recycling
|
|
3464
|
+
// child's closing pipe; the caller retries once the recycle completes. The
|
|
3465
|
+
// phase is read through the full union type because the pre-await guard
|
|
3466
|
+
// control-flow-narrowed `state`, which the yield may have invalidated.
|
|
3467
|
+
const phaseAfterRefresh = state.phase;
|
|
3468
|
+
if (phaseAfterRefresh !== "running" && phaseAfterRefresh !== "starting") {
|
|
3469
|
+
throw new Error(`supervisor: deliverSignal raced a recycle in phase ${phaseAfterRefresh}; expected starting/running`);
|
|
3470
|
+
}
|
|
2001
3471
|
await state.controlSender.send({
|
|
2002
3472
|
type: "signal.deliver",
|
|
2003
3473
|
data: {
|
|
@@ -2026,6 +3496,85 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
2026
3496
|
},
|
|
2027
3497
|
});
|
|
2028
3498
|
}
|
|
3499
|
+
async function deliverCredentials(opts) {
|
|
3500
|
+
// Compute the next mirror first. It must advance regardless of phase so the
|
|
3501
|
+
// NEXT spawn/barrier re-asserts this update: a credential revoked while the
|
|
3502
|
+
// supervisor holds no live child (a crash-loop retry, a recycle transient)
|
|
3503
|
+
// must still be gone when the child (re)starts, not resurrected from the
|
|
3504
|
+
// frozen deploy delivery. This is what makes an offline-revoke reconcile
|
|
3505
|
+
// durable even when it arrives at a supervisor without a running child.
|
|
3506
|
+
const next = mergeCredentialDelivery(currentCredentialDelivery, opts.delivery, opts.revoke);
|
|
3507
|
+
// Send to the live child ONLY in starting/running. Outside those the control
|
|
3508
|
+
// sender points at a dying/absent child, so a frame would buffer behind a
|
|
3509
|
+
// SIGTERM or write into a closed pipe; the mirror advance below is the sole
|
|
3510
|
+
// effect, and the child that eventually spawns is seeded from it. The
|
|
3511
|
+
// supervisor is the single producer of `credentials-updated` frames.
|
|
3512
|
+
if (state.phase === "running" || state.phase === "starting") {
|
|
3513
|
+
await state.controlSender.send({
|
|
3514
|
+
type: "credentials-updated",
|
|
3515
|
+
data: {
|
|
3516
|
+
delivery: opts.delivery,
|
|
3517
|
+
...(opts.revoke !== undefined ? { revoke: opts.revoke } : {}),
|
|
3518
|
+
},
|
|
3519
|
+
});
|
|
3520
|
+
}
|
|
3521
|
+
// Advance the mirror. A throw from `send` above skips this, leaving the
|
|
3522
|
+
// mirror matching the child that never received the frame. A whole-object
|
|
3523
|
+
// swap via the same merge the child applies, so a concurrent reader sees a
|
|
3524
|
+
// coherent object.
|
|
3525
|
+
currentCredentialDelivery = next;
|
|
3526
|
+
}
|
|
3527
|
+
/**
|
|
3528
|
+
* Refresh a live run's grant floor mid-run: re-read this run's durable
|
|
3529
|
+
* `runs/<runId>/grants.json` (via `onRunStart`, the same read the pre-trigger
|
|
3530
|
+
* barrier uses) and push it to the child as a `grants-updated` frame. The
|
|
3531
|
+
* enforcement path for a standing (`scope: "always"`) approval, which lowers
|
|
3532
|
+
* a tool's `ask` to `allow` in that file: the barrier only runs before a
|
|
3533
|
+
* trigger/signal dispatch, so a run already executing (or being resumed
|
|
3534
|
+
* without a fresh barrier) needs this to observe the change now.
|
|
3535
|
+
*
|
|
3536
|
+
* Distinct from `pushRunGrants` on two axes, both deliberate:
|
|
3537
|
+
* - It NEVER synthesizes a `RunFailed`. A refresh for a run whose child is
|
|
3538
|
+
* not live is normal (the durable file already carries the change and the
|
|
3539
|
+
* next barrier or respawn re-reads it), so it no-ops (`skipped`) rather
|
|
3540
|
+
* than failing the run, and a send failure to a live child is logged
|
|
3541
|
+
* loudly but stays non-fatal (the file still wins at the next barrier).
|
|
3542
|
+
* - It only ever pushes the durable file's contents through `onRunStart`; it
|
|
3543
|
+
* accepts no caller-supplied grants, so it can only tighten or refresh a
|
|
3544
|
+
* floor, never inject one a deploy did not approve.
|
|
3545
|
+
*/
|
|
3546
|
+
async function deliverGrants(runId) {
|
|
3547
|
+
if (bindings.onRunStart === undefined)
|
|
3548
|
+
return "skipped";
|
|
3549
|
+
if (state.phase !== "running" && state.phase !== "starting") {
|
|
3550
|
+
return "skipped";
|
|
3551
|
+
}
|
|
3552
|
+
try {
|
|
3553
|
+
const snapshot = await bindings.onRunStart({
|
|
3554
|
+
runId,
|
|
3555
|
+
anchorRunId: bindings.anchorRunId,
|
|
3556
|
+
});
|
|
3557
|
+
await state.controlSender.send({
|
|
3558
|
+
type: "grants-updated",
|
|
3559
|
+
data: {
|
|
3560
|
+
snapshot: {
|
|
3561
|
+
steps: snapshot.steps.map((s) => ({
|
|
3562
|
+
stepId: s.stepId,
|
|
3563
|
+
address: s.address,
|
|
3564
|
+
grants: [...s.grants],
|
|
3565
|
+
contentHash: s.contentHash,
|
|
3566
|
+
})),
|
|
3567
|
+
},
|
|
3568
|
+
},
|
|
3569
|
+
});
|
|
3570
|
+
return "pushed";
|
|
3571
|
+
}
|
|
3572
|
+
catch (cause) {
|
|
3573
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
3574
|
+
logger.error `deliverGrants refresh failed for run ${runId}; the durable grants file still governs the next barrier/respawn: ${message}`;
|
|
3575
|
+
return "skipped";
|
|
3576
|
+
}
|
|
3577
|
+
}
|
|
2029
3578
|
function getCredentialsSnapshot() {
|
|
2030
3579
|
if (state.phase === "starting" || state.phase === "running") {
|
|
2031
3580
|
return state.credentialsSnapshot;
|
|
@@ -2040,6 +3589,9 @@ export function createWorkflowSupervisor(bindings) {
|
|
|
2040
3589
|
recycle,
|
|
2041
3590
|
deliverSignal,
|
|
2042
3591
|
deliverSources,
|
|
3592
|
+
deliverCredentials,
|
|
3593
|
+
deliverGrants,
|
|
3594
|
+
reEmitParkedCorrelations,
|
|
2043
3595
|
getCredentialsSnapshot,
|
|
2044
3596
|
};
|
|
2045
3597
|
}
|
|
@@ -2097,71 +3649,6 @@ async function pumpEvents(iter, onInferenceEvent) {
|
|
|
2097
3649
|
function defaultInProcessMailAuditRef(messageId, _rawMessage) {
|
|
2098
3650
|
return { store: "in-process", path: messageId };
|
|
2099
3651
|
}
|
|
2100
|
-
/**
|
|
2101
|
-
* Derive a stable message identifier from the raw bytes the bus
|
|
2102
|
-
* delivered. The RFC 2822 `Message-ID` header (if present) is the
|
|
2103
|
-
* canonical identifier the audit log surfaces as
|
|
2104
|
-
* `RunStarted.consumedMessageId`; downstream consumers join inbound
|
|
2105
|
-
* mail to workflow-run events on this value, so the header parse must
|
|
2106
|
-
* win when the sender emitted one. A message that lacks a
|
|
2107
|
-
* `Message-ID` header falls back to a sha256 of the raw bytes so
|
|
2108
|
-
* runs originating from non-RFC 2822 transports still receive a
|
|
2109
|
-
* deterministic identifier.
|
|
2110
|
-
*
|
|
2111
|
-
* The parser walks the message until the headers/body separator
|
|
2112
|
-
* (`CRLF CRLF` per RFC 2822 §2.1, with the lone-`LF` variant tolerated
|
|
2113
|
-
* to match common in-memory senders). Header-field unfolding follows
|
|
2114
|
-
* RFC 2822 §2.2.3: a continuation line begins with whitespace and
|
|
2115
|
-
* appends to the prior line. Header-name comparison is
|
|
2116
|
-
* case-insensitive per RFC 2822 §1.2.2.
|
|
2117
|
-
*/
|
|
2118
|
-
async function deriveMessageId(rawMessage) {
|
|
2119
|
-
const messageIdFromHeader = parseMessageIdHeader(rawMessage);
|
|
2120
|
-
if (messageIdFromHeader !== null) {
|
|
2121
|
-
return messageIdFromHeader;
|
|
2122
|
-
}
|
|
2123
|
-
const digest = await crypto.subtle.digest("SHA-256",
|
|
2124
|
-
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- ArrayBuffer-backed at the call site; Web Crypto's BufferSource type rejects Uint8Array<ArrayBufferLike> under TS 5.9 (microsoft/TypeScript#62240)
|
|
2125
|
-
rawMessage);
|
|
2126
|
-
return hexEncode(new Uint8Array(digest));
|
|
2127
|
-
}
|
|
2128
|
-
function parseMessageIdHeader(rawMessage) {
|
|
2129
|
-
const text = new TextDecoder("utf-8", { fatal: false }).decode(rawMessage);
|
|
2130
|
-
// Headers end at the first blank line. RFC 2822 mandates `CRLF CRLF`
|
|
2131
|
-
// but tolerate `LF LF` for callers that normalize line endings.
|
|
2132
|
-
let headerSection = text;
|
|
2133
|
-
const crlfBoundary = text.indexOf("\r\n\r\n");
|
|
2134
|
-
const lfBoundary = text.indexOf("\n\n");
|
|
2135
|
-
if (crlfBoundary >= 0 && (lfBoundary < 0 || crlfBoundary < lfBoundary)) {
|
|
2136
|
-
headerSection = text.slice(0, crlfBoundary);
|
|
2137
|
-
}
|
|
2138
|
-
else if (lfBoundary >= 0) {
|
|
2139
|
-
headerSection = text.slice(0, lfBoundary);
|
|
2140
|
-
}
|
|
2141
|
-
// Unfold continuation lines (a line starting with WSP belongs to
|
|
2142
|
-
// the prior header field).
|
|
2143
|
-
const lines = headerSection.split(/\r?\n/);
|
|
2144
|
-
const unfolded = [];
|
|
2145
|
-
for (const line of lines) {
|
|
2146
|
-
if (line.length > 0 && (line[0] === " " || line[0] === "\t")) {
|
|
2147
|
-
if (unfolded.length === 0)
|
|
2148
|
-
continue;
|
|
2149
|
-
unfolded[unfolded.length - 1] += " " + line.trim();
|
|
2150
|
-
continue;
|
|
2151
|
-
}
|
|
2152
|
-
unfolded.push(line);
|
|
2153
|
-
}
|
|
2154
|
-
for (const line of unfolded) {
|
|
2155
|
-
const colon = line.indexOf(":");
|
|
2156
|
-
if (colon < 0)
|
|
2157
|
-
continue;
|
|
2158
|
-
const name = line.slice(0, colon).trim().toLowerCase();
|
|
2159
|
-
if (name !== "message-id")
|
|
2160
|
-
continue;
|
|
2161
|
-
return line.slice(colon + 1).trim();
|
|
2162
|
-
}
|
|
2163
|
-
return null;
|
|
2164
|
-
}
|
|
2165
3652
|
/**
|
|
2166
3653
|
* Project the wire shape of a `terminal.event` upstream control frame
|
|
2167
3654
|
* into the workflow-vocabulary `TerminalRunEvent` discriminated union
|
|
@@ -2220,6 +3707,8 @@ function outboundMessageFromPayload(payload) {
|
|
|
2220
3707
|
message.summary = payload.summary;
|
|
2221
3708
|
if (payload.inReplyTo !== undefined)
|
|
2222
3709
|
message.inReplyTo = payload.inReplyTo;
|
|
3710
|
+
if (payload.references !== undefined)
|
|
3711
|
+
message.references = payload.references;
|
|
2223
3712
|
if (payload.correlationId !== undefined) {
|
|
2224
3713
|
message.correlationId = payload.correlationId;
|
|
2225
3714
|
}
|