@intx/workflow-host 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +56 -10
  2. package/dist/adapters/repo-store.d.ts +22 -1
  3. package/dist/adapters/repo-store.js +53 -53
  4. package/dist/adapters/spawn-child.d.ts +71 -42
  5. package/dist/adapters/spawn-child.js +83 -77
  6. package/dist/adapters/step-invoker.js +84 -7
  7. package/dist/child/env-bootstrap.d.ts +20 -6
  8. package/dist/child/env-bootstrap.js +9 -1
  9. package/dist/child/index.d.ts +2 -1
  10. package/dist/child/parked-correlations.d.ts +42 -0
  11. package/dist/child/parked-correlations.js +80 -0
  12. package/dist/child/proxy-repo-store.d.ts +3 -2
  13. package/dist/child/proxy-repo-store.js +2 -0
  14. package/dist/child/run-child.d.ts +107 -13
  15. package/dist/child/run-child.js +290 -108
  16. package/dist/child/self-discovery.d.ts +10 -0
  17. package/dist/child/self-discovery.js +25 -1
  18. package/dist/child/verified-definition-loader.d.ts +33 -0
  19. package/dist/child/verified-definition-loader.js +43 -0
  20. package/dist/conversation-text.d.ts +23 -0
  21. package/dist/conversation-text.js +56 -0
  22. package/dist/index.d.ts +4 -3
  23. package/dist/index.js +3 -2
  24. package/dist/ipc/control-channel.d.ts +58 -0
  25. package/dist/ipc/control-channel.js +94 -1
  26. package/dist/ipc/event-channel.d.ts +32 -1
  27. package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
  28. package/dist/mail-bus/hub-transport-adapter.js +9 -5
  29. package/dist/seams/scheduler.d.ts +4 -6
  30. package/dist/seams/scheduler.js +74 -93
  31. package/dist/supervisor/cancel-signing.d.ts +2 -2
  32. package/dist/supervisor/cancel-signing.js +1 -1
  33. package/dist/supervisor/credentials.d.ts +11 -10
  34. package/dist/supervisor/credentials.js +7 -7
  35. package/dist/supervisor/dispatch-attribution.js +1 -1
  36. package/dist/supervisor/drain-timeout.d.ts +2 -2
  37. package/dist/supervisor/drain-timeout.js +1 -1
  38. package/dist/supervisor/index.d.ts +3 -3
  39. package/dist/supervisor/index.js +2 -2
  40. package/dist/supervisor/recycle.d.ts +5 -2
  41. package/dist/supervisor/recycle.js +18 -7
  42. package/dist/supervisor/run-event-compaction.d.ts +5 -5
  43. package/dist/supervisor/run-event-compaction.js +5 -5
  44. package/dist/supervisor/spawn-env.d.ts +2 -2
  45. package/dist/supervisor/spawn-env.js +1 -1
  46. package/dist/supervisor/supervisor.d.ts +82 -25
  47. package/dist/supervisor/supervisor.js +1313 -410
  48. package/dist/supervisor/terminal-commit.d.ts +36 -0
  49. package/dist/supervisor/terminal-commit.js +134 -0
  50. package/dist/supervisor/types.d.ts +150 -23
  51. package/dist/workflow-definition-loader.d.ts +131 -0
  52. package/dist/workflow-definition-loader.js +316 -0
  53. package/package.json +12 -11
@@ -0,0 +1,36 @@
1
+ import type { RepoId, RepoStore as SubstrateRepoStore } from "@intx/hub-sessions/substrate";
2
+ export type CommitRunFailedOpts = {
3
+ /** Substrate handle the supervisor writes through. */
4
+ substrate: SubstrateRepoStore;
5
+ /** Workflow-run repo for this deployment. */
6
+ repoId: RepoId;
7
+ /** Events ref the workflow-run repo writes to. */
8
+ ref: string;
9
+ /** Anchor run id used to construct the supervisor principal. */
10
+ anchorRunId: string;
11
+ /** Run id whose event log receives the RunFailed entry. */
12
+ runId: string;
13
+ /** ISO-8601 commit timestamp the event carries. */
14
+ at: string;
15
+ /** Operator-facing failure reason on `RunFailed.error.message`. */
16
+ message: string;
17
+ };
18
+ export type CommitRunFailedResult = {
19
+ /** Substrate-assigned commit SHA the write produced. */
20
+ commitSha: string;
21
+ /**
22
+ * True when a `RunFailed` was appended; false when the run was already
23
+ * terminal and the write was a no-op (terminal-lock respected).
24
+ */
25
+ appended: boolean;
26
+ };
27
+ /**
28
+ * Append a supervisor-authored `RunFailed` to a run's event log, unless the
29
+ * run is already terminal. The next seq is computed inside the substrate
30
+ * merge (atomic against any concurrent writer under the per-repo lock): the
31
+ * first event on an empty tree lands at seq 1 (the runtime's convention),
32
+ * otherwise at `maxSeq + 1` so the log stays seq-contiguous. If the run's
33
+ * highest-seq event is already terminal, the write is a no-op so push
34
+ * validation's terminal-lock is never tripped.
35
+ */
36
+ export declare function commitRunFailed(opts: CommitRunFailedOpts): Promise<CommitRunFailedResult>;
@@ -0,0 +1,134 @@
1
+ // Supervisor-authored terminal-event commit for the crash-loop guard.
2
+ //
3
+ // When the crash-loop guard latches, the workflow-process child is dead
4
+ // and cannot commit its own terminal event, yet the deployment's run must
5
+ // reach a terminal state so its external `workflow_run.status` flips to
6
+ // `failed` -- the sole durable, queryable signal of a crash-loop. The
7
+ // supervisor's `crash-looping` phase is in-memory and per-process; no
8
+ // external reader observes it. The supervisor, as the sole writer of the
9
+ // workflow-run repo, authors a `RunFailed` for the deployment's stable
10
+ // run so the crash-loop leaves a durable tombstone.
11
+ //
12
+ // Unlike `commitCancelRequested`, a terminal workflow event carries no
13
+ // signature: only `CancelRequested` is signed (for its origin<->principal
14
+ // cross-check at push validation), so this writer is unsigned. It writes
15
+ // under the `supervisor` principal, which the workflow-run kind handler
16
+ // authorizes for the deployment's own event log (`repoId.id ===
17
+ // anchorRunId`); terminal events have no per-type authorship check.
18
+ import { type } from "arktype";
19
+ import { getLogger } from "@intx/log";
20
+ import { workflowEventToOnDisk } from "../adapters/repo-store.js";
21
+ const logger = getLogger(["workflow-host", "supervisor", "terminal-commit"]);
22
+ /** Path layout inside the workflow-run repo: `runs/<runId>/events/<seq>.json`. */
23
+ const RUNS_PREFIX = "runs";
24
+ const EVENTS_DIR = "events";
25
+ const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
26
+ /**
27
+ * Terminal run-event kinds, mirroring the runtime's terminal vocabulary
28
+ * (`RunCompleted`/`RunFailed`/`RunCancelled`). Inlined because the workflow
29
+ * package exports only the phase-level `isTerminalRunPhase`, not an
30
+ * event-kind set; the child runtime (`run-child`) and the substrate adapter
31
+ * (`repo-store`) inline the same three kinds. Reducing the log to a phase to
32
+ * reuse `isTerminalRunPhase` would be strictly heavier for a last-event
33
+ * type check.
34
+ */
35
+ const TERMINAL_EVENT_KINDS = new Set([
36
+ "RunCompleted",
37
+ "RunFailed",
38
+ "RunCancelled",
39
+ ]);
40
+ const SUPERVISOR_PRINCIPAL_KIND = "supervisor";
41
+ /**
42
+ * On-disk event envelope, validated at the substrate read boundary. Only
43
+ * `seq` and `type` are load-bearing here (max-seq computation and the
44
+ * terminal-lock check); the rest of the event body is ignored.
45
+ */
46
+ const OnDiskEventEnvelope = type({
47
+ seq: "number >= 0",
48
+ type: "string",
49
+ "+": "ignore",
50
+ });
51
+ /**
52
+ * Append a supervisor-authored `RunFailed` to a run's event log, unless the
53
+ * run is already terminal. The next seq is computed inside the substrate
54
+ * merge (atomic against any concurrent writer under the per-repo lock): the
55
+ * first event on an empty tree lands at seq 1 (the runtime's convention),
56
+ * otherwise at `maxSeq + 1` so the log stays seq-contiguous. If the run's
57
+ * highest-seq event is already terminal, the write is a no-op so push
58
+ * validation's terminal-lock is never tripped.
59
+ */
60
+ export async function commitRunFailed(opts) {
61
+ const prefix = `${RUNS_PREFIX}/${opts.runId}/${EVENTS_DIR}/`;
62
+ const principal = {
63
+ kind: SUPERVISOR_PRINCIPAL_KIND,
64
+ anchorRunId: opts.anchorRunId,
65
+ };
66
+ const decoder = new TextDecoder();
67
+ let appended = false;
68
+ const { commitSha } = await opts.substrate.writeTreePreservingPrefix(principal, opts.repoId, opts.ref, {
69
+ preservePrefix: prefix,
70
+ merge: async (existing) => {
71
+ let maxSeq = -1;
72
+ let maxPath = null;
73
+ for (const filepath of existing.keys()) {
74
+ const name = filepath.slice(prefix.length);
75
+ const match = EVENT_FILENAME_RE.exec(name);
76
+ if (match === null)
77
+ continue;
78
+ const seqStr = match[1];
79
+ if (seqStr === undefined)
80
+ continue;
81
+ const seq = Number.parseInt(seqStr, 10);
82
+ if (seq > maxSeq) {
83
+ maxSeq = seq;
84
+ maxPath = filepath;
85
+ }
86
+ }
87
+ const carried = {};
88
+ for (const [k, v] of existing)
89
+ carried[k] = v;
90
+ // Terminal-lock: appending a terminal event after an existing one
91
+ // is rejected at push validation. If the run already ended, its
92
+ // `workflow_run.status` is already terminal, so the crash-loop
93
+ // tombstone is redundant -- no-op rather than push a rejected write.
94
+ //
95
+ // This detects the per-event (`events/<seq>.json`) form only, not
96
+ // the sealed combined-log (`events.jsonl`) form, which lives outside
97
+ // `preservePrefix`. That is sufficient here because the anchor run
98
+ // this commit targets is sealed only when the DEPLOYMENT itself is
99
+ // terminal, and the crash-loop latch fires only while the deployment
100
+ // is running -- so the run is never in the sealed form at this call.
101
+ // If a sealed run ever reached here, the append would produce a tree
102
+ // carrying both forms, which push validation rejects, and the caller
103
+ // logs the failure (best-effort tombstone) rather than corrupting.
104
+ if (maxPath !== null) {
105
+ const raw = existing.get(maxPath);
106
+ if (raw !== undefined) {
107
+ const parsed = OnDiskEventEnvelope(JSON.parse(decoder.decode(raw)));
108
+ if (parsed instanceof type.errors) {
109
+ throw new Error(`commitRunFailed: event blob ${maxPath} failed validation: ${parsed.summary}`);
110
+ }
111
+ if (TERMINAL_EVENT_KINDS.has(parsed.type)) {
112
+ return carried;
113
+ }
114
+ }
115
+ }
116
+ const nextSeq = maxSeq < 0 ? 1 : maxSeq + 1;
117
+ const event = {
118
+ kind: "RunFailed",
119
+ seq: nextSeq,
120
+ at: opts.at,
121
+ error: { message: opts.message },
122
+ };
123
+ const onDisk = workflowEventToOnDisk(event, nextSeq);
124
+ carried[`${prefix}${String(nextSeq)}.json`] = JSON.stringify(onDisk);
125
+ appended = true;
126
+ return carried;
127
+ },
128
+ message: `append RunFailed (crash-loop) for run ${opts.runId}`,
129
+ });
130
+ if (!appended) {
131
+ logger.info `commitRunFailed: run ${opts.runId} already terminal; no RunFailed appended`;
132
+ }
133
+ return { commitSha, appended };
134
+ }
@@ -1,5 +1,6 @@
1
- import type { DequeueToProcessingResult, EnqueueInboxArgs, EnqueueInboxResult, MarkConsumedArgs, MarkConsumedResult, Principal, RepoId, RepoStore as SubstrateRepoStore, ReplayProcessingToInboxOpts, ReplayProcessingToInboxResult } from "@intx/hub-sessions/substrate";
2
- import type { OutboundMessage, SendReceipt } from "@intx/types/runtime";
1
+ import type { DequeueToProcessingResult, EnqueueInboxArgs, EnqueueInboxOutcome, MarkConsumedArgs, MarkConsumedResult, Principal, RepoId, RepoStore as SubstrateRepoStore, ReplayProcessingToInboxOpts, ReplayProcessingToInboxResult } from "@intx/hub-sessions/substrate";
2
+ import type { SignalKind } from "@intx/types";
3
+ import type { ApprovalSnapshot, OutboundMessage, SendReceipt } from "@intx/types/runtime";
3
4
  import type { RunCancelled, RunCompleted, RunFailed } from "@intx/workflow";
4
5
  import type { FrameReader, NdjsonReader, NdjsonWriter } from "../ipc/index.js";
5
6
  /**
@@ -69,7 +70,13 @@ export type PrincipalSigner = (kind: WorkflowSupervisorPrincipalKind, payload: U
69
70
  *
70
71
  * `subscribeMailForAddress` returns a disposer the supervisor calls
71
72
  * during teardown. The supplied handler is invoked with the raw RFC
72
- * 2822 message bytes of each inbound message at the address.
73
+ * 2822 message bytes of each inbound message at the address, and returns
74
+ * a promise that resolves once the message is durably accepted (its inbox
75
+ * write landed, or the message was already durably present) and rejects
76
+ * when it was not (a transient failure, a stale refusal, or a phase where
77
+ * the deployment is tearing down). The host propagates that settlement to
78
+ * the wire so a durable-receipt ack is sent only on resolution -- resolve
79
+ * is the ack signal, reject is the withhold signal.
73
80
  *
74
81
  * `sendOutbound` is the OUTBOUND half of mailbox ownership (§3a). The
75
82
  * supervisor is the sole mail owner: the workflow-process child never
@@ -92,7 +99,7 @@ export type PrincipalSigner = (kind: WorkflowSupervisorPrincipalKind, payload: U
92
99
  export interface MailBusBindings {
93
100
  registerAddress(address: string): void;
94
101
  unregisterAddress(address: string): void;
95
- subscribeMailForAddress(address: string, handler: (rawMessage: Uint8Array) => void): () => void;
102
+ subscribeMailForAddress(address: string, handler: (rawMessage: Uint8Array) => Promise<void>): () => void;
96
103
  sendOutbound(senderAddress: string, message: OutboundMessage): Promise<SendReceipt>;
97
104
  }
98
105
  /**
@@ -181,11 +188,33 @@ export type DeriveMailAuditRef = (messageId: string, rawMessage: Uint8Array) =>
181
188
  * surprise.
182
189
  */
183
190
  export interface InboxPrimitives {
184
- enqueueInbox(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, args: EnqueueInboxArgs): Promise<EnqueueInboxResult>;
191
+ enqueueInbox(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, args: EnqueueInboxArgs): Promise<EnqueueInboxOutcome>;
185
192
  dequeueToProcessing(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, address: string): Promise<DequeueToProcessingResult>;
186
193
  markConsumed(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, args: MarkConsumedArgs): Promise<MarkConsumedResult>;
187
194
  replayProcessingToInbox(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, address: string, opts?: ReplayProcessingToInboxOpts): Promise<ReplayProcessingToInboxResult>;
188
195
  }
196
+ /**
197
+ * A control-plane suspension the supervisor forwards to the host after a
198
+ * workflow-process child reports a park. The child supplies `runId`,
199
+ * `correlationId`, and `kind`; the supervisor stamps `anchorRunId` and
200
+ * `agentAddress` from its own bindings before invoking the host's
201
+ * `onSuspensionRegister`. The host (production: the sidecar) turns this into
202
+ * a `signal.correlation.register` frame the hub co-writes the run's routing +
203
+ * approval rows from.
204
+ */
205
+ export interface SuspensionRegistration {
206
+ runId: string;
207
+ correlationId: string;
208
+ kind: SignalKind;
209
+ anchorRunId: string;
210
+ agentAddress: string;
211
+ /**
212
+ * Approver-facing snapshot of the parked tool call, forwarded from the
213
+ * child's `park.notify`. Present only for an ask-rail suspension; the host
214
+ * turns it into the register frame's snapshot.
215
+ */
216
+ approvalSnapshot?: ApprovalSnapshot;
217
+ }
189
218
  /**
190
219
  * Constructor arguments for `createWorkflowSupervisor`. One
191
220
  * `RepoStore` handle plus a `signAsPrincipal` callback that mints
@@ -201,6 +230,49 @@ export interface WorkflowSupervisorBindings {
201
230
  signAsPrincipal: PrincipalSigner;
202
231
  /** Mail-bus surface for address registration and inbound subscription. */
203
232
  mailBus: MailBusBindings;
233
+ /**
234
+ * Optional control-plane suspension sink. Two callers invoke it, both
235
+ * stamping `anchorRunId` and `deploymentMailAddress` (as `agentAddress`)
236
+ * onto the child-supplied `runId`/`correlationId`/`kind`: the upstream-control
237
+ * pump's `park.notify` arm (the happy-path emit at suspend), and
238
+ * `reEmitParkedCorrelations` (the re-establishment re-emit that recovers a
239
+ * register the hub missed while it was down). Production wires this to the
240
+ * sidecar's hub link so a `signal.correlation.register` frame reaches the hub;
241
+ * a host that does not wire it does not register suspensions (today, the
242
+ * tests). Best-effort: a throwing sink is logged and
243
+ * both callers keep going, so one bad register cannot wedge the control pump
244
+ * or abort a re-emit partway through the parked set.
245
+ */
246
+ onSuspensionRegister?: (registration: SuspensionRegistration) => void;
247
+ /**
248
+ * Per-run grants source the dispatch loop consults before it forwards a
249
+ * `trigger.fire`. Unlike `onSuspensionRegister` (best-effort, fire-and-
250
+ * forget), this is a request/response contract: the supervisor awaits the
251
+ * returned `CredentialsSnapshot` and pushes it to the child over the
252
+ * control channel BEFORE firing the run's trigger, so the child's
253
+ * authorize closure binds to the run's grants rather than a stale
254
+ * spawn-time snapshot. A throwing sink is NOT swallowed -- the dispatch
255
+ * loop fails that run (a synthesized `RunFailed`) rather than firing the
256
+ * trigger against absent grants.
257
+ *
258
+ * When this binding is wired, it is the SOLE grants push: `spawn` does not
259
+ * push a spawn-time snapshot, so the per-run push is the only thing that
260
+ * satisfies the child's throw-on-null authorize guard. A caller that
261
+ * injects no per-run sink -- today, the supervisor's own tests -- keeps the
262
+ * spawn-time push instead. Production (the sidecar) wires it to the
263
+ * walk-derived per-step credentials assembly.
264
+ */
265
+ onRunStart?: (args: {
266
+ runId: string;
267
+ anchorRunId: string;
268
+ }) => Promise<import("./credentials.js").CredentialsSnapshot>;
269
+ /**
270
+ * Decrypted credential material for the deployment's tools, delivered to the
271
+ * child on the pre-trigger barrier alongside the grants. Absent when the
272
+ * deployment binds no credentials. A rotation flows through
273
+ * `deliverCredentials`, not this static binding.
274
+ */
275
+ credentialDelivery?: import("@intx/types/sidecar").CredentialDelivery;
204
276
  /** Subprocess spawner the supervisor invokes per spawn. */
205
277
  subprocessSpawner: SubprocessSpawner;
206
278
  /**
@@ -235,8 +307,8 @@ export interface WorkflowSupervisorBindings {
235
307
  workflowRunRepoId: import("@intx/hub-sessions").RepoId;
236
308
  /** Workflow-run repo ref the supervisor commits events to. */
237
309
  workflowRunRef: string;
238
- /** Deployment id baked into the supervisor's principal claims. */
239
- deploymentId: string;
310
+ /** Anchor run id baked into the supervisor's principal claims. */
311
+ anchorRunId: string;
240
312
  /**
241
313
  * Number of steps in the deployed `WorkflowDefinition`
242
314
  * (`stepOrder.length`). The supervisor threads this into the child's
@@ -268,7 +340,7 @@ export interface WorkflowSupervisorBindings {
268
340
  deriveStepAddress: import("./credentials.js").DeriveStepAddress;
269
341
  /**
270
342
  * Optional override for the step's agent-state repo identity. The
271
- * default convention is `<deploymentId>-<stepId>`.
343
+ * default convention is `<anchorRunId>-<stepId>`.
272
344
  */
273
345
  deriveStepRepoId?: import("./credentials.js").DeriveStepRepoId;
274
346
  /**
@@ -377,11 +449,11 @@ export interface WorkflowSupervisorBindings {
377
449
  /**
378
450
  * Workflow-run substrate principal the supervisor uses to author
379
451
  * inbox/processing/consumed writes. The substrate's workflow-run
380
- * kind handler accepts a `{ kind: "supervisor", deploymentId }`
452
+ * kind handler accepts a `{ kind: "supervisor", anchorRunId }`
381
453
  * principal for claim-check writes; the supervisor constructs this
382
454
  * value once at bindings construction and reuses it for every
383
455
  * claim-check operation. Defaults to `{ kind: "supervisor",
384
- * deploymentId }` derived from `bindings.deploymentId`; tests
456
+ * anchorRunId }` derived from `bindings.anchorRunId`; tests
385
457
  * override it when they need to assert on a structurally distinct
386
458
  * principal shape.
387
459
  */
@@ -413,14 +485,65 @@ export interface WorkflowSupervisorBindings {
413
485
  */
414
486
  readyTimeoutMs?: number;
415
487
  /**
416
- * Watchdog timeout (ms) for the supervisor's substrate-write
417
- * handler's wait on the dispatch loop's `markConsumed` when a
418
- * terminal-event blob lands in a proxied write. Defaults to
419
- * `DEFAULT_TERMINAL_WRITE_WATCHDOG_MS`. Tests inject a small value
420
- * so the watchdog path is observable without holding a test loop
421
- * for the production duration.
488
+ * Crash-loop guard: the maximum number of UNEXPECTED workflow-process
489
+ * child exits (crash, OOM, panic, signal) the supervisor tolerates
490
+ * within `crashLoopWindowMs` before it stops respawning and latches the
491
+ * deployment to a terminal state. The boot edge resolves the operator's
492
+ * config and supplies it; absent, `DEFAULT_CRASH_LOOP_MAX_COUNT` (3)
493
+ * applies. The operator owns this bound.
422
494
  */
423
- terminalWriteWatchdogMs?: number;
495
+ crashLoopMaxCount?: number;
496
+ /**
497
+ * Sliding window (ms) over which `crashLoopMaxCount` unexpected exits
498
+ * latch the deployment. Absent, `DEFAULT_CRASH_LOOP_WINDOW_MS` (60s)
499
+ * applies.
500
+ */
501
+ crashLoopWindowMs?: number;
502
+ /**
503
+ * Stable-run duration (ms) after a respawn: once a respawned child has
504
+ * stayed up this long, the crash counter resets, so a burst of crashes
505
+ * followed by stability does not permanently latch. Absent,
506
+ * `DEFAULT_CRASH_LOOP_STABLE_RESET_MS` (60s) applies. Driven by the
507
+ * injectable `setTimer`/`clearTimer` pair so tests are deterministic.
508
+ */
509
+ crashLoopStableResetMs?: number;
510
+ /**
511
+ * Initial respawn backoff (ms): the wait before the FIRST respawn after
512
+ * an unexpected exit. Each subsequent respawn doubles the wait, capped at
513
+ * `respawnBackoffMaxMs`; a stable run resets it to this value. Absent,
514
+ * `DEFAULT_RESPAWN_BACKOFF_INITIAL_MS` (1s) applies.
515
+ */
516
+ respawnBackoffInitialMs?: number;
517
+ /**
518
+ * Maximum respawn backoff (ms) the exponential doubling is capped at, so
519
+ * a rapidly-flapping child does not saturate the host. Absent,
520
+ * `DEFAULT_RESPAWN_BACKOFF_MAX_MS` (30s) applies.
521
+ *
522
+ * Config invariant: keep this comfortably below `crashLoopWindowMs`.
523
+ * Crashes must fall within the window to accumulate toward the latch, and
524
+ * the backoff spaces them apart; a cap at or above the window guarantees a
525
+ * slow flapper's timestamps age out before the count reaches
526
+ * `crashLoopMaxCount`, so the guard never latches. This is necessary but
527
+ * not sufficient: the child's own lifetime also spaces crashes, so a child
528
+ * whose healthy lifetime approaches `crashLoopStableResetMs` can still
529
+ * out-space the window (and never earn a counter reset either), respawning
530
+ * indefinitely. Windowed crash detection cannot catch an arbitrarily slow
531
+ * flapper; the window/backoff/stable-reset trio bounds the FAST flap this
532
+ * guard targets.
533
+ */
534
+ respawnBackoffMaxMs?: number;
535
+ /**
536
+ * Watchdog timeout (ms) for `reEmitParkedCorrelations`' wait on the
537
+ * child's `parked-correlations.response`. Caps the wait so a
538
+ * wedged-but-alive child (whose cohort never tears down, so the
539
+ * cohort-abort rejection never fires) cannot hang the re-registration
540
+ * driver -- and therefore the reconnect/re-establishment caller that
541
+ * awaits it. On expiry the pending query is dropped and the driver
542
+ * returns; the next re-establishment re-drives it. Defaults to
543
+ * `DEFAULT_PARKED_QUERY_WATCHDOG_MS`. Tests inject a small value so the
544
+ * timeout path is observable.
545
+ */
546
+ parkedQueryWatchdogMs?: number;
424
547
  /**
425
548
  * Optional per-message dispatch-timing observer. When supplied, the
426
549
  * dispatch loop invokes it twice per dispatched inbox entry: once with
@@ -490,9 +613,8 @@ export type DispatchSubstrateLeg = "enqueue" | "dequeue" | "runevent" | "markcon
490
613
  * (design §10b). All are cheap filesystem reads against the workflow-run
491
614
  * repo's on-disk working tree, taken only when the observer is wired.
492
615
  *
493
- * - `runsFanOut` — entry count under `runs/` (one subdir per message;
494
- * never pruned). The candidate-(i) "collapse runs"
495
- * win is sized by this.
616
+ * - `runsFanOut` — entry count under `runs/` (the stable top-level run
617
+ * plus any internal body-child runs).
496
618
  * - `consumedFanOut` — entry count under
497
619
  * `addresses/<addr>/consumed/` (one dedup entry per
498
620
  * message; never pruned). The candidate-(iv) "prune
@@ -512,8 +634,13 @@ export type DispatchStructuralCounters = {
512
634
  /**
513
635
  * One observation emitted by `WorkflowSupervisorBindings.onDispatchTiming`.
514
636
  *
637
+ * Both variants key on `messageId`, the per-message identifier (the mail's
638
+ * Message-ID). The top-level run id cannot serve as the key: one deployment
639
+ * keeps that stable id across all of its live trigger occurrences, so it does
640
+ * not distinguish one dispatched message from the next.
641
+ *
515
642
  * The `"roundtrip"` variant is the 4.7 latency-gate bracket: pair the
516
- * `"dispatch-start"` and `"reply-produced"` marks for the same `runId` to
643
+ * `"dispatch-start"` and `"reply-produced"` marks for the same `messageId` to
517
644
  * recover the per-message round-trip. `atMs` is a high-resolution
518
645
  * monotonic timestamp (`performance.now()`).
519
646
  *
@@ -528,12 +655,12 @@ export type DispatchStructuralCounters = {
528
655
  */
529
656
  export type DispatchTimingMark = {
530
657
  kind: "roundtrip";
531
- runId: string;
658
+ messageId: string;
532
659
  marker: "dispatch-start" | "reply-produced";
533
660
  atMs: number;
534
661
  } | {
535
662
  kind: "leg";
536
- runId: string;
663
+ messageId: string;
537
664
  leg: DispatchSubstrateLeg;
538
665
  phase: "start" | "end";
539
666
  atMs: number;
@@ -0,0 +1,131 @@
1
+ import { type AnnotatedPluginFactory, type DirectorRegistry, type ToolDeclaration } from "@intx/agent";
2
+ import type { WorkflowDefinition } from "@intx/workflow/definition";
3
+ export interface LoadWorkflowDefinitionFromClosureArgs {
4
+ /**
5
+ * Directory of the materialized workflow package within the closure:
6
+ * the directory holding the package's `package.json`, with its
7
+ * `node_modules/` already laid out by the closure-materialization
8
+ * machinery so the entry module's bare-specifier imports resolve.
9
+ */
10
+ readonly packageDir: string;
11
+ /**
12
+ * Optional token mixed into the import URL's query string to bust
13
+ * Node's ESM module cache. Node keys the ESM cache by resolved
14
+ * URL/path, not by content: a process that imports the same package
15
+ * directory twice with different bytes underneath (a rare re-apply in
16
+ * a reused child) would otherwise resolve to the first-imported module
17
+ * instance. Passing a per-materialization token (the closure's
18
+ * integrity SRI is the natural choice) makes each materialization a
19
+ * distinct ESM cache entry. Omit it when the process imports a given
20
+ * package directory at most once.
21
+ */
22
+ readonly importCacheKey?: string;
23
+ /**
24
+ * Test seam for dynamic import. Production omits this and the loader
25
+ * uses the native dynamic-import expression. The argument is the
26
+ * `file://` URL the loader resolves for the `interchange.workflow`
27
+ * entry.
28
+ */
29
+ readonly importModule?: (importUrl: string) => Promise<unknown>;
30
+ }
31
+ /**
32
+ * Import the `interchange.workflow` entry from a materialized workflow
33
+ * package closure, evaluate it, and return the validated
34
+ * `WorkflowDefinition` its `defineWorkflow(...)` call produced.
35
+ *
36
+ * @param args - the materialized package directory plus optional import
37
+ * seams
38
+ * @returns the validated `WorkflowDefinition`
39
+ * @throws if the package.json is missing/malformed, declares no
40
+ * `interchange.workflow` entry, the entry path escapes the package
41
+ * directory, the module cannot be imported, or its evaluation does not
42
+ * produce exactly one value that validates as a `WorkflowDefinition`
43
+ */
44
+ export declare function loadWorkflowDefinitionFromClosure(args: LoadWorkflowDefinitionFromClosureArgs): Promise<WorkflowDefinition>;
45
+ export interface LoadWorkflowDirectorRegistryFromClosureArgs {
46
+ /**
47
+ * Directory of the materialized workflow package within the closure --
48
+ * the same directory `loadWorkflowDefinitionFromClosure` reads. Both the
49
+ * approval-time probe and the run-child call this over the SAME frozen
50
+ * closure, so the director set they compose cannot drift.
51
+ */
52
+ readonly packageDir: string;
53
+ /** See `LoadWorkflowDefinitionFromClosureArgs.importCacheKey`. */
54
+ readonly importCacheKey?: string;
55
+ /** Test seam for dynamic import; see the definition loader's variant. */
56
+ readonly importModule?: (importUrl: string) => Promise<unknown>;
57
+ }
58
+ /**
59
+ * Compose the `DirectorRegistry` for a workflow closure from the closure
60
+ * package's OWN `interchange.directors` module (if any), alongside the
61
+ * built-in default director. A package with no `interchange.directors`
62
+ * field composes to the built-ins-only registry -- absence is valid, a
63
+ * workflow need not ship a director. A present-but-empty directors module
64
+ * is malformed and throws, matching the tool-package loader.
65
+ *
66
+ * Only the workflow's OWN package directors are loaded here. Directors
67
+ * shipped by PINNED dependency packages are deliberately not resolved on
68
+ * the source-ref path yet: the airlocked probe does not materialize pinned
69
+ * packages, so loading them here would let the runtime resolve a director
70
+ * the probe never advertised for approval. A workflow referencing a
71
+ * pinned-package director fails closed (the capability walk reports it as
72
+ * unresolved).
73
+ *
74
+ * @throws if the directors entry path escapes the package, the module
75
+ * cannot be imported, or it exports no `AnnotatedDirectorFactory` value
76
+ */
77
+ export declare function loadWorkflowDirectorRegistryFromClosure(args: LoadWorkflowDirectorRegistryFromClosureArgs): Promise<DirectorRegistry>;
78
+ export interface LoadWorkflowPluginsFromClosureArgs {
79
+ /**
80
+ * Directory of the materialized workflow package within the closure --
81
+ * the same directory `loadWorkflowDefinitionFromClosure` reads. Each
82
+ * declared plugin package is resolved from this package's laid-out
83
+ * `node_modules/`, exactly as the workflow entry's own bare-specifier
84
+ * imports resolve.
85
+ */
86
+ readonly packageDir: string;
87
+ /**
88
+ * Plugin-package names the workflow's agents declare via
89
+ * `AgentDefinition.plugins` (`["@intx/tools-lsp"]`). Each MUST be a
90
+ * direct dependency of the workflow package so it is laid out under the
91
+ * workflow package's `node_modules/`. Empty is valid (no plugins).
92
+ */
93
+ readonly plugins: readonly string[];
94
+ /** See `LoadWorkflowDefinitionFromClosureArgs.importCacheKey`. */
95
+ readonly importCacheKey?: string;
96
+ /** Test seam for dynamic import; see the definition loader's variant. */
97
+ readonly importModule?: (importUrl: string) => Promise<unknown>;
98
+ }
99
+ /**
100
+ * Import each declared plugin package's `interchange.tools` module from the
101
+ * materialized workflow closure and collect the `AnnotatedPluginFactory`
102
+ * values it exports. This is the run-child counterpart to the tool-package
103
+ * loader's plugin channel: a source-ref workflow contributes no plugin factory
104
+ * through its agent definition (a plugin has no agent slot), so the child
105
+ * materializes the declared plugins straight from the already-laid-out closure
106
+ * -- no re-download, no manifest -- and feeds them into the existing per-step
107
+ * plugin chain. The closure bytes were SRI-verified when the deploy applied the
108
+ * frozen closure, and resolution walks the same `node_modules/` graph the
109
+ * workflow entry's imports use.
110
+ *
111
+ * @throws if a declared plugin package cannot be resolved, declares no
112
+ * `interchange.tools` entry, the entry escapes the package, cannot be
113
+ * imported, or exports no `AnnotatedPluginFactory` value
114
+ */
115
+ export declare function loadWorkflowPluginFactoriesFromClosure(args: LoadWorkflowPluginsFromClosureArgs): Promise<AnnotatedPluginFactory[]>;
116
+ /**
117
+ * Read the static tool `definitions` each declared plugin package
118
+ * contributes, keyed by plugin-package name, WITHOUT retaining the plugin
119
+ * factory (so the caller never instantiates a plugin, which for LSP would
120
+ * start a subprocess). This is the probe/capability-walk counterpart to
121
+ * `loadWorkflowPluginFactoriesFromClosure`: it loads the SAME plugin module
122
+ * from the SAME frozen closure so the tool grant surface the walk approves
123
+ * matches the plugin the run-child materializes.
124
+ *
125
+ * A plugin package that exports plugin factories but declares no tool
126
+ * definitions (a middleware-only plugin) maps to an empty array -- valid,
127
+ * it contributes no tool grant.
128
+ *
129
+ * @throws under the same conditions as `loadWorkflowPluginFactoriesFromClosure`
130
+ */
131
+ export declare function loadWorkflowPluginToolDefinitionsFromClosure(args: LoadWorkflowPluginsFromClosureArgs): Promise<Map<string, readonly ToolDeclaration[]>>;