@intx/workflow-host 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -10
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +53 -53
- package/dist/adapters/spawn-child.d.ts +71 -42
- package/dist/adapters/spawn-child.js +83 -77
- package/dist/adapters/step-invoker.js +84 -7
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/index.d.ts +2 -1
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +107 -13
- package/dist/child/run-child.js +290 -108
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/conversation-text.d.ts +23 -0
- package/dist/conversation-text.js +56 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3 -2
- package/dist/ipc/control-channel.d.ts +58 -0
- package/dist/ipc/control-channel.js +94 -1
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/seams/scheduler.d.ts +4 -6
- package/dist/seams/scheduler.js +74 -93
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +1 -1
- package/dist/supervisor/credentials.d.ts +11 -10
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +5 -2
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +5 -5
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +82 -25
- package/dist/supervisor/supervisor.js +1313 -410
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +134 -0
- package/dist/supervisor/types.d.ts +150 -23
- package/dist/workflow-definition-loader.d.ts +131 -0
- package/dist/workflow-definition-loader.js +316 -0
- package/package.json +12 -11
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { RepoId, RepoStore as SubstrateRepoStore } from "@intx/hub-sessions/substrate";
|
|
2
|
+
export type CommitRunFailedOpts = {
|
|
3
|
+
/** Substrate handle the supervisor writes through. */
|
|
4
|
+
substrate: SubstrateRepoStore;
|
|
5
|
+
/** Workflow-run repo for this deployment. */
|
|
6
|
+
repoId: RepoId;
|
|
7
|
+
/** Events ref the workflow-run repo writes to. */
|
|
8
|
+
ref: string;
|
|
9
|
+
/** Anchor run id used to construct the supervisor principal. */
|
|
10
|
+
anchorRunId: string;
|
|
11
|
+
/** Run id whose event log receives the RunFailed entry. */
|
|
12
|
+
runId: string;
|
|
13
|
+
/** ISO-8601 commit timestamp the event carries. */
|
|
14
|
+
at: string;
|
|
15
|
+
/** Operator-facing failure reason on `RunFailed.error.message`. */
|
|
16
|
+
message: string;
|
|
17
|
+
};
|
|
18
|
+
export type CommitRunFailedResult = {
|
|
19
|
+
/** Substrate-assigned commit SHA the write produced. */
|
|
20
|
+
commitSha: string;
|
|
21
|
+
/**
|
|
22
|
+
* True when a `RunFailed` was appended; false when the run was already
|
|
23
|
+
* terminal and the write was a no-op (terminal-lock respected).
|
|
24
|
+
*/
|
|
25
|
+
appended: boolean;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Append a supervisor-authored `RunFailed` to a run's event log, unless the
|
|
29
|
+
* run is already terminal. The next seq is computed inside the substrate
|
|
30
|
+
* merge (atomic against any concurrent writer under the per-repo lock): the
|
|
31
|
+
* first event on an empty tree lands at seq 1 (the runtime's convention),
|
|
32
|
+
* otherwise at `maxSeq + 1` so the log stays seq-contiguous. If the run's
|
|
33
|
+
* highest-seq event is already terminal, the write is a no-op so push
|
|
34
|
+
* validation's terminal-lock is never tripped.
|
|
35
|
+
*/
|
|
36
|
+
export declare function commitRunFailed(opts: CommitRunFailedOpts): Promise<CommitRunFailedResult>;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// Supervisor-authored terminal-event commit for the crash-loop guard.
|
|
2
|
+
//
|
|
3
|
+
// When the crash-loop guard latches, the workflow-process child is dead
|
|
4
|
+
// and cannot commit its own terminal event, yet the deployment's run must
|
|
5
|
+
// reach a terminal state so its external `workflow_run.status` flips to
|
|
6
|
+
// `failed` -- the sole durable, queryable signal of a crash-loop. The
|
|
7
|
+
// supervisor's `crash-looping` phase is in-memory and per-process; no
|
|
8
|
+
// external reader observes it. The supervisor, as the sole writer of the
|
|
9
|
+
// workflow-run repo, authors a `RunFailed` for the deployment's stable
|
|
10
|
+
// run so the crash-loop leaves a durable tombstone.
|
|
11
|
+
//
|
|
12
|
+
// Unlike `commitCancelRequested`, a terminal workflow event carries no
|
|
13
|
+
// signature: only `CancelRequested` is signed (for its origin<->principal
|
|
14
|
+
// cross-check at push validation), so this writer is unsigned. It writes
|
|
15
|
+
// under the `supervisor` principal, which the workflow-run kind handler
|
|
16
|
+
// authorizes for the deployment's own event log (`repoId.id ===
|
|
17
|
+
// anchorRunId`); terminal events have no per-type authorship check.
|
|
18
|
+
import { type } from "arktype";
|
|
19
|
+
import { getLogger } from "@intx/log";
|
|
20
|
+
import { workflowEventToOnDisk } from "../adapters/repo-store.js";
|
|
21
|
+
const logger = getLogger(["workflow-host", "supervisor", "terminal-commit"]);
|
|
22
|
+
/** Path layout inside the workflow-run repo: `runs/<runId>/events/<seq>.json`. */
|
|
23
|
+
const RUNS_PREFIX = "runs";
|
|
24
|
+
const EVENTS_DIR = "events";
|
|
25
|
+
const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
|
|
26
|
+
/**
|
|
27
|
+
* Terminal run-event kinds, mirroring the runtime's terminal vocabulary
|
|
28
|
+
* (`RunCompleted`/`RunFailed`/`RunCancelled`). Inlined because the workflow
|
|
29
|
+
* package exports only the phase-level `isTerminalRunPhase`, not an
|
|
30
|
+
* event-kind set; the child runtime (`run-child`) and the substrate adapter
|
|
31
|
+
* (`repo-store`) inline the same three kinds. Reducing the log to a phase to
|
|
32
|
+
* reuse `isTerminalRunPhase` would be strictly heavier for a last-event
|
|
33
|
+
* type check.
|
|
34
|
+
*/
|
|
35
|
+
const TERMINAL_EVENT_KINDS = new Set([
|
|
36
|
+
"RunCompleted",
|
|
37
|
+
"RunFailed",
|
|
38
|
+
"RunCancelled",
|
|
39
|
+
]);
|
|
40
|
+
const SUPERVISOR_PRINCIPAL_KIND = "supervisor";
|
|
41
|
+
/**
|
|
42
|
+
* On-disk event envelope, validated at the substrate read boundary. Only
|
|
43
|
+
* `seq` and `type` are load-bearing here (max-seq computation and the
|
|
44
|
+
* terminal-lock check); the rest of the event body is ignored.
|
|
45
|
+
*/
|
|
46
|
+
const OnDiskEventEnvelope = type({
|
|
47
|
+
seq: "number >= 0",
|
|
48
|
+
type: "string",
|
|
49
|
+
"+": "ignore",
|
|
50
|
+
});
|
|
51
|
+
/**
|
|
52
|
+
* Append a supervisor-authored `RunFailed` to a run's event log, unless the
|
|
53
|
+
* run is already terminal. The next seq is computed inside the substrate
|
|
54
|
+
* merge (atomic against any concurrent writer under the per-repo lock): the
|
|
55
|
+
* first event on an empty tree lands at seq 1 (the runtime's convention),
|
|
56
|
+
* otherwise at `maxSeq + 1` so the log stays seq-contiguous. If the run's
|
|
57
|
+
* highest-seq event is already terminal, the write is a no-op so push
|
|
58
|
+
* validation's terminal-lock is never tripped.
|
|
59
|
+
*/
|
|
60
|
+
export async function commitRunFailed(opts) {
|
|
61
|
+
const prefix = `${RUNS_PREFIX}/${opts.runId}/${EVENTS_DIR}/`;
|
|
62
|
+
const principal = {
|
|
63
|
+
kind: SUPERVISOR_PRINCIPAL_KIND,
|
|
64
|
+
anchorRunId: opts.anchorRunId,
|
|
65
|
+
};
|
|
66
|
+
const decoder = new TextDecoder();
|
|
67
|
+
let appended = false;
|
|
68
|
+
const { commitSha } = await opts.substrate.writeTreePreservingPrefix(principal, opts.repoId, opts.ref, {
|
|
69
|
+
preservePrefix: prefix,
|
|
70
|
+
merge: async (existing) => {
|
|
71
|
+
let maxSeq = -1;
|
|
72
|
+
let maxPath = null;
|
|
73
|
+
for (const filepath of existing.keys()) {
|
|
74
|
+
const name = filepath.slice(prefix.length);
|
|
75
|
+
const match = EVENT_FILENAME_RE.exec(name);
|
|
76
|
+
if (match === null)
|
|
77
|
+
continue;
|
|
78
|
+
const seqStr = match[1];
|
|
79
|
+
if (seqStr === undefined)
|
|
80
|
+
continue;
|
|
81
|
+
const seq = Number.parseInt(seqStr, 10);
|
|
82
|
+
if (seq > maxSeq) {
|
|
83
|
+
maxSeq = seq;
|
|
84
|
+
maxPath = filepath;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const carried = {};
|
|
88
|
+
for (const [k, v] of existing)
|
|
89
|
+
carried[k] = v;
|
|
90
|
+
// Terminal-lock: appending a terminal event after an existing one
|
|
91
|
+
// is rejected at push validation. If the run already ended, its
|
|
92
|
+
// `workflow_run.status` is already terminal, so the crash-loop
|
|
93
|
+
// tombstone is redundant -- no-op rather than push a rejected write.
|
|
94
|
+
//
|
|
95
|
+
// This detects the per-event (`events/<seq>.json`) form only, not
|
|
96
|
+
// the sealed combined-log (`events.jsonl`) form, which lives outside
|
|
97
|
+
// `preservePrefix`. That is sufficient here because the anchor run
|
|
98
|
+
// this commit targets is sealed only when the DEPLOYMENT itself is
|
|
99
|
+
// terminal, and the crash-loop latch fires only while the deployment
|
|
100
|
+
// is running -- so the run is never in the sealed form at this call.
|
|
101
|
+
// If a sealed run ever reached here, the append would produce a tree
|
|
102
|
+
// carrying both forms, which push validation rejects, and the caller
|
|
103
|
+
// logs the failure (best-effort tombstone) rather than corrupting.
|
|
104
|
+
if (maxPath !== null) {
|
|
105
|
+
const raw = existing.get(maxPath);
|
|
106
|
+
if (raw !== undefined) {
|
|
107
|
+
const parsed = OnDiskEventEnvelope(JSON.parse(decoder.decode(raw)));
|
|
108
|
+
if (parsed instanceof type.errors) {
|
|
109
|
+
throw new Error(`commitRunFailed: event blob ${maxPath} failed validation: ${parsed.summary}`);
|
|
110
|
+
}
|
|
111
|
+
if (TERMINAL_EVENT_KINDS.has(parsed.type)) {
|
|
112
|
+
return carried;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const nextSeq = maxSeq < 0 ? 1 : maxSeq + 1;
|
|
117
|
+
const event = {
|
|
118
|
+
kind: "RunFailed",
|
|
119
|
+
seq: nextSeq,
|
|
120
|
+
at: opts.at,
|
|
121
|
+
error: { message: opts.message },
|
|
122
|
+
};
|
|
123
|
+
const onDisk = workflowEventToOnDisk(event, nextSeq);
|
|
124
|
+
carried[`${prefix}${String(nextSeq)}.json`] = JSON.stringify(onDisk);
|
|
125
|
+
appended = true;
|
|
126
|
+
return carried;
|
|
127
|
+
},
|
|
128
|
+
message: `append RunFailed (crash-loop) for run ${opts.runId}`,
|
|
129
|
+
});
|
|
130
|
+
if (!appended) {
|
|
131
|
+
logger.info `commitRunFailed: run ${opts.runId} already terminal; no RunFailed appended`;
|
|
132
|
+
}
|
|
133
|
+
return { commitSha, appended };
|
|
134
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { DequeueToProcessingResult, EnqueueInboxArgs,
|
|
2
|
-
import type {
|
|
1
|
+
import type { DequeueToProcessingResult, EnqueueInboxArgs, EnqueueInboxOutcome, MarkConsumedArgs, MarkConsumedResult, Principal, RepoId, RepoStore as SubstrateRepoStore, ReplayProcessingToInboxOpts, ReplayProcessingToInboxResult } from "@intx/hub-sessions/substrate";
|
|
2
|
+
import type { SignalKind } from "@intx/types";
|
|
3
|
+
import type { ApprovalSnapshot, OutboundMessage, SendReceipt } from "@intx/types/runtime";
|
|
3
4
|
import type { RunCancelled, RunCompleted, RunFailed } from "@intx/workflow";
|
|
4
5
|
import type { FrameReader, NdjsonReader, NdjsonWriter } from "../ipc/index.js";
|
|
5
6
|
/**
|
|
@@ -69,7 +70,13 @@ export type PrincipalSigner = (kind: WorkflowSupervisorPrincipalKind, payload: U
|
|
|
69
70
|
*
|
|
70
71
|
* `subscribeMailForAddress` returns a disposer the supervisor calls
|
|
71
72
|
* during teardown. The supplied handler is invoked with the raw RFC
|
|
72
|
-
* 2822 message bytes of each inbound message at the address
|
|
73
|
+
* 2822 message bytes of each inbound message at the address, and returns
|
|
74
|
+
* a promise that resolves once the message is durably accepted (its inbox
|
|
75
|
+
* write landed, or the message was already durably present) and rejects
|
|
76
|
+
* when it was not (a transient failure, a stale refusal, or a phase where
|
|
77
|
+
* the deployment is tearing down). The host propagates that settlement to
|
|
78
|
+
* the wire so a durable-receipt ack is sent only on resolution -- resolve
|
|
79
|
+
* is the ack signal, reject is the withhold signal.
|
|
73
80
|
*
|
|
74
81
|
* `sendOutbound` is the OUTBOUND half of mailbox ownership (§3a). The
|
|
75
82
|
* supervisor is the sole mail owner: the workflow-process child never
|
|
@@ -92,7 +99,7 @@ export type PrincipalSigner = (kind: WorkflowSupervisorPrincipalKind, payload: U
|
|
|
92
99
|
export interface MailBusBindings {
|
|
93
100
|
registerAddress(address: string): void;
|
|
94
101
|
unregisterAddress(address: string): void;
|
|
95
|
-
subscribeMailForAddress(address: string, handler: (rawMessage: Uint8Array) => void): () => void;
|
|
102
|
+
subscribeMailForAddress(address: string, handler: (rawMessage: Uint8Array) => Promise<void>): () => void;
|
|
96
103
|
sendOutbound(senderAddress: string, message: OutboundMessage): Promise<SendReceipt>;
|
|
97
104
|
}
|
|
98
105
|
/**
|
|
@@ -181,11 +188,33 @@ export type DeriveMailAuditRef = (messageId: string, rawMessage: Uint8Array) =>
|
|
|
181
188
|
* surprise.
|
|
182
189
|
*/
|
|
183
190
|
export interface InboxPrimitives {
|
|
184
|
-
enqueueInbox(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, args: EnqueueInboxArgs): Promise<
|
|
191
|
+
enqueueInbox(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, args: EnqueueInboxArgs): Promise<EnqueueInboxOutcome>;
|
|
185
192
|
dequeueToProcessing(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, address: string): Promise<DequeueToProcessingResult>;
|
|
186
193
|
markConsumed(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, args: MarkConsumedArgs): Promise<MarkConsumedResult>;
|
|
187
194
|
replayProcessingToInbox(store: SubstrateRepoStore, principal: Principal, repoId: RepoId, address: string, opts?: ReplayProcessingToInboxOpts): Promise<ReplayProcessingToInboxResult>;
|
|
188
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* A control-plane suspension the supervisor forwards to the host after a
|
|
198
|
+
* workflow-process child reports a park. The child supplies `runId`,
|
|
199
|
+
* `correlationId`, and `kind`; the supervisor stamps `anchorRunId` and
|
|
200
|
+
* `agentAddress` from its own bindings before invoking the host's
|
|
201
|
+
* `onSuspensionRegister`. The host (production: the sidecar) turns this into
|
|
202
|
+
* a `signal.correlation.register` frame the hub co-writes the run's routing +
|
|
203
|
+
* approval rows from.
|
|
204
|
+
*/
|
|
205
|
+
export interface SuspensionRegistration {
|
|
206
|
+
runId: string;
|
|
207
|
+
correlationId: string;
|
|
208
|
+
kind: SignalKind;
|
|
209
|
+
anchorRunId: string;
|
|
210
|
+
agentAddress: string;
|
|
211
|
+
/**
|
|
212
|
+
* Approver-facing snapshot of the parked tool call, forwarded from the
|
|
213
|
+
* child's `park.notify`. Present only for an ask-rail suspension; the host
|
|
214
|
+
* turns it into the register frame's snapshot.
|
|
215
|
+
*/
|
|
216
|
+
approvalSnapshot?: ApprovalSnapshot;
|
|
217
|
+
}
|
|
189
218
|
/**
|
|
190
219
|
* Constructor arguments for `createWorkflowSupervisor`. One
|
|
191
220
|
* `RepoStore` handle plus a `signAsPrincipal` callback that mints
|
|
@@ -201,6 +230,49 @@ export interface WorkflowSupervisorBindings {
|
|
|
201
230
|
signAsPrincipal: PrincipalSigner;
|
|
202
231
|
/** Mail-bus surface for address registration and inbound subscription. */
|
|
203
232
|
mailBus: MailBusBindings;
|
|
233
|
+
/**
|
|
234
|
+
* Optional control-plane suspension sink. Two callers invoke it, both
|
|
235
|
+
* stamping `anchorRunId` and `deploymentMailAddress` (as `agentAddress`)
|
|
236
|
+
* onto the child-supplied `runId`/`correlationId`/`kind`: the upstream-control
|
|
237
|
+
* pump's `park.notify` arm (the happy-path emit at suspend), and
|
|
238
|
+
* `reEmitParkedCorrelations` (the re-establishment re-emit that recovers a
|
|
239
|
+
* register the hub missed while it was down). Production wires this to the
|
|
240
|
+
* sidecar's hub link so a `signal.correlation.register` frame reaches the hub;
|
|
241
|
+
* a host that does not wire it does not register suspensions (today, the
|
|
242
|
+
* tests). Best-effort: a throwing sink is logged and
|
|
243
|
+
* both callers keep going, so one bad register cannot wedge the control pump
|
|
244
|
+
* or abort a re-emit partway through the parked set.
|
|
245
|
+
*/
|
|
246
|
+
onSuspensionRegister?: (registration: SuspensionRegistration) => void;
|
|
247
|
+
/**
|
|
248
|
+
* Per-run grants source the dispatch loop consults before it forwards a
|
|
249
|
+
* `trigger.fire`. Unlike `onSuspensionRegister` (best-effort, fire-and-
|
|
250
|
+
* forget), this is a request/response contract: the supervisor awaits the
|
|
251
|
+
* returned `CredentialsSnapshot` and pushes it to the child over the
|
|
252
|
+
* control channel BEFORE firing the run's trigger, so the child's
|
|
253
|
+
* authorize closure binds to the run's grants rather than a stale
|
|
254
|
+
* spawn-time snapshot. A throwing sink is NOT swallowed -- the dispatch
|
|
255
|
+
* loop fails that run (a synthesized `RunFailed`) rather than firing the
|
|
256
|
+
* trigger against absent grants.
|
|
257
|
+
*
|
|
258
|
+
* When this binding is wired, it is the SOLE grants push: `spawn` does not
|
|
259
|
+
* push a spawn-time snapshot, so the per-run push is the only thing that
|
|
260
|
+
* satisfies the child's throw-on-null authorize guard. A caller that
|
|
261
|
+
* injects no per-run sink -- today, the supervisor's own tests -- keeps the
|
|
262
|
+
* spawn-time push instead. Production (the sidecar) wires it to the
|
|
263
|
+
* walk-derived per-step credentials assembly.
|
|
264
|
+
*/
|
|
265
|
+
onRunStart?: (args: {
|
|
266
|
+
runId: string;
|
|
267
|
+
anchorRunId: string;
|
|
268
|
+
}) => Promise<import("./credentials.js").CredentialsSnapshot>;
|
|
269
|
+
/**
|
|
270
|
+
* Decrypted credential material for the deployment's tools, delivered to the
|
|
271
|
+
* child on the pre-trigger barrier alongside the grants. Absent when the
|
|
272
|
+
* deployment binds no credentials. A rotation flows through
|
|
273
|
+
* `deliverCredentials`, not this static binding.
|
|
274
|
+
*/
|
|
275
|
+
credentialDelivery?: import("@intx/types/sidecar").CredentialDelivery;
|
|
204
276
|
/** Subprocess spawner the supervisor invokes per spawn. */
|
|
205
277
|
subprocessSpawner: SubprocessSpawner;
|
|
206
278
|
/**
|
|
@@ -235,8 +307,8 @@ export interface WorkflowSupervisorBindings {
|
|
|
235
307
|
workflowRunRepoId: import("@intx/hub-sessions").RepoId;
|
|
236
308
|
/** Workflow-run repo ref the supervisor commits events to. */
|
|
237
309
|
workflowRunRef: string;
|
|
238
|
-
/**
|
|
239
|
-
|
|
310
|
+
/** Anchor run id baked into the supervisor's principal claims. */
|
|
311
|
+
anchorRunId: string;
|
|
240
312
|
/**
|
|
241
313
|
* Number of steps in the deployed `WorkflowDefinition`
|
|
242
314
|
* (`stepOrder.length`). The supervisor threads this into the child's
|
|
@@ -268,7 +340,7 @@ export interface WorkflowSupervisorBindings {
|
|
|
268
340
|
deriveStepAddress: import("./credentials.js").DeriveStepAddress;
|
|
269
341
|
/**
|
|
270
342
|
* Optional override for the step's agent-state repo identity. The
|
|
271
|
-
* default convention is `<
|
|
343
|
+
* default convention is `<anchorRunId>-<stepId>`.
|
|
272
344
|
*/
|
|
273
345
|
deriveStepRepoId?: import("./credentials.js").DeriveStepRepoId;
|
|
274
346
|
/**
|
|
@@ -377,11 +449,11 @@ export interface WorkflowSupervisorBindings {
|
|
|
377
449
|
/**
|
|
378
450
|
* Workflow-run substrate principal the supervisor uses to author
|
|
379
451
|
* inbox/processing/consumed writes. The substrate's workflow-run
|
|
380
|
-
* kind handler accepts a `{ kind: "supervisor",
|
|
452
|
+
* kind handler accepts a `{ kind: "supervisor", anchorRunId }`
|
|
381
453
|
* principal for claim-check writes; the supervisor constructs this
|
|
382
454
|
* value once at bindings construction and reuses it for every
|
|
383
455
|
* claim-check operation. Defaults to `{ kind: "supervisor",
|
|
384
|
-
*
|
|
456
|
+
* anchorRunId }` derived from `bindings.anchorRunId`; tests
|
|
385
457
|
* override it when they need to assert on a structurally distinct
|
|
386
458
|
* principal shape.
|
|
387
459
|
*/
|
|
@@ -413,14 +485,65 @@ export interface WorkflowSupervisorBindings {
|
|
|
413
485
|
*/
|
|
414
486
|
readyTimeoutMs?: number;
|
|
415
487
|
/**
|
|
416
|
-
*
|
|
417
|
-
*
|
|
418
|
-
*
|
|
419
|
-
*
|
|
420
|
-
*
|
|
421
|
-
*
|
|
488
|
+
* Crash-loop guard: the maximum number of UNEXPECTED workflow-process
|
|
489
|
+
* child exits (crash, OOM, panic, signal) the supervisor tolerates
|
|
490
|
+
* within `crashLoopWindowMs` before it stops respawning and latches the
|
|
491
|
+
* deployment to a terminal state. The boot edge resolves the operator's
|
|
492
|
+
* config and supplies it; absent, `DEFAULT_CRASH_LOOP_MAX_COUNT` (3)
|
|
493
|
+
* applies. The operator owns this bound.
|
|
422
494
|
*/
|
|
423
|
-
|
|
495
|
+
crashLoopMaxCount?: number;
|
|
496
|
+
/**
|
|
497
|
+
* Sliding window (ms) over which `crashLoopMaxCount` unexpected exits
|
|
498
|
+
* latch the deployment. Absent, `DEFAULT_CRASH_LOOP_WINDOW_MS` (60s)
|
|
499
|
+
* applies.
|
|
500
|
+
*/
|
|
501
|
+
crashLoopWindowMs?: number;
|
|
502
|
+
/**
|
|
503
|
+
* Stable-run duration (ms) after a respawn: once a respawned child has
|
|
504
|
+
* stayed up this long, the crash counter resets, so a burst of crashes
|
|
505
|
+
* followed by stability does not permanently latch. Absent,
|
|
506
|
+
* `DEFAULT_CRASH_LOOP_STABLE_RESET_MS` (60s) applies. Driven by the
|
|
507
|
+
* injectable `setTimer`/`clearTimer` pair so tests are deterministic.
|
|
508
|
+
*/
|
|
509
|
+
crashLoopStableResetMs?: number;
|
|
510
|
+
/**
|
|
511
|
+
* Initial respawn backoff (ms): the wait before the FIRST respawn after
|
|
512
|
+
* an unexpected exit. Each subsequent respawn doubles the wait, capped at
|
|
513
|
+
* `respawnBackoffMaxMs`; a stable run resets it to this value. Absent,
|
|
514
|
+
* `DEFAULT_RESPAWN_BACKOFF_INITIAL_MS` (1s) applies.
|
|
515
|
+
*/
|
|
516
|
+
respawnBackoffInitialMs?: number;
|
|
517
|
+
/**
|
|
518
|
+
* Maximum respawn backoff (ms) the exponential doubling is capped at, so
|
|
519
|
+
* a rapidly-flapping child does not saturate the host. Absent,
|
|
520
|
+
* `DEFAULT_RESPAWN_BACKOFF_MAX_MS` (30s) applies.
|
|
521
|
+
*
|
|
522
|
+
* Config invariant: keep this comfortably below `crashLoopWindowMs`.
|
|
523
|
+
* Crashes must fall within the window to accumulate toward the latch, and
|
|
524
|
+
* the backoff spaces them apart; a cap at or above the window guarantees a
|
|
525
|
+
* slow flapper's timestamps age out before the count reaches
|
|
526
|
+
* `crashLoopMaxCount`, so the guard never latches. This is necessary but
|
|
527
|
+
* not sufficient: the child's own lifetime also spaces crashes, so a child
|
|
528
|
+
* whose healthy lifetime approaches `crashLoopStableResetMs` can still
|
|
529
|
+
* out-space the window (and never earn a counter reset either), respawning
|
|
530
|
+
* indefinitely. Windowed crash detection cannot catch an arbitrarily slow
|
|
531
|
+
* flapper; the window/backoff/stable-reset trio bounds the FAST flap this
|
|
532
|
+
* guard targets.
|
|
533
|
+
*/
|
|
534
|
+
respawnBackoffMaxMs?: number;
|
|
535
|
+
/**
|
|
536
|
+
* Watchdog timeout (ms) for `reEmitParkedCorrelations`' wait on the
|
|
537
|
+
* child's `parked-correlations.response`. Caps the wait so a
|
|
538
|
+
* wedged-but-alive child (whose cohort never tears down, so the
|
|
539
|
+
* cohort-abort rejection never fires) cannot hang the re-registration
|
|
540
|
+
* driver -- and therefore the reconnect/re-establishment caller that
|
|
541
|
+
* awaits it. On expiry the pending query is dropped and the driver
|
|
542
|
+
* returns; the next re-establishment re-drives it. Defaults to
|
|
543
|
+
* `DEFAULT_PARKED_QUERY_WATCHDOG_MS`. Tests inject a small value so the
|
|
544
|
+
* timeout path is observable.
|
|
545
|
+
*/
|
|
546
|
+
parkedQueryWatchdogMs?: number;
|
|
424
547
|
/**
|
|
425
548
|
* Optional per-message dispatch-timing observer. When supplied, the
|
|
426
549
|
* dispatch loop invokes it twice per dispatched inbox entry: once with
|
|
@@ -490,9 +613,8 @@ export type DispatchSubstrateLeg = "enqueue" | "dequeue" | "runevent" | "markcon
|
|
|
490
613
|
* (design §10b). All are cheap filesystem reads against the workflow-run
|
|
491
614
|
* repo's on-disk working tree, taken only when the observer is wired.
|
|
492
615
|
*
|
|
493
|
-
* - `runsFanOut` — entry count under `runs/` (
|
|
494
|
-
*
|
|
495
|
-
* win is sized by this.
|
|
616
|
+
* - `runsFanOut` — entry count under `runs/` (the stable top-level run
|
|
617
|
+
* plus any internal body-child runs).
|
|
496
618
|
* - `consumedFanOut` — entry count under
|
|
497
619
|
* `addresses/<addr>/consumed/` (one dedup entry per
|
|
498
620
|
* message; never pruned). The candidate-(iv) "prune
|
|
@@ -512,8 +634,13 @@ export type DispatchStructuralCounters = {
|
|
|
512
634
|
/**
|
|
513
635
|
* One observation emitted by `WorkflowSupervisorBindings.onDispatchTiming`.
|
|
514
636
|
*
|
|
637
|
+
* Both variants key on `messageId`, the per-message identifier (the mail's
|
|
638
|
+
* Message-ID). The top-level run id cannot serve as the key: one deployment
|
|
639
|
+
* keeps that stable id across all of its live trigger occurrences, so it does
|
|
640
|
+
* not distinguish one dispatched message from the next.
|
|
641
|
+
*
|
|
515
642
|
* The `"roundtrip"` variant is the 4.7 latency-gate bracket: pair the
|
|
516
|
-
* `"dispatch-start"` and `"reply-produced"` marks for the same `
|
|
643
|
+
* `"dispatch-start"` and `"reply-produced"` marks for the same `messageId` to
|
|
517
644
|
* recover the per-message round-trip. `atMs` is a high-resolution
|
|
518
645
|
* monotonic timestamp (`performance.now()`).
|
|
519
646
|
*
|
|
@@ -528,12 +655,12 @@ export type DispatchStructuralCounters = {
|
|
|
528
655
|
*/
|
|
529
656
|
export type DispatchTimingMark = {
|
|
530
657
|
kind: "roundtrip";
|
|
531
|
-
|
|
658
|
+
messageId: string;
|
|
532
659
|
marker: "dispatch-start" | "reply-produced";
|
|
533
660
|
atMs: number;
|
|
534
661
|
} | {
|
|
535
662
|
kind: "leg";
|
|
536
|
-
|
|
663
|
+
messageId: string;
|
|
537
664
|
leg: DispatchSubstrateLeg;
|
|
538
665
|
phase: "start" | "end";
|
|
539
666
|
atMs: number;
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { type AnnotatedPluginFactory, type DirectorRegistry, type ToolDeclaration } from "@intx/agent";
|
|
2
|
+
import type { WorkflowDefinition } from "@intx/workflow/definition";
|
|
3
|
+
export interface LoadWorkflowDefinitionFromClosureArgs {
|
|
4
|
+
/**
|
|
5
|
+
* Directory of the materialized workflow package within the closure:
|
|
6
|
+
* the directory holding the package's `package.json`, with its
|
|
7
|
+
* `node_modules/` already laid out by the closure-materialization
|
|
8
|
+
* machinery so the entry module's bare-specifier imports resolve.
|
|
9
|
+
*/
|
|
10
|
+
readonly packageDir: string;
|
|
11
|
+
/**
|
|
12
|
+
* Optional token mixed into the import URL's query string to bust
|
|
13
|
+
* Node's ESM module cache. Node keys the ESM cache by resolved
|
|
14
|
+
* URL/path, not by content: a process that imports the same package
|
|
15
|
+
* directory twice with different bytes underneath (a rare re-apply in
|
|
16
|
+
* a reused child) would otherwise resolve to the first-imported module
|
|
17
|
+
* instance. Passing a per-materialization token (the closure's
|
|
18
|
+
* integrity SRI is the natural choice) makes each materialization a
|
|
19
|
+
* distinct ESM cache entry. Omit it when the process imports a given
|
|
20
|
+
* package directory at most once.
|
|
21
|
+
*/
|
|
22
|
+
readonly importCacheKey?: string;
|
|
23
|
+
/**
|
|
24
|
+
* Test seam for dynamic import. Production omits this and the loader
|
|
25
|
+
* uses the native dynamic-import expression. The argument is the
|
|
26
|
+
* `file://` URL the loader resolves for the `interchange.workflow`
|
|
27
|
+
* entry.
|
|
28
|
+
*/
|
|
29
|
+
readonly importModule?: (importUrl: string) => Promise<unknown>;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Import the `interchange.workflow` entry from a materialized workflow
|
|
33
|
+
* package closure, evaluate it, and return the validated
|
|
34
|
+
* `WorkflowDefinition` its `defineWorkflow(...)` call produced.
|
|
35
|
+
*
|
|
36
|
+
* @param args - the materialized package directory plus optional import
|
|
37
|
+
* seams
|
|
38
|
+
* @returns the validated `WorkflowDefinition`
|
|
39
|
+
* @throws if the package.json is missing/malformed, declares no
|
|
40
|
+
* `interchange.workflow` entry, the entry path escapes the package
|
|
41
|
+
* directory, the module cannot be imported, or its evaluation does not
|
|
42
|
+
* produce exactly one value that validates as a `WorkflowDefinition`
|
|
43
|
+
*/
|
|
44
|
+
export declare function loadWorkflowDefinitionFromClosure(args: LoadWorkflowDefinitionFromClosureArgs): Promise<WorkflowDefinition>;
|
|
45
|
+
export interface LoadWorkflowDirectorRegistryFromClosureArgs {
|
|
46
|
+
/**
|
|
47
|
+
* Directory of the materialized workflow package within the closure --
|
|
48
|
+
* the same directory `loadWorkflowDefinitionFromClosure` reads. Both the
|
|
49
|
+
* approval-time probe and the run-child call this over the SAME frozen
|
|
50
|
+
* closure, so the director set they compose cannot drift.
|
|
51
|
+
*/
|
|
52
|
+
readonly packageDir: string;
|
|
53
|
+
/** See `LoadWorkflowDefinitionFromClosureArgs.importCacheKey`. */
|
|
54
|
+
readonly importCacheKey?: string;
|
|
55
|
+
/** Test seam for dynamic import; see the definition loader's variant. */
|
|
56
|
+
readonly importModule?: (importUrl: string) => Promise<unknown>;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Compose the `DirectorRegistry` for a workflow closure from the closure
|
|
60
|
+
* package's OWN `interchange.directors` module (if any), alongside the
|
|
61
|
+
* built-in default director. A package with no `interchange.directors`
|
|
62
|
+
* field composes to the built-ins-only registry -- absence is valid, a
|
|
63
|
+
* workflow need not ship a director. A present-but-empty directors module
|
|
64
|
+
* is malformed and throws, matching the tool-package loader.
|
|
65
|
+
*
|
|
66
|
+
* Only the workflow's OWN package directors are loaded here. Directors
|
|
67
|
+
* shipped by PINNED dependency packages are deliberately not resolved on
|
|
68
|
+
* the source-ref path yet: the airlocked probe does not materialize pinned
|
|
69
|
+
* packages, so loading them here would let the runtime resolve a director
|
|
70
|
+
* the probe never advertised for approval. A workflow referencing a
|
|
71
|
+
* pinned-package director fails closed (the capability walk reports it as
|
|
72
|
+
* unresolved).
|
|
73
|
+
*
|
|
74
|
+
* @throws if the directors entry path escapes the package, the module
|
|
75
|
+
* cannot be imported, or it exports no `AnnotatedDirectorFactory` value
|
|
76
|
+
*/
|
|
77
|
+
export declare function loadWorkflowDirectorRegistryFromClosure(args: LoadWorkflowDirectorRegistryFromClosureArgs): Promise<DirectorRegistry>;
|
|
78
|
+
export interface LoadWorkflowPluginsFromClosureArgs {
|
|
79
|
+
/**
|
|
80
|
+
* Directory of the materialized workflow package within the closure --
|
|
81
|
+
* the same directory `loadWorkflowDefinitionFromClosure` reads. Each
|
|
82
|
+
* declared plugin package is resolved from this package's laid-out
|
|
83
|
+
* `node_modules/`, exactly as the workflow entry's own bare-specifier
|
|
84
|
+
* imports resolve.
|
|
85
|
+
*/
|
|
86
|
+
readonly packageDir: string;
|
|
87
|
+
/**
|
|
88
|
+
* Plugin-package names the workflow's agents declare via
|
|
89
|
+
* `AgentDefinition.plugins` (`["@intx/tools-lsp"]`). Each MUST be a
|
|
90
|
+
* direct dependency of the workflow package so it is laid out under the
|
|
91
|
+
* workflow package's `node_modules/`. Empty is valid (no plugins).
|
|
92
|
+
*/
|
|
93
|
+
readonly plugins: readonly string[];
|
|
94
|
+
/** See `LoadWorkflowDefinitionFromClosureArgs.importCacheKey`. */
|
|
95
|
+
readonly importCacheKey?: string;
|
|
96
|
+
/** Test seam for dynamic import; see the definition loader's variant. */
|
|
97
|
+
readonly importModule?: (importUrl: string) => Promise<unknown>;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Import each declared plugin package's `interchange.tools` module from the
|
|
101
|
+
* materialized workflow closure and collect the `AnnotatedPluginFactory`
|
|
102
|
+
* values it exports. This is the run-child counterpart to the tool-package
|
|
103
|
+
* loader's plugin channel: a source-ref workflow contributes no plugin factory
|
|
104
|
+
* through its agent definition (a plugin has no agent slot), so the child
|
|
105
|
+
* materializes the declared plugins straight from the already-laid-out closure
|
|
106
|
+
* -- no re-download, no manifest -- and feeds them into the existing per-step
|
|
107
|
+
* plugin chain. The closure bytes were SRI-verified when the deploy applied the
|
|
108
|
+
* frozen closure, and resolution walks the same `node_modules/` graph the
|
|
109
|
+
* workflow entry's imports use.
|
|
110
|
+
*
|
|
111
|
+
* @throws if a declared plugin package cannot be resolved, declares no
|
|
112
|
+
* `interchange.tools` entry, the entry escapes the package, cannot be
|
|
113
|
+
* imported, or exports no `AnnotatedPluginFactory` value
|
|
114
|
+
*/
|
|
115
|
+
export declare function loadWorkflowPluginFactoriesFromClosure(args: LoadWorkflowPluginsFromClosureArgs): Promise<AnnotatedPluginFactory[]>;
|
|
116
|
+
/**
|
|
117
|
+
* Read the static tool `definitions` each declared plugin package
|
|
118
|
+
* contributes, keyed by plugin-package name, WITHOUT retaining the plugin
|
|
119
|
+
* factory (so the caller never instantiates a plugin, which for LSP would
|
|
120
|
+
* start a subprocess). This is the probe/capability-walk counterpart to
|
|
121
|
+
* `loadWorkflowPluginFactoriesFromClosure`: it loads the SAME plugin module
|
|
122
|
+
* from the SAME frozen closure so the tool grant surface the walk approves
|
|
123
|
+
* matches the plugin the run-child materializes.
|
|
124
|
+
*
|
|
125
|
+
* A plugin package that exports plugin factories but declares no tool
|
|
126
|
+
* definitions (a middleware-only plugin) maps to an empty array -- valid,
|
|
127
|
+
* it contributes no tool grant.
|
|
128
|
+
*
|
|
129
|
+
* @throws under the same conditions as `loadWorkflowPluginFactoriesFromClosure`
|
|
130
|
+
*/
|
|
131
|
+
export declare function loadWorkflowPluginToolDefinitionsFromClosure(args: LoadWorkflowPluginsFromClosureArgs): Promise<Map<string, readonly ToolDeclaration[]>>;
|