@intx/workflow-host 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -10
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +53 -53
- package/dist/adapters/spawn-child.d.ts +71 -42
- package/dist/adapters/spawn-child.js +83 -77
- package/dist/adapters/step-invoker.js +84 -7
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/index.d.ts +2 -1
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +107 -13
- package/dist/child/run-child.js +290 -108
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/conversation-text.d.ts +23 -0
- package/dist/conversation-text.js +56 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3 -2
- package/dist/ipc/control-channel.d.ts +58 -0
- package/dist/ipc/control-channel.js +94 -1
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/seams/scheduler.d.ts +4 -6
- package/dist/seams/scheduler.js +74 -93
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +1 -1
- package/dist/supervisor/credentials.d.ts +11 -10
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +5 -2
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +5 -5
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +82 -25
- package/dist/supervisor/supervisor.js +1313 -410
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +134 -0
- package/dist/supervisor/types.d.ts +150 -23
- package/dist/workflow-definition-loader.d.ts +131 -0
- package/dist/workflow-definition-loader.js +316 -0
- package/package.json +12 -11
package/README.md
CHANGED
|
@@ -95,7 +95,7 @@ The constructor argument shape:
|
|
|
95
95
|
`require.resolve` / `import.meta.resolve` against the host's own
|
|
96
96
|
package, `@intx/<host>`).
|
|
97
97
|
- `substrateEnv`, `workflowRunRepoId`, `workflowRunRef`,
|
|
98
|
-
`
|
|
98
|
+
`anchorRunId`, `deploymentMailAddress`, `readPrincipal`,
|
|
99
99
|
`deriveStepAddress`, `deriveStepRepoId?`, `ipcKeyPairFactory?` —
|
|
100
100
|
per-deployment configuration the supervisor needs in its closure
|
|
101
101
|
state.
|
|
@@ -144,9 +144,61 @@ signed shape as the operator and drain origins.
|
|
|
144
144
|
`shutdown()` unregisters the mail address, kills the child, and
|
|
145
145
|
disposes subscriptions.
|
|
146
146
|
|
|
147
|
-
`drain`
|
|
148
|
-
|
|
149
|
-
|
|
147
|
+
`drain(opts)` sends the drain control mail and waits for in-flight
|
|
148
|
+
runs to drain per each step's `drainBehavior`; on the drain-timeout it
|
|
149
|
+
escalates to a signed `CancelRequested{origin: "supervisor-drain"}`.
|
|
150
|
+
|
|
151
|
+
`recycle(opts)` tears the current child down and stands a fresh one up
|
|
152
|
+
against the SAME deploy tree (same materialized source closure, same
|
|
153
|
+
per-step credential repos). It is strictly orthogonal to redeploy,
|
|
154
|
+
which mints a new deploy tree. Operator, supervisor-policy (max-uptime
|
|
155
|
+
/ max-rss / grants-staleness), and workflow-process-self-initiated
|
|
156
|
+
origins all funnel through the same path.
|
|
157
|
+
|
|
158
|
+
### Respawn policy
|
|
159
|
+
|
|
160
|
+
An UNEXPECTED child exit — a crash, OOM, panic, or signal, as opposed
|
|
161
|
+
to a supervisor-initiated shutdown or recycle — is detected by watching
|
|
162
|
+
the child process's `exited`, not the IPC channel: a clean process
|
|
163
|
+
death ends the channel readers without a protocol-level crash callback,
|
|
164
|
+
so `exited` is the only universal death signal. The supervisor
|
|
165
|
+
classifies the exit by cohort generation and lifecycle phase — an exit
|
|
166
|
+
of the current running cohort that no planned teardown owns is
|
|
167
|
+
unexpected.
|
|
168
|
+
|
|
169
|
+
On an unexpected exit the supervisor, with no external intervention:
|
|
170
|
+
|
|
171
|
+
1. Replays any mail stranded mid-flight — entries the dead child's
|
|
172
|
+
in-flight dispatch left in the per-address `processing/` subtree —
|
|
173
|
+
back into `inbox/` under their original `<receivedAt>-<messageId>`
|
|
174
|
+
keys. Those keys sort ahead of any mail that arrived during the
|
|
175
|
+
kill/respawn gap, so the stranded entry is re-dispatched first and
|
|
176
|
+
FIFO ordering holds across the respawn boundary.
|
|
177
|
+
2. Spawns a fresh workflow-process child against the same deploy tree
|
|
178
|
+
(reusing the recycle path's respawn machinery) and resumes dispatch.
|
|
179
|
+
|
|
180
|
+
The respawn is bounded so a persistently-broken child cannot saturate
|
|
181
|
+
the host. Every bound is operator-overridable via
|
|
182
|
+
`WorkflowSupervisorBindings`; the defaults are:
|
|
183
|
+
|
|
184
|
+
- **Exponential backoff.** Each respawn waits before spawning, starting
|
|
185
|
+
at `respawnBackoffInitialMs` (1s) and doubling to a
|
|
186
|
+
`respawnBackoffMaxMs` (30s) cap.
|
|
187
|
+
- **Crash-loop guard.** If the child exits unexpectedly
|
|
188
|
+
`crashLoopMaxCount` (3) times within `crashLoopWindowMs` (60s), the
|
|
189
|
+
supervisor stops respawning and latches the deployment to a terminal
|
|
190
|
+
`crash-looping` state.
|
|
191
|
+
- **Stable-run reset.** Once a crash-respawned child stays up for
|
|
192
|
+
`crashLoopStableResetMs` (60s), the crash counter and the backoff
|
|
193
|
+
reset, so a flap followed by stability does not permanently latch.
|
|
194
|
+
|
|
195
|
+
`crash-looping` is an in-memory, per-process phase — no external reader
|
|
196
|
+
observes it. The durable, externally-queryable signal is the run's
|
|
197
|
+
status: on latch the supervisor (the sole writer of the workflow-run
|
|
198
|
+
repo) commits a `RunFailed` for the deployment's stable run, flipping
|
|
199
|
+
its `workflow_run.status` to `failed` through the same pack path every
|
|
200
|
+
other terminal run uses. External automation that watches run status
|
|
201
|
+
sees the crash-loop as a failed run.
|
|
150
202
|
|
|
151
203
|
### Host wiring
|
|
152
204
|
|
|
@@ -208,12 +260,6 @@ without reconstructing the env. The closure looks up the
|
|
|
208
260
|
originating step's grants by `stepId` and delegates to a
|
|
209
261
|
host-supplied `GrantEvaluator`.
|
|
210
262
|
|
|
211
|
-
### Placeholders
|
|
212
|
-
|
|
213
|
-
`DrainController` is a no-op placeholder in this commit; the real
|
|
214
|
-
controller lands separately. `recycle` is a no-op pending the
|
|
215
|
-
recycle path.
|
|
216
|
-
|
|
217
263
|
## Hosting the workflow-process child
|
|
218
264
|
|
|
219
265
|
`@intx/workflow-host` ships the runtime body
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Principal, RepoId, RepoStore as SubstrateRepoStore } from "@intx/hub-sessions/substrate";
|
|
2
|
-
import type { RepoStore } from "@intx/workflow";
|
|
2
|
+
import type { RepoStore, WorkflowEvent } from "@intx/workflow";
|
|
3
3
|
export type WorkflowRunRepoStoreOpts = {
|
|
4
4
|
/**
|
|
5
5
|
* Substrate handle the adapter reads from and writes to. The caller
|
|
@@ -23,6 +23,18 @@ export type WorkflowRunRepoStoreOpts = {
|
|
|
23
23
|
* shape the production wiring supplies.
|
|
24
24
|
*/
|
|
25
25
|
principal: Principal;
|
|
26
|
+
/**
|
|
27
|
+
* Principal used for a control-plane cancel append (a batch that is
|
|
28
|
+
* entirely `CancelRequested`). The workflow-run kind handler requires a
|
|
29
|
+
* `CancelRequested` be signed by a `supervisor` principal -- a
|
|
30
|
+
* `workflow-process` principal may write run-body events but not a cancel.
|
|
31
|
+
* An in-process child runs under real supervisor authority, so its host
|
|
32
|
+
* supplies a supervisor principal here while run-body events keep their
|
|
33
|
+
* `workflow-process` attribution. Absent when the writer issues no
|
|
34
|
+
* in-process cancel, in which case a cancel fails loud at the push boundary
|
|
35
|
+
* rather than being silently mis-attributed.
|
|
36
|
+
*/
|
|
37
|
+
controlPlanePrincipal?: Principal;
|
|
26
38
|
/**
|
|
27
39
|
* Events ref the adapter reads from and writes to. The workflow-run
|
|
28
40
|
* repo layout pins all `runs/<runId>/events/` blobs under a single
|
|
@@ -37,3 +49,12 @@ export type WorkflowRunRepoStoreOpts = {
|
|
|
37
49
|
* routing live in closure.
|
|
38
50
|
*/
|
|
39
51
|
export declare function createWorkflowRunRepoStore(opts: WorkflowRunRepoStoreOpts): RepoStore;
|
|
52
|
+
/**
|
|
53
|
+
* Translate a state-machine `WorkflowEvent` (using `kind` as the
|
|
54
|
+
* discriminator) into the on-disk envelope shape (`{seq, type,
|
|
55
|
+
* ...rest}`) the workflow-run kind handler validates and the
|
|
56
|
+
* substrate's `subscribeKind` helper filters on. Exported so the
|
|
57
|
+
* supervisor's terminal-commit path encodes a supervisor-authored
|
|
58
|
+
* `RunFailed` through the same single source of the on-disk shape.
|
|
59
|
+
*/
|
|
60
|
+
export declare function workflowEventToOnDisk(event: WorkflowEvent, seq: number): Record<string, unknown>;
|
|
@@ -61,6 +61,7 @@ const ALL_WORKFLOW_EVENT_TYPES = [
|
|
|
61
61
|
"AttemptScheduled",
|
|
62
62
|
"SignalAwaited",
|
|
63
63
|
"SignalReceived",
|
|
64
|
+
"SignalAwaitAbandoned",
|
|
64
65
|
"TimerSet",
|
|
65
66
|
"TimerFired",
|
|
66
67
|
"CancelRequested",
|
|
@@ -95,66 +96,60 @@ export function createWorkflowRunRepoStore(opts) {
|
|
|
95
96
|
};
|
|
96
97
|
}
|
|
97
98
|
async function readAllEventsForRun(opts, runId) {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
99
|
+
// Read the committed tree through the substrate, never the working
|
|
100
|
+
// checkout under `getRepoDir`. `openCommittedReads` pins the ref to its
|
|
101
|
+
// tip commit and serves every read from the git object store, so an
|
|
102
|
+
// enumerate-then-read sequence is a single coherent snapshot even while
|
|
103
|
+
// a concurrent append re-materializes the checkout. The prior
|
|
104
|
+
// implementation read the working tree directly (raw readdir/readFile)
|
|
105
|
+
// and raced that materialization: a blob `readdir` had just enumerated
|
|
106
|
+
// could vanish before `readFile` on a contended filesystem, surfacing a
|
|
107
|
+
// spurious ENOENT. Reading under the substrate's per-repo write lock was
|
|
108
|
+
// the alternative considered and rejected -- it would serialize every
|
|
109
|
+
// read behind the single writer and couple read latency to write
|
|
110
|
+
// contention, whereas the pinned committed tree is lock-free and already
|
|
111
|
+
// consistent because every append lands as exactly one commit.
|
|
112
|
+
const reads = await opts.substrate.openCommittedReads(opts.principal, opts.repoId, opts.ref);
|
|
113
|
+
// Null mirrors the prior readdir-ENOENT contract: an uninitialised repo
|
|
114
|
+
// or an unresolved ref holds no runs at all.
|
|
115
|
+
if (reads === null)
|
|
116
|
+
return [];
|
|
117
|
+
const runDir = `${RUNS_PREFIX}/${runId}`;
|
|
118
|
+
const runChildren = await reads.listDir(runDir);
|
|
119
|
+
const decoder = new TextDecoder();
|
|
102
120
|
const entries = [];
|
|
103
121
|
// A terminated run is sealed into a single combined `events.jsonl`; an
|
|
104
|
-
// in-flight run keeps per-event `events/<seq>.json` files. The
|
|
105
|
-
//
|
|
106
|
-
//
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
catch (cause) {
|
|
112
|
-
if (!isErrnoNotFound(cause))
|
|
113
|
-
throw cause;
|
|
114
|
-
combinedRaw = null;
|
|
115
|
-
}
|
|
116
|
-
if (combinedRaw !== null) {
|
|
117
|
-
// The two forms are mutually exclusive; a run carrying both is a
|
|
118
|
-
// botched seal, and silently reading only the combined file would
|
|
119
|
-
// mask it, so surface it instead.
|
|
120
|
-
let perEventPresent = false;
|
|
121
|
-
try {
|
|
122
|
-
await fs.access(path.join(runDir, EVENTS_DIR));
|
|
123
|
-
perEventPresent = true;
|
|
124
|
-
}
|
|
125
|
-
catch (cause) {
|
|
126
|
-
if (!isErrnoNotFound(cause))
|
|
127
|
-
throw cause;
|
|
128
|
-
}
|
|
129
|
-
if (perEventPresent) {
|
|
122
|
+
// in-flight run keeps per-event `events/<seq>.json` files. The two forms
|
|
123
|
+
// are mutually exclusive; a run carrying both is a botched seal, and
|
|
124
|
+
// silently reading only the combined file would mask it, so surface it.
|
|
125
|
+
const combined = runChildren.find((e) => e.type === "blob" && e.name === WORKFLOW_RUN_EVENTS_FILE);
|
|
126
|
+
const perEventDir = runChildren.find((e) => e.type === "tree" && e.name === EVENTS_DIR);
|
|
127
|
+
if (combined !== undefined) {
|
|
128
|
+
if (perEventDir !== undefined) {
|
|
130
129
|
throw new Error(`workflow-runtime: run ${runId} carries both a combined ${WORKFLOW_RUN_EVENTS_FILE} and a per-event ${EVENTS_DIR}/ directory`);
|
|
131
130
|
}
|
|
131
|
+
const combinedRaw = decoder.decode(await reads.readBlobByOid(combined.oid));
|
|
132
132
|
for (const line of splitCombinedEventLog(combinedRaw)) {
|
|
133
133
|
entries.push(parseEventEnvelope(line, `${opts.repoId.id}/${runId}/${WORKFLOW_RUN_EVENTS_FILE}`));
|
|
134
134
|
}
|
|
135
135
|
entries.sort((a, b) => a.seq - b.seq);
|
|
136
136
|
return entries.map((e) => e.event);
|
|
137
137
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
return [];
|
|
146
|
-
throw cause;
|
|
147
|
-
}
|
|
148
|
-
for (const name of filenames) {
|
|
149
|
-
const match = EVENT_FILENAME_RE.exec(name);
|
|
138
|
+
// Per-event form. `listDir` on an absent or non-tree path returns the
|
|
139
|
+
// empty array, so a run with no events reads as the empty log.
|
|
140
|
+
const eventBlobs = await reads.listDir(`${runDir}/${EVENTS_DIR}`);
|
|
141
|
+
for (const child of eventBlobs) {
|
|
142
|
+
if (child.type !== "blob")
|
|
143
|
+
continue;
|
|
144
|
+
const match = EVENT_FILENAME_RE.exec(child.name);
|
|
150
145
|
if (match === null)
|
|
151
146
|
continue;
|
|
152
147
|
const seqStr = match[1];
|
|
153
148
|
if (seqStr === undefined)
|
|
154
149
|
continue;
|
|
155
150
|
const seqFromName = Number.parseInt(seqStr, 10);
|
|
156
|
-
const raw = await
|
|
157
|
-
const source = `${opts.repoId.id}/${runId}/${EVENTS_DIR}/${name}`;
|
|
151
|
+
const raw = decoder.decode(await reads.readBlobByOid(child.oid));
|
|
152
|
+
const source = `${opts.repoId.id}/${runId}/${EVENTS_DIR}/${child.name}`;
|
|
158
153
|
const entry = parseEventEnvelope(raw, source);
|
|
159
154
|
if (entry.seq !== seqFromName) {
|
|
160
155
|
throw new Error(`workflow-runtime: read ${source} body.seq ${String(entry.seq)} does not match filename seq ${String(seqFromName)}`);
|
|
@@ -213,9 +208,11 @@ function onDiskToWorkflowEvent(envelope) {
|
|
|
213
208
|
* Translate a state-machine `WorkflowEvent` (using `kind` as the
|
|
214
209
|
* discriminator) into the on-disk envelope shape (`{seq, type,
|
|
215
210
|
* ...rest}`) the workflow-run kind handler validates and the
|
|
216
|
-
* substrate's `subscribeKind` helper filters on.
|
|
211
|
+
* substrate's `subscribeKind` helper filters on. Exported so the
|
|
212
|
+
* supervisor's terminal-commit path encodes a supervisor-authored
|
|
213
|
+
* `RunFailed` through the same single source of the on-disk shape.
|
|
217
214
|
*/
|
|
218
|
-
function workflowEventToOnDisk(event, seq) {
|
|
215
|
+
export function workflowEventToOnDisk(event, seq) {
|
|
219
216
|
const { kind, seq: _eventSeq, ...rest } = event;
|
|
220
217
|
return { seq, type: kind, ...rest };
|
|
221
218
|
}
|
|
@@ -241,9 +238,18 @@ async function appendBatchEvents(opts, runId, events) {
|
|
|
241
238
|
const lastEvent = events[events.length - 1];
|
|
242
239
|
if (lastEvent === undefined)
|
|
243
240
|
throw new Error("unreachable");
|
|
241
|
+
// A control-plane cancel append (an isolated `CancelRequested`, which the
|
|
242
|
+
// kind handler requires be signed by a supervisor principal) is written under
|
|
243
|
+
// `controlPlanePrincipal` when the host supplied one; every other batch --
|
|
244
|
+
// including run-body events -- keeps the workflow-process `principal`. A mixed
|
|
245
|
+
// batch is never a cancel, so it stays on `principal`.
|
|
246
|
+
const principal = opts.controlPlanePrincipal !== undefined &&
|
|
247
|
+
events.every((event) => event.kind === "CancelRequested")
|
|
248
|
+
? opts.controlPlanePrincipal
|
|
249
|
+
: opts.principal;
|
|
244
250
|
let seqConflict = null;
|
|
245
251
|
try {
|
|
246
|
-
await opts.substrate.writeTreePreservingPrefix(
|
|
252
|
+
await opts.substrate.writeTreePreservingPrefix(principal, opts.repoId, opts.ref, {
|
|
247
253
|
preservePrefix: prefix,
|
|
248
254
|
merge: async (existing) => {
|
|
249
255
|
// The runtime body emits events at `state.lastSeq + 1` and
|
|
@@ -336,9 +342,3 @@ async function* subscribeRun(opts, runId, subOpts) {
|
|
|
336
342
|
yield { seq: entry.event.seq, event };
|
|
337
343
|
}
|
|
338
344
|
}
|
|
339
|
-
function isErrnoNotFound(cause) {
|
|
340
|
-
if (cause === null || typeof cause !== "object")
|
|
341
|
-
return false;
|
|
342
|
-
const code = cause.code;
|
|
343
|
-
return code === "ENOENT";
|
|
344
|
-
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { SpawnChildWorkflow, WorkflowDefinition } from "@intx/workflow";
|
|
1
|
+
import type { InferenceEvent } from "@intx/types/runtime";
|
|
2
|
+
import type { SpawnChildWorkflow, SpawnSuspendableChild, SuspendableChildHandle, WorkflowDefinition, WorkflowEvent } from "@intx/workflow";
|
|
3
3
|
/**
|
|
4
4
|
* The terminal-status shape the runtime body expects back from a
|
|
5
5
|
* spawn. Mirrored from `SpawnChildWorkflow`'s return type so the
|
|
@@ -29,46 +29,75 @@ export type RunChildWorkflow = (input: {
|
|
|
29
29
|
}) => Promise<{
|
|
30
30
|
terminalStatus: ChildTerminalStatus;
|
|
31
31
|
}>;
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
* read path.
|
|
49
|
-
*/
|
|
50
|
-
principal: Principal;
|
|
51
|
-
/**
|
|
52
|
-
* Ref under the workflow asset's repo whose tree holds the
|
|
53
|
-
* deployed `workflow.json`. Callers typically supply
|
|
54
|
-
* `"refs/heads/main"` -- the workflow-kind handler enforces the
|
|
55
|
-
* envelope's structural shape at push time so a deploy ref read
|
|
56
|
-
* here either yields a valid envelope or surfaces a targeted
|
|
57
|
-
* parse/validation error.
|
|
58
|
-
*/
|
|
59
|
-
deployRef: string;
|
|
60
|
-
/**
|
|
61
|
-
* Runtime-supplied child execution callback. The adapter delegates
|
|
62
|
-
* here once the `WorkflowDefinition` is resolved; the supervisor
|
|
63
|
-
* owns the child `WorkflowRuntimeEnv` and the `runtimeRun`
|
|
64
|
-
* invocation.
|
|
65
|
-
*/
|
|
32
|
+
/**
|
|
33
|
+
* Construct the terminal `WorkflowRuntimeEnv.SpawnChildWorkflow` adapter for an
|
|
34
|
+
* owned childWorkflow import. The child re-evaluated the whole pinned closure
|
|
35
|
+
* and lifted every inline child to an internal `{ ref }`, so the child
|
|
36
|
+
* definitions are in hand and already covered by the parent's re-verify.
|
|
37
|
+
* Resolve each `definitionRef`
|
|
38
|
+
* from the in-memory `bodies` map and delegate to the runtime-supplied
|
|
39
|
+
* `runChild`, with NO on-disk round-trip and NO separate per-child re-verify:
|
|
40
|
+
* materializing the child back out and re-fingerprinting it would round-trip
|
|
41
|
+
* trusted-in-hand data for no gain, and the closure re-eval on restart
|
|
42
|
+
* re-derives the same bodies durably. Mirrors
|
|
43
|
+
* {@link createInMemorySpawnSuspendableChild} but drives the child terminal-only
|
|
44
|
+
* (await its terminal status) rather than across approval parks.
|
|
45
|
+
*/
|
|
46
|
+
export declare function createInMemorySpawnChild(opts: {
|
|
47
|
+
bodies: ReadonlyMap<string, WorkflowDefinition>;
|
|
66
48
|
runChild: RunChildWorkflow;
|
|
67
|
-
}
|
|
49
|
+
}): SpawnChildWorkflow;
|
|
50
|
+
/**
|
|
51
|
+
* Runtime-supplied suspendable child execution callback. The park-aware
|
|
52
|
+
* analog of {@link RunChildWorkflow}: the supervisor owns the child
|
|
53
|
+
* `WorkflowRuntimeEnv` construction and the `runtimeRun` invocation and
|
|
54
|
+
* returns a live `SuspendableChildHandle` the caller drives across the
|
|
55
|
+
* body's approval parks, rather than awaiting a terminal. The adapter is
|
|
56
|
+
* the single resolution point that hands the supervisor a concrete
|
|
57
|
+
* `WorkflowDefinition` alongside the parent attribution the runtime body
|
|
58
|
+
* produced.
|
|
59
|
+
*/
|
|
60
|
+
export type RunSuspendableChild = (input: {
|
|
61
|
+
definition: WorkflowDefinition;
|
|
62
|
+
definitionRef: string;
|
|
63
|
+
childRunId: string;
|
|
64
|
+
input: unknown;
|
|
65
|
+
parentRunId: string;
|
|
66
|
+
parentStepId: string;
|
|
67
|
+
signal: AbortSignal;
|
|
68
|
+
resumeFromEvents?: readonly WorkflowEvent[];
|
|
69
|
+
},
|
|
70
|
+
/**
|
|
71
|
+
* Live inference-event sink for the child's agent steps. Threaded from the
|
|
72
|
+
* host's per-run funnel (the parent run's event-channel closure) so the
|
|
73
|
+
* body's inference events reach the hub's live stream instead of being
|
|
74
|
+
* silently dropped. Per-run durable attribution is unaffected -- the child
|
|
75
|
+
* runtime commits its events under `runs/<childRunId>/events/` regardless.
|
|
76
|
+
*/
|
|
77
|
+
onEvent: (event: InferenceEvent) => void) => Promise<SuspendableChildHandle>;
|
|
78
|
+
/**
|
|
79
|
+
* Host-side widening of the runtime {@link SpawnSuspendableChild} contract: the
|
|
80
|
+
* same input plus the per-run `onEvent` sink the host injects. The runtime
|
|
81
|
+
* calls the narrow `SpawnSuspendableChild` (no event slot); the host binding
|
|
82
|
+
* wired into the runtime env closes over the run's funnel and forwards it here,
|
|
83
|
+
* mirroring how `ChildStepInvoker` widens the runtime `StepInvoker` with
|
|
84
|
+
* `onEvent`. The runtime contract in `@intx/workflow` stays untouched.
|
|
85
|
+
*/
|
|
86
|
+
export type HostSpawnSuspendableChild = (input: Parameters<SpawnSuspendableChild>[0], onEvent: (event: InferenceEvent) => void) => ReturnType<SpawnSuspendableChild>;
|
|
68
87
|
/**
|
|
69
|
-
* Construct the
|
|
70
|
-
*
|
|
71
|
-
* the
|
|
72
|
-
*
|
|
88
|
+
* Construct the `WorkflowRuntimeEnv.SpawnSuspendableChild` adapter for the
|
|
89
|
+
* source-ref (code-sourced) path -- the only deploy lineage. The parent child
|
|
90
|
+
* re-evaluated the whole pinned closure in one sandbox and re-verified it
|
|
91
|
+
* against the approved hash -- which already covers every inline onTrigger body
|
|
92
|
+
* -- so the body definitions are in hand and already proven. Resolve each
|
|
93
|
+
* `definitionRef` from that in-memory `bodies` map and run it in-process, with
|
|
94
|
+
* NO disk round-trip and NO separate per-body re-verify: materializing the body
|
|
95
|
+
* back out and re-fingerprinting it would round-trip trusted-in-hand data for no
|
|
96
|
+
* gain, and the closure re-eval on restart re-derives the same bodies durably.
|
|
97
|
+
* The body still runs in the parent's sandbox (in-process today; a stricter
|
|
98
|
+
* per-body boundary is the deferred, opt-in SandboxBoundary case).
|
|
73
99
|
*/
|
|
74
|
-
export declare function
|
|
100
|
+
export declare function createInMemorySpawnSuspendableChild(opts: {
|
|
101
|
+
bodies: ReadonlyMap<string, WorkflowDefinition>;
|
|
102
|
+
runSuspendableChild: RunSuspendableChild;
|
|
103
|
+
}): HostSpawnSuspendableChild;
|
|
@@ -1,34 +1,34 @@
|
|
|
1
1
|
// Production `WorkflowRuntimeEnv.SpawnChildWorkflow` adapter.
|
|
2
2
|
//
|
|
3
|
-
// The runtime body sees the spawn callback shape: given a
|
|
4
|
-
//
|
|
5
|
-
// `childRunId`, the materialized child
|
|
6
|
-
// settle once the child run reaches a terminal
|
|
7
|
-
// itself does not execute the child workflow -- it resolves
|
|
8
|
-
// `definitionRef` into a concrete `WorkflowDefinition`
|
|
9
|
-
//
|
|
10
|
-
// runtime-supplied `runChild` callback. The supervisor wires the
|
|
3
|
+
// The runtime body sees the spawn callback shape: given a `definitionRef`
|
|
4
|
+
// (the internal ref the deploy step assigned when it lifted the authored
|
|
5
|
+
// inline child), a parent-allocated `childRunId`, the materialized child
|
|
6
|
+
// input, and parent attribution, settle once the child run reaches a terminal
|
|
7
|
+
// phase. The adapter itself does not execute the child workflow -- it resolves
|
|
8
|
+
// the `definitionRef` into a concrete `WorkflowDefinition` and delegates the
|
|
9
|
+
// spawn to a runtime-supplied `runChild` callback. The supervisor wires the
|
|
11
10
|
// callback against a child `WorkflowRuntimeEnv` and `runtimeRun`.
|
|
12
11
|
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
12
|
+
// Two spawn types with DIFFERENT trust structures resolve here, so they
|
|
13
|
+
// take different resolution paths -- not one shared resolver that pretends
|
|
14
|
+
// they are the same:
|
|
15
|
+
//
|
|
16
|
+
// - onTrigger BODY (the suspendable adapter): a body is a section
|
|
17
|
+
// extracted from the PARENT's own approved definition. Source-ref is the
|
|
18
|
+
// only deploy lineage, so the body is resolved in-memory from the parent's
|
|
19
|
+
// re-evaluated closure (`createInMemorySpawnSuspendableChild`), already
|
|
20
|
+
// covered by the parent's re-verify -- no separate on-disk read and no
|
|
21
|
+
// separate per-body re-verify.
|
|
22
|
+
//
|
|
23
|
+
// - childWorkflow (the terminal adapter): an owned import embedded inline in
|
|
24
|
+
// the parent's definition. It is lifted to an internal `{ ref }` at child
|
|
25
|
+
// boot and resolved in-memory from the parent's closure map
|
|
26
|
+
// (`createInMemorySpawnChild`) -- exactly like a source-ref onTrigger
|
|
27
|
+
// body, with NO on-disk asset and NO separate per-child re-verify (the
|
|
28
|
+
// parent's re-verify already covers it, since the inline child rides the
|
|
29
|
+
// parent's hashed projection). The terminal-only drive (await the child's
|
|
30
|
+
// terminal, no park) is the only thing that distinguishes it from the
|
|
31
|
+
// suspendable body adapter.
|
|
32
32
|
//
|
|
33
33
|
// Drain coordination is handled by the supervisor's drain primitive
|
|
34
34
|
// (`packages/workflow-host/src/supervisor`), not by this adapter. The
|
|
@@ -59,21 +59,31 @@
|
|
|
59
59
|
// `runChild` does -- but the callback's input shape (`{ definition,
|
|
60
60
|
// childRunId, ... }`) is the seam that makes the scoping unambiguous
|
|
61
61
|
// at the boundary.
|
|
62
|
-
import { type } from "arktype";
|
|
63
|
-
import { workflowDefinitionEnvelopeSchema } from "@intx/hub-sessions/substrate";
|
|
64
|
-
const WORKFLOW_JSON_PATH = "workflow.json";
|
|
65
62
|
/**
|
|
66
|
-
* Construct the
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
63
|
+
* Construct the terminal `WorkflowRuntimeEnv.SpawnChildWorkflow` adapter for an
|
|
64
|
+
* owned childWorkflow import. The child re-evaluated the whole pinned closure
|
|
65
|
+
* and lifted every inline child to an internal `{ ref }`, so the child
|
|
66
|
+
* definitions are in hand and already covered by the parent's re-verify.
|
|
67
|
+
* Resolve each `definitionRef`
|
|
68
|
+
* from the in-memory `bodies` map and delegate to the runtime-supplied
|
|
69
|
+
* `runChild`, with NO on-disk round-trip and NO separate per-child re-verify:
|
|
70
|
+
* materializing the child back out and re-fingerprinting it would round-trip
|
|
71
|
+
* trusted-in-hand data for no gain, and the closure re-eval on restart
|
|
72
|
+
* re-derives the same bodies durably. Mirrors
|
|
73
|
+
* {@link createInMemorySpawnSuspendableChild} but drives the child terminal-only
|
|
74
|
+
* (await its terminal status) rather than across approval parks.
|
|
70
75
|
*/
|
|
71
|
-
export function
|
|
76
|
+
export function createInMemorySpawnChild(opts) {
|
|
72
77
|
return async ({ definitionRef, childRunId, input, parentRunId, parentStepId, signal, }) => {
|
|
73
78
|
if (signal.aborted) {
|
|
74
79
|
throw abortError(signal);
|
|
75
80
|
}
|
|
76
|
-
const definition =
|
|
81
|
+
const definition = opts.bodies.get(definitionRef);
|
|
82
|
+
if (definition === undefined) {
|
|
83
|
+
throw new Error(`workflow-runtime: spawn-child has no in-memory childWorkflow ` +
|
|
84
|
+
`definition for ${JSON.stringify(definitionRef)}; the parent's ` +
|
|
85
|
+
`closure should have lifted every inline child`);
|
|
86
|
+
}
|
|
77
87
|
// Re-check the abort signal after the resolution await. The
|
|
78
88
|
// caller can fire `signal.abort()` between the entry-time check
|
|
79
89
|
// and here; without this re-check the child callback would be
|
|
@@ -95,42 +105,44 @@ export function createWorkflowSpawnChild(opts) {
|
|
|
95
105
|
return { terminalStatus: result.terminalStatus };
|
|
96
106
|
};
|
|
97
107
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
108
|
+
/**
|
|
109
|
+
* Construct the `WorkflowRuntimeEnv.SpawnSuspendableChild` adapter for the
|
|
110
|
+
* source-ref (code-sourced) path -- the only deploy lineage. The parent child
|
|
111
|
+
* re-evaluated the whole pinned closure in one sandbox and re-verified it
|
|
112
|
+
* against the approved hash -- which already covers every inline onTrigger body
|
|
113
|
+
* -- so the body definitions are in hand and already proven. Resolve each
|
|
114
|
+
* `definitionRef` from that in-memory `bodies` map and run it in-process, with
|
|
115
|
+
* NO disk round-trip and NO separate per-body re-verify: materializing the body
|
|
116
|
+
* back out and re-fingerprinting it would round-trip trusted-in-hand data for no
|
|
117
|
+
* gain, and the closure re-eval on restart re-derives the same bodies durably.
|
|
118
|
+
* The body still runs in the parent's sandbox (in-process today; a stricter
|
|
119
|
+
* per-body boundary is the deferred, opt-in SandboxBoundary case).
|
|
120
|
+
*/
|
|
121
|
+
export function createInMemorySpawnSuspendableChild(opts) {
|
|
122
|
+
return async ({ definitionRef, childRunId, input, parentRunId, parentStepId, signal, resumeFromEvents, }, onEvent) => {
|
|
123
|
+
if (signal.aborted) {
|
|
124
|
+
throw abortError(signal);
|
|
125
|
+
}
|
|
126
|
+
const definition = opts.bodies.get(definitionRef);
|
|
127
|
+
if (definition === undefined) {
|
|
128
|
+
throw new Error(`workflow-runtime: source-ref spawn-child has no in-memory onTrigger ` +
|
|
129
|
+
`body for ${JSON.stringify(definitionRef)}; the parent's closure ` +
|
|
130
|
+
`re-eval should have extracted every inline body`);
|
|
131
|
+
}
|
|
132
|
+
if (signal.aborted) {
|
|
133
|
+
throw abortError(signal);
|
|
111
134
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
throw new Error(`workflow-runtime: spawn-child ${WORKFLOW_JSON_PATH} for ${repoId.kind}/${repoId.id} on ${opts.deployRef} failed envelope validation: ${validated.summary}`);
|
|
124
|
-
}
|
|
125
|
-
// The envelope schema enforces the structural shape the workflow
|
|
126
|
-
// body and state machine consume; the discriminated narrow over
|
|
127
|
-
// every `Primitive` variant lives downstream (the runtime body
|
|
128
|
-
// walks the steps and dispatches per-kind). Re-deriving the
|
|
129
|
-
// primitive narrow here would duplicate `defineWorkflow`'s
|
|
130
|
-
// validation, and the workflow-kind handler already enforced the
|
|
131
|
-
// same envelope at push time.
|
|
132
|
-
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- WorkflowDefinition's primitive union is narrowed downstream by the runtime body; the envelope schema enforces the structural shape this adapter cares about
|
|
133
|
-
return validated;
|
|
135
|
+
return opts.runSuspendableChild({
|
|
136
|
+
definition,
|
|
137
|
+
definitionRef,
|
|
138
|
+
childRunId,
|
|
139
|
+
input,
|
|
140
|
+
parentRunId,
|
|
141
|
+
parentStepId,
|
|
142
|
+
signal,
|
|
143
|
+
...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}),
|
|
144
|
+
}, onEvent);
|
|
145
|
+
};
|
|
134
146
|
}
|
|
135
147
|
/**
|
|
136
148
|
* Construct the rejection used when `signal.aborted` short-circuits.
|
|
@@ -144,9 +156,3 @@ function abortError(signal) {
|
|
|
144
156
|
return reason;
|
|
145
157
|
return new DOMException("aborted", "AbortError");
|
|
146
158
|
}
|
|
147
|
-
function isErrnoNotFound(cause) {
|
|
148
|
-
if (cause === null || typeof cause !== "object")
|
|
149
|
-
return false;
|
|
150
|
-
const code = cause.code;
|
|
151
|
-
return code === "ENOENT";
|
|
152
|
-
}
|