@intx/workflow-host 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -10
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +53 -53
- package/dist/adapters/spawn-child.d.ts +71 -42
- package/dist/adapters/spawn-child.js +83 -77
- package/dist/adapters/step-invoker.js +84 -7
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/index.d.ts +2 -1
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +107 -13
- package/dist/child/run-child.js +290 -108
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/conversation-text.d.ts +23 -0
- package/dist/conversation-text.js +56 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3 -2
- package/dist/ipc/control-channel.d.ts +58 -0
- package/dist/ipc/control-channel.js +94 -1
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/seams/scheduler.d.ts +4 -6
- package/dist/seams/scheduler.js +74 -93
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +1 -1
- package/dist/supervisor/credentials.d.ts +11 -10
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +5 -2
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +5 -5
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +82 -25
- package/dist/supervisor/supervisor.js +1313 -410
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +134 -0
- package/dist/supervisor/types.d.ts +150 -23
- package/dist/workflow-definition-loader.d.ts +131 -0
- package/dist/workflow-definition-loader.js +316 -0
- package/package.json +12 -11
package/dist/child/run-child.js
CHANGED
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
// each one whose log lacks a terminal event.
|
|
26
26
|
// 4. Emit `ready` on the control channel.
|
|
27
27
|
// 5. Loop on control-channel frames:
|
|
28
|
-
// - `trigger.fired` ->
|
|
28
|
+
// - `trigger.fired` -> first-fire the deployment's top-level run via
|
|
29
|
+
// `runtimeRun` (the supervisor only sends this for an absent log).
|
|
29
30
|
// - `grants-updated` -> replace the credentialsSnapshot.
|
|
30
31
|
// - `drain` -> forward to the drain controller (no-op here).
|
|
31
32
|
// - `shutdown` -> stop accepting new triggers and exit the
|
|
@@ -47,24 +48,26 @@
|
|
|
47
48
|
// classifies each in-flight step as cancel-mode or wait-mode. The
|
|
48
49
|
// supervisor's recycle policy is OS-driven (drain, SIGTERM, SIGKILL,
|
|
49
50
|
// respawn) and does not require a child-side control frame.
|
|
50
|
-
import { type } from "arktype";
|
|
51
51
|
import { getLogger } from "@intx/log";
|
|
52
52
|
import { generateKeyPair } from "@intx/crypto";
|
|
53
53
|
import { base64Decode, hexEncode } from "@intx/types";
|
|
54
|
-
import { readProcessingEntry
|
|
55
|
-
import {
|
|
56
|
-
import {
|
|
57
|
-
import { emptyState, runtimeRun } from "@intx/workflow";
|
|
54
|
+
import { readProcessingEntry } from "@intx/hub-sessions/substrate";
|
|
55
|
+
import { rewriteInlineOnTriggerBodies, rewriteInlineChildWorkflowBodies, } from "@intx/workflow";
|
|
56
|
+
import { baseStepId, emptyState, runtimeRun } from "@intx/workflow";
|
|
58
57
|
import { createWorkflowHostDrainController, } from "../drain-controller.js";
|
|
59
58
|
import { createWorkflowRunRepoStore } from "../adapters/repo-store.js";
|
|
60
59
|
import { createWorkflowRunBlobSubstrate } from "../adapters/blob-substrate.js";
|
|
60
|
+
import { createInMemorySpawnSuspendableChild, createInMemorySpawnChild, } from "../adapters/spawn-child.js";
|
|
61
61
|
import { createControlChannelSender, createEventChannelSender, receiveControlChannel, } from "../ipc/index.js";
|
|
62
62
|
import { createWorkflowHostSignalChannel } from "../seams/signal-channel.js";
|
|
63
|
+
import { extractConversationText } from "../conversation-text.js";
|
|
63
64
|
import { hashGrants } from "../supervisor/credentials.js";
|
|
65
|
+
import { loadVerifiedWorkflowDefinitionFromClosure } from "./verified-definition-loader.js";
|
|
66
|
+
import { loadWorkflowDirectorRegistryFromClosure } from "../workflow-definition-loader.js";
|
|
64
67
|
import { discoverInFlightRuns } from "./self-discovery.js";
|
|
68
|
+
import { collectParkedApprovalCorrelations, } from "./parked-correlations.js";
|
|
65
69
|
import { createWarmAgentCache } from "./warm-agent-cache.js";
|
|
66
70
|
const logger = getLogger(["workflow-host", "child"]);
|
|
67
|
-
const WORKFLOW_JSON_PATH = "workflow.json";
|
|
68
71
|
export function createCredentialsBackedAuthorize(ref, evaluate) {
|
|
69
72
|
return async (resource, action, ctx) => {
|
|
70
73
|
const stepId = ctx?.stepId;
|
|
@@ -75,9 +78,17 @@ export function createCredentialsBackedAuthorize(ref, evaluate) {
|
|
|
75
78
|
if (snapshot === null) {
|
|
76
79
|
throw new Error("workflow-child authorize: no credentialsSnapshot active; the supervisor must push one before any step runs");
|
|
77
80
|
}
|
|
78
|
-
|
|
81
|
+
// The credentials snapshot is keyed per base step; a map iteration's
|
|
82
|
+
// scoped id `<base>[<index>]` resolves to its base entry so every
|
|
83
|
+
// iteration shares the base step's grants. `baseStepId` is the identity
|
|
84
|
+
// on an unscoped id, so a plain step is unaffected.
|
|
85
|
+
const lookupStepId = baseStepId(stepId);
|
|
86
|
+
const entry = snapshot.steps.find((s) => s.stepId === lookupStepId);
|
|
79
87
|
if (entry === undefined) {
|
|
80
|
-
|
|
88
|
+
const scopedNote = lookupStepId === stepId
|
|
89
|
+
? ""
|
|
90
|
+
: ` (normalized from scoped invocation id ${stepId})`;
|
|
91
|
+
throw new Error(`workflow-child authorize: credentialsSnapshot has no entry for stepId ${lookupStepId}${scopedNote}`);
|
|
81
92
|
}
|
|
82
93
|
return evaluate({
|
|
83
94
|
resource,
|
|
@@ -101,7 +112,28 @@ export async function runWorkflowChild(opts) {
|
|
|
101
112
|
const sourcesRef = {
|
|
102
113
|
current: opts.bindings.initialSources ?? {},
|
|
103
114
|
};
|
|
104
|
-
const
|
|
115
|
+
const credentialMaterialRef = {
|
|
116
|
+
current: opts.bindings.initialCredentialMaterial ?? null,
|
|
117
|
+
};
|
|
118
|
+
// The per-run credential wiring the top-level step invoker carries to the
|
|
119
|
+
// substrate: the live material cell and a resolver for a step's grants from
|
|
120
|
+
// the same credentials snapshot `authorize` reads. Built once over the two
|
|
121
|
+
// refs; every step build reads them live, so a rotation -- or a revoking
|
|
122
|
+
// re-push that swaps a ref -- is reflected without rebuilding the wiring.
|
|
123
|
+
const credentialWiring = {
|
|
124
|
+
materialRef: credentialMaterialRef,
|
|
125
|
+
resolveStepGrants: (stepId) => {
|
|
126
|
+
const snapshot = credentialsRef.current;
|
|
127
|
+
if (snapshot === null) {
|
|
128
|
+
throw new Error(`workflow-child credential wiring: no credentials snapshot for step ${stepId}; a tool-bearing step cannot resolve its grants before the run carries any`);
|
|
129
|
+
}
|
|
130
|
+
const entry = snapshot.steps.find((step) => step.stepId === baseStepId(stepId));
|
|
131
|
+
if (entry === undefined) {
|
|
132
|
+
throw new Error(`workflow-child credential wiring: credentials snapshot has no entry for step ${baseStepId(stepId)}`);
|
|
133
|
+
}
|
|
134
|
+
return entry.grants;
|
|
135
|
+
},
|
|
136
|
+
};
|
|
105
137
|
const clock = opts.bindings.clock ?? defaultClock;
|
|
106
138
|
const newId = opts.bindings.newId ?? defaultNewId;
|
|
107
139
|
// Mint the child's own upstream-signing keypair. The private half
|
|
@@ -120,7 +152,103 @@ export async function runWorkflowChild(opts) {
|
|
|
120
152
|
channelId: opts.env.channelId,
|
|
121
153
|
writer: opts.eventWriter,
|
|
122
154
|
});
|
|
123
|
-
|
|
155
|
+
// Re-verify barrier at the load boundary. Source-ref is the only deploy
|
|
156
|
+
// lineage: the inert projection is a non-executable approval surface (agents
|
|
157
|
+
// carry `modelSources`/no `inference`, tool factories are plain data), so the
|
|
158
|
+
// child EVALUATES the pinned code closure to a live definition and re-verifies
|
|
159
|
+
// by projecting it back to inert and hashing (`computeLiveDefinitionHash`)
|
|
160
|
+
// against `opts.env.definitionHash`; a divergent closure fails closed. The
|
|
161
|
+
// load happens once before both the resume loop and the trigger loop, so the
|
|
162
|
+
// same verified definition serves every fresh trigger AND every resume.
|
|
163
|
+
//
|
|
164
|
+
// Post-verify structural rewrite: the re-verify above hashed the closure's
|
|
165
|
+
// INLINE onTrigger bodies (matching the frozen approval); now lift each to a
|
|
166
|
+
// `{ ref }` so the runtime dispatches to the body child, and keep the
|
|
167
|
+
// extracted body definitions in an in-memory map. The suspendable-child
|
|
168
|
+
// resolver runs each body from THIS map -- the parent's already-re-verified
|
|
169
|
+
// closure -- with no disk read and no separate per-body re-verify. The rewrite
|
|
170
|
+
// MUST follow the re-verify: rewriting first would diverge from the frozen
|
|
171
|
+
// inline-body hash.
|
|
172
|
+
const verifiedDefinition = await loadVerifiedWorkflowDefinitionFromClosure({
|
|
173
|
+
packageDir: opts.env.closurePackageDir,
|
|
174
|
+
approvedHash: opts.env.definitionHash,
|
|
175
|
+
});
|
|
176
|
+
const { workflow, bodies } = rewriteInlineOnTriggerBodies(verifiedDefinition);
|
|
177
|
+
let definition = workflow;
|
|
178
|
+
const bodiesMap = new Map(bodies.map((b) => [b.ref, b.definition]));
|
|
179
|
+
// An owned `childWorkflow` import embeds its child inline in the parent's
|
|
180
|
+
// definition (folded into the parent's hash and approval), so it is already
|
|
181
|
+
// covered by the re-verify above. Lift each inline child to an internal
|
|
182
|
+
// `{ ref }` -- the form the runtime dispatches -- and keep the lifted
|
|
183
|
+
// definitions in an in-memory map. The terminal childWorkflow resolver below
|
|
184
|
+
// runs each child from THIS map, with no on-disk asset read and no separate
|
|
185
|
+
// per-child re-verify.
|
|
186
|
+
const childRewrite = rewriteInlineChildWorkflowBodies(definition);
|
|
187
|
+
definition = childRewrite.workflow;
|
|
188
|
+
const childBodiesMap = new Map(childRewrite.bodies.map((b) => [b.ref, b.definition]));
|
|
189
|
+
// Directors resolve from the pinned closure so a custom director authored in
|
|
190
|
+
// the workflow's own package runs. Loading directors OUTSIDE the
|
|
191
|
+
// definition-hash re-verify is safe: the approved hash pins each director's
|
|
192
|
+
// id + config (which director runs cannot change post-approval) and the
|
|
193
|
+
// closure's SRI pins its module bytes. Folding directors into the hash would
|
|
194
|
+
// be redundant, so it is deliberately not done -- see
|
|
195
|
+
// `loadWorkflowDirectorRegistryFromClosure`.
|
|
196
|
+
const directors = await loadWorkflowDirectorRegistryFromClosure({
|
|
197
|
+
packageDir: opts.env.closurePackageDir,
|
|
198
|
+
});
|
|
199
|
+
// Suspendable-child (onTrigger body) resolver, selected ONCE per deployment:
|
|
200
|
+
// the bodies map is immutable and the per-run `onEvent` is injected later in
|
|
201
|
+
// `buildRuntimeEnv`. Resolve each body from the parent's in-memory closure
|
|
202
|
+
// (already re-verified above) via the raw executor binding. A deployment that
|
|
203
|
+
// carries bodies but whose host wired no executor is a misconfiguration --
|
|
204
|
+
// fail loud at startup rather than silently falling back to a disk read (the
|
|
205
|
+
// exact behaviour this arm exists to avoid). A deployment with no onTrigger
|
|
206
|
+
// body leaves the host undefined; its suspendable-child slot is never invoked.
|
|
207
|
+
let suspendableChildHost;
|
|
208
|
+
if (bodiesMap.size > 0) {
|
|
209
|
+
const executor = opts.bindings.runSuspendableChild;
|
|
210
|
+
if (executor === undefined) {
|
|
211
|
+
throw new Error("workflow-child: source-ref deployment carries onTrigger bodies but " +
|
|
212
|
+
"the host wired no runSuspendableChild executor; cannot resolve " +
|
|
213
|
+
"bodies in-memory");
|
|
214
|
+
}
|
|
215
|
+
suspendableChildHost = createInMemorySpawnSuspendableChild({
|
|
216
|
+
bodies: bodiesMap,
|
|
217
|
+
runSuspendableChild: executor,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
// Terminal childWorkflow resolver, selected ONCE per deployment. When the
|
|
221
|
+
// definition embeds any inline child (the lifted map is non-empty), resolve
|
|
222
|
+
// each from that in-memory map via the raw terminal executor -- the parent's
|
|
223
|
+
// own re-verified closure -- so an owned child spawns with no disk read. A
|
|
224
|
+
// deployment that embeds a childWorkflow but whose host wired no executor is
|
|
225
|
+
// a misconfiguration and fails loud at startup rather than falling back to a
|
|
226
|
+
// disk read. A definition with no inline child keeps the injected binding (a
|
|
227
|
+
// test seam); its childWorkflow slot is never invoked.
|
|
228
|
+
let spawnChild;
|
|
229
|
+
if (childBodiesMap.size > 0) {
|
|
230
|
+
const executor = opts.bindings.runChild;
|
|
231
|
+
if (executor === undefined) {
|
|
232
|
+
throw new Error("workflow-child: deployment embeds childWorkflow imports but the " +
|
|
233
|
+
"host wired no runChild executor; cannot resolve children in-memory");
|
|
234
|
+
}
|
|
235
|
+
spawnChild = createInMemorySpawnChild({
|
|
236
|
+
bodies: childBodiesMap,
|
|
237
|
+
runChild: executor,
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
else if (opts.bindings.spawnChild !== undefined) {
|
|
241
|
+
spawnChild = opts.bindings.spawnChild;
|
|
242
|
+
}
|
|
243
|
+
else {
|
|
244
|
+
// No inline child and no injected binding: a workflow that nonetheless
|
|
245
|
+
// reaches a childWorkflow spawn fails loud here rather than silently
|
|
246
|
+
// completing against a child that never ran.
|
|
247
|
+
spawnChild = async ({ definitionRef }) => {
|
|
248
|
+
throw new Error(`workflow-child: childWorkflow ${definitionRef} reached the runtime ` +
|
|
249
|
+
`but no child executor is wired`);
|
|
250
|
+
};
|
|
251
|
+
}
|
|
124
252
|
const authorize = createCredentialsBackedAuthorize(credentialsRef, opts.bindings.evaluateGrants);
|
|
125
253
|
const drainController = createWorkflowHostDrainController({ definition });
|
|
126
254
|
// Warm-agent cache (design §3b). Built only when the deployment is a
|
|
@@ -174,16 +302,20 @@ export async function runWorkflowChild(opts) {
|
|
|
174
302
|
runtimeRepoStore,
|
|
175
303
|
authorize,
|
|
176
304
|
directors,
|
|
305
|
+
suspendableChildHost,
|
|
306
|
+
spawnChild,
|
|
177
307
|
clock,
|
|
178
308
|
newId,
|
|
179
309
|
drainController,
|
|
180
310
|
warmCache,
|
|
181
311
|
sourcesRef,
|
|
312
|
+
credentialWiring,
|
|
182
313
|
onEvent: (event) => {
|
|
183
314
|
void eventSender.send(event).catch((cause) => {
|
|
184
315
|
logger.error `event-channel send failed during resume run ${run.runId}: ${String(cause)}`;
|
|
185
316
|
});
|
|
186
317
|
},
|
|
318
|
+
upstreamSender,
|
|
187
319
|
});
|
|
188
320
|
const handle = runtimeRun(definition, env, {
|
|
189
321
|
runId: run.runId,
|
|
@@ -202,13 +334,12 @@ export async function runWorkflowChild(opts) {
|
|
|
202
334
|
cleanupRunStorage: opts.bindings.cleanupRunStorage,
|
|
203
335
|
runId: run.runId,
|
|
204
336
|
});
|
|
337
|
+
runsInFlight.delete(run.runId);
|
|
205
338
|
return emitTerminalEvent(upstreamSender, result);
|
|
206
339
|
})
|
|
207
340
|
.catch((cause) => {
|
|
208
|
-
logger.error `resumed run ${run.runId} failed: ${String(cause)}`;
|
|
209
|
-
})
|
|
210
|
-
.finally(() => {
|
|
211
341
|
runsInFlight.delete(run.runId);
|
|
342
|
+
logger.error `resumed run ${run.runId} failed: ${String(cause)}`;
|
|
212
343
|
});
|
|
213
344
|
resumedRunIds.push(run.runId);
|
|
214
345
|
}
|
|
@@ -231,6 +362,12 @@ export async function runWorkflowChild(opts) {
|
|
|
231
362
|
childPublicKey: hexEncode(childKeyPair.publicKey),
|
|
232
363
|
},
|
|
233
364
|
});
|
|
365
|
+
// Report self-discovered runs so the supervisor seeds its cohort
|
|
366
|
+
// tracking before the dispatch loop starts.
|
|
367
|
+
await upstreamSender.send({
|
|
368
|
+
type: "resumed.runs",
|
|
369
|
+
data: { runIds: resumedRunIds },
|
|
370
|
+
});
|
|
234
371
|
const triggeredRunIds = [];
|
|
235
372
|
// Control-loop. The receiver iterator yields one verified payload
|
|
236
373
|
// per call; any signature/channelId/seq violation crashes the
|
|
@@ -253,6 +390,8 @@ export async function runWorkflowChild(opts) {
|
|
|
253
390
|
definition,
|
|
254
391
|
authorize,
|
|
255
392
|
directors,
|
|
393
|
+
suspendableChildHost,
|
|
394
|
+
spawnChild,
|
|
256
395
|
clock,
|
|
257
396
|
newId,
|
|
258
397
|
eventSender,
|
|
@@ -262,6 +401,8 @@ export async function runWorkflowChild(opts) {
|
|
|
262
401
|
runsInFlight,
|
|
263
402
|
warmCache,
|
|
264
403
|
sourcesRef,
|
|
404
|
+
credentialMaterialRef,
|
|
405
|
+
credentialWiring,
|
|
265
406
|
...(opts.substrateWriteBridge !== undefined
|
|
266
407
|
? { substrateWriteBridge: opts.substrateWriteBridge }
|
|
267
408
|
: {}),
|
|
@@ -320,16 +461,16 @@ async function handleControlPayload(payload, ctx) {
|
|
|
320
461
|
case "trigger.fire": {
|
|
321
462
|
// One driver per runId. If this child is already driving this
|
|
322
463
|
// runId -- self-discovery resumed it, or an earlier trigger opened
|
|
323
|
-
// it --
|
|
324
|
-
//
|
|
464
|
+
// it -- a duplicate/stale trigger frame (which carries the local part
|
|
465
|
+
// of the deployment's mail address as the runId and no resumeFromEvents)
|
|
466
|
+
// must NOT spawn a second `runtimeRun`. A
|
|
325
467
|
// second concurrent driver would race the live one to settle the
|
|
326
468
|
// same residual and the loser throws an uncaught TransitionError,
|
|
327
469
|
// and even a driver that avoided the throw would double-emit the
|
|
328
470
|
// terminal. The live driver's completion continuation owns the
|
|
329
471
|
// single terminal emission; the supervisor's terminal-event-driven
|
|
330
|
-
// `markConsumed` consumes the message off that one terminal,
|
|
331
|
-
// work is dropped by declining here. Record the runId
|
|
332
|
-
// supervisor did fire a trigger and it was accepted) and signal
|
|
472
|
+
// `markConsumed` consumes the original message off that one terminal,
|
|
473
|
+
// so no work is dropped by declining here. Record the runId and signal
|
|
333
474
|
// "handled, not shutdown" the same way the normal trigger case
|
|
334
475
|
// returns, without awaiting the live handle's `complete` inline
|
|
335
476
|
// (that would block the control loop).
|
|
@@ -360,16 +501,20 @@ async function handleControlPayload(payload, ctx) {
|
|
|
360
501
|
runtimeRepoStore: ctx.runtimeRepoStore,
|
|
361
502
|
authorize: ctx.authorize,
|
|
362
503
|
directors: ctx.directors,
|
|
504
|
+
suspendableChildHost: ctx.suspendableChildHost,
|
|
505
|
+
spawnChild: ctx.spawnChild,
|
|
363
506
|
clock: ctx.clock,
|
|
364
507
|
newId: ctx.newId,
|
|
365
508
|
drainController: ctx.drainController,
|
|
366
509
|
warmCache: ctx.warmCache,
|
|
367
510
|
sourcesRef: ctx.sourcesRef,
|
|
511
|
+
credentialWiring: ctx.credentialWiring,
|
|
368
512
|
onEvent: (event) => {
|
|
369
513
|
void ctx.eventSender.send(event).catch((cause) => {
|
|
370
514
|
logger.error `event-channel send failed during run ${payload.data.runId}: ${String(cause)}`;
|
|
371
515
|
});
|
|
372
516
|
},
|
|
517
|
+
upstreamSender: ctx.upstreamSender,
|
|
373
518
|
});
|
|
374
519
|
const handle = runtimeRun(ctx.definition, env, {
|
|
375
520
|
runId: payload.data.runId,
|
|
@@ -392,13 +537,12 @@ async function handleControlPayload(payload, ctx) {
|
|
|
392
537
|
cleanupRunStorage: ctx.bindings.cleanupRunStorage,
|
|
393
538
|
runId: payload.data.runId,
|
|
394
539
|
});
|
|
540
|
+
ctx.runsInFlight.delete(payload.data.runId);
|
|
395
541
|
return emitTerminalEvent(ctx.upstreamSender, result);
|
|
396
542
|
})
|
|
397
543
|
.catch((cause) => {
|
|
398
|
-
logger.error `triggered run ${payload.data.runId} failed: ${String(cause)}`;
|
|
399
|
-
})
|
|
400
|
-
.finally(() => {
|
|
401
544
|
ctx.runsInFlight.delete(payload.data.runId);
|
|
545
|
+
logger.error `triggered run ${payload.data.runId} failed: ${String(cause)}`;
|
|
402
546
|
});
|
|
403
547
|
ctx.triggeredRunIds.push(payload.data.runId);
|
|
404
548
|
return false;
|
|
@@ -432,7 +576,26 @@ async function handleControlPayload(payload, ctx) {
|
|
|
432
576
|
ctx.credentialsRef.current = snapshot;
|
|
433
577
|
return false;
|
|
434
578
|
}
|
|
579
|
+
case "credentials-updated": {
|
|
580
|
+
// Replace the in-memory credential material wholesale. A revoked
|
|
581
|
+
// credential arrives by omission -- its material entry is absent from
|
|
582
|
+
// the delivery -- so the swap evicts it. Atomic whole-object assignment,
|
|
583
|
+
// so a concurrent reader never observes a torn cell. The secret stays on
|
|
584
|
+
// this ref only; nothing here copies it into a snapshot, event, or state.
|
|
585
|
+
ctx.credentialMaterialRef.current = payload.data.delivery;
|
|
586
|
+
return false;
|
|
587
|
+
}
|
|
435
588
|
case "signal.deliver": {
|
|
589
|
+
// Drop a delivery for a run this child is not driving. The dispatch path
|
|
590
|
+
// only ever targets a live run id, but a stale or mis-routed frame -- a
|
|
591
|
+
// synthetic body-child id, or a run that crashed and has not been
|
|
592
|
+
// re-discovered -- must not commit an orphan `SignalReceived` to a log no
|
|
593
|
+
// awaiter is tailing. `runsInFlight` is the one-driver authority on which
|
|
594
|
+
// runs this child drives.
|
|
595
|
+
if (!ctx.runsInFlight.has(payload.data.runId)) {
|
|
596
|
+
logger.warn `signal.deliver for run ${payload.data.runId} which is not in flight; dropping (signalName=${payload.data.signalName})`;
|
|
597
|
+
return false;
|
|
598
|
+
}
|
|
436
599
|
// Land the signal as a `SignalReceived` commit on the run's
|
|
437
600
|
// event log. The signal-channel substrate's `subscribeKind`
|
|
438
601
|
// peer (the per-run signal channel installed at run start) is
|
|
@@ -566,6 +729,13 @@ async function handleControlPayload(payload, ctx) {
|
|
|
566
729
|
// `ready` or `recycle.request`.
|
|
567
730
|
throw new Error("workflow-child received a `terminal.event` frame on its inbound control channel; this is a child-only upstream payload");
|
|
568
731
|
}
|
|
732
|
+
case "park.notify": {
|
|
733
|
+
// `park.notify` is the child->supervisor suspension-notification
|
|
734
|
+
// frame; receiving one on the child's downstream side is a
|
|
735
|
+
// protocol violation in the same shape as a downstream
|
|
736
|
+
// `terminal.event`.
|
|
737
|
+
throw new Error("workflow-child received a `park.notify` frame on its inbound control channel; this is a child-only upstream payload");
|
|
738
|
+
}
|
|
569
739
|
case "outbound.message": {
|
|
570
740
|
// `outbound.message` is the child->supervisor outbound-mail
|
|
571
741
|
// request frame; receiving one on the child's downstream side is a
|
|
@@ -609,6 +779,42 @@ async function handleControlPayload(payload, ctx) {
|
|
|
609
779
|
ctx.substrateWriteBridge.handleWriteResponse(payload.data);
|
|
610
780
|
return false;
|
|
611
781
|
}
|
|
782
|
+
case "parked-correlations.request": {
|
|
783
|
+
// Answer the supervisor's re-registration enumeration from durable
|
|
784
|
+
// state. Awaiting inline is safe -- unlike `signal.deliver`, this
|
|
785
|
+
// reads (self-discovery + the snapshot binding) and sends one upstream
|
|
786
|
+
// reply without awaiting any downstream frame, so it cannot deadlock
|
|
787
|
+
// the iterator against a response it is itself blocking. A store
|
|
788
|
+
// inconsistency (an enumerated park with no durable snapshot, or no
|
|
789
|
+
// binding to recover one) throws out of the loop like the other
|
|
790
|
+
// invariant-violation arms rather than dropping a correlation the hub
|
|
791
|
+
// is waiting to register.
|
|
792
|
+
const parked = await collectParkedApprovalCorrelations({
|
|
793
|
+
substrate: ctx.bindings.substrate,
|
|
794
|
+
repoId: ctx.bindings.workflowRunRepoId,
|
|
795
|
+
runtimeRepoStore: ctx.runtimeRepoStore,
|
|
796
|
+
...(ctx.bindings.loadParkedApproval !== undefined
|
|
797
|
+
? { loadParkedApproval: ctx.bindings.loadParkedApproval }
|
|
798
|
+
: {}),
|
|
799
|
+
});
|
|
800
|
+
await ctx.upstreamSender.send({
|
|
801
|
+
type: "parked-correlations.response",
|
|
802
|
+
data: { requestId: payload.data.requestId, parked },
|
|
803
|
+
});
|
|
804
|
+
return false;
|
|
805
|
+
}
|
|
806
|
+
case "resumed.runs": {
|
|
807
|
+
// `resumed.runs` is the child->supervisor self-discovery report;
|
|
808
|
+
// receiving one on the child's downstream side is a protocol
|
|
809
|
+
// violation in the same shape as a downstream `ready`.
|
|
810
|
+
throw new Error("workflow-child received a `resumed.runs` frame on its inbound control channel; this is a child-only upstream payload");
|
|
811
|
+
}
|
|
812
|
+
case "parked-correlations.response": {
|
|
813
|
+
// `parked-correlations.response` is the child->supervisor reply frame;
|
|
814
|
+
// receiving one on the child's downstream side is a protocol violation
|
|
815
|
+
// in the same shape as a downstream `substrate.merge.response`.
|
|
816
|
+
throw new Error("workflow-child received a `parked-correlations.response` frame on its inbound control channel; this is a child-only upstream payload");
|
|
817
|
+
}
|
|
612
818
|
}
|
|
613
819
|
}
|
|
614
820
|
/**
|
|
@@ -643,8 +849,18 @@ function buildRuntimeEnv(args) {
|
|
|
643
849
|
// `ChildStepInvoker` shape (carries onEvent), so the workflow-
|
|
644
850
|
// runtime never has to know an event firehose exists.
|
|
645
851
|
const invokeStep = async (req) => {
|
|
646
|
-
return args.bindings.invokeStep(req, args.onEvent, args.authorize, args.warmCache, args.sourcesRef);
|
|
852
|
+
return args.bindings.invokeStep(req, args.onEvent, args.authorize, args.warmCache, args.sourcesRef, args.credentialWiring);
|
|
647
853
|
};
|
|
854
|
+
// Adapt the host binding (which takes the run's `onEvent` sink) down to the
|
|
855
|
+
// runtime's narrow `SpawnSuspendableChild` by injecting THIS run's event
|
|
856
|
+
// funnel -- the same closure `invokeStep` forwards -- so a body's live
|
|
857
|
+
// inference events ride the parent run's event channel to the hub stream
|
|
858
|
+
// (and inherit its loud-on-failure logging), while the runtime env keeps the
|
|
859
|
+
// narrow contract with no event slot.
|
|
860
|
+
const hostSuspendable = args.suspendableChildHost;
|
|
861
|
+
const spawnSuspendableChild = hostSuspendable === undefined
|
|
862
|
+
? undefined
|
|
863
|
+
: (spawnInput) => hostSuspendable(spawnInput, args.onEvent);
|
|
648
864
|
return {
|
|
649
865
|
repoStore: args.runtimeRepoStore,
|
|
650
866
|
scheduler: args.bindings.scheduler,
|
|
@@ -653,12 +869,62 @@ function buildRuntimeEnv(args) {
|
|
|
653
869
|
directors: args.directors,
|
|
654
870
|
authorize: args.authorize,
|
|
655
871
|
invokeStep,
|
|
656
|
-
spawnChild: args.
|
|
872
|
+
spawnChild: args.spawnChild,
|
|
873
|
+
// Wire the suspendable-child seam only when the host supplied it; a child
|
|
874
|
+
// that never runs an onTrigger section omits the binding, and the runtime
|
|
875
|
+
// body fails loud if a workflow reaches a section the env did not wire.
|
|
876
|
+
...(spawnSuspendableChild !== undefined ? { spawnSuspendableChild } : {}),
|
|
657
877
|
clock: args.clock,
|
|
658
878
|
newId: args.newId,
|
|
659
879
|
drain: args.drainController,
|
|
880
|
+
// Forward a control-plane suspension up the same upstream control
|
|
881
|
+
// channel `terminal.event` rides, so the supervisor can stamp the
|
|
882
|
+
// deployment identity and register the correlation at the hub. The
|
|
883
|
+
// runtime body fires this once per fresh park on a reserved
|
|
884
|
+
// `signalName(correlationId)` channel.
|
|
885
|
+
onPark: (park) => {
|
|
886
|
+
void emitParkNotify(args.upstreamSender, park);
|
|
887
|
+
},
|
|
888
|
+
// Let the resume classifier recover a step that crashed across the park
|
|
889
|
+
// boundary. Absent (tests, the recursive child-workflow adapter) leaves a
|
|
890
|
+
// crashed invocation a terminal failure.
|
|
891
|
+
...(args.bindings.readParkedApprovalOps !== undefined
|
|
892
|
+
? { readParkedApprovalOps: args.bindings.readParkedApprovalOps }
|
|
893
|
+
: {}),
|
|
660
894
|
};
|
|
661
895
|
}
|
|
896
|
+
/**
|
|
897
|
+
* Forward a control-plane suspension to the supervisor over the upstream
|
|
898
|
+
* control channel. Fired from `env.onPark` each time a workflow agent step
|
|
899
|
+
* parks on a reserved `signalName(correlationId)` channel. The supervisor's
|
|
900
|
+
* `park.notify` arm stamps the deployment identity it owns and sends a
|
|
901
|
+
* `signal.correlation.register` frame to the hub.
|
|
902
|
+
*
|
|
903
|
+
* Best-effort like `emitTerminalEvent`'s send: a transport failure is logged,
|
|
904
|
+
* not rethrown. A lost frame means the correlation is not registered and the
|
|
905
|
+
* parked run cannot be resumed until it is re-registered; the failure surfaces
|
|
906
|
+
* structurally as a run that never resumes rather than a silent lifecycle
|
|
907
|
+
* corruption. The register at the hub is idempotent, so a re-park resume's
|
|
908
|
+
* re-emit is safe.
|
|
909
|
+
*/
|
|
910
|
+
export function emitParkNotify(upstreamSender, park) {
|
|
911
|
+
return upstreamSender
|
|
912
|
+
.send({
|
|
913
|
+
type: "park.notify",
|
|
914
|
+
data: {
|
|
915
|
+
runId: park.runId,
|
|
916
|
+
correlationId: park.correlationId,
|
|
917
|
+
parkKind: park.parkKind,
|
|
918
|
+
...(park.approvalSnapshot !== undefined
|
|
919
|
+
? { snapshot: park.approvalSnapshot }
|
|
920
|
+
: {}),
|
|
921
|
+
},
|
|
922
|
+
})
|
|
923
|
+
.catch((cause) => {
|
|
924
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
925
|
+
logger.error `park.notify upstream send failed for runId=${park.runId} correlationId=${park.correlationId}: ${message}`;
|
|
926
|
+
});
|
|
927
|
+
}
|
|
662
928
|
/**
|
|
663
929
|
* Mirror a run's terminal status back to the supervisor over the
|
|
664
930
|
* upstream control channel. Fired once per run from the resume and
|
|
@@ -804,90 +1070,6 @@ async function resolveTriggerPayload(args) {
|
|
|
804
1070
|
const raw = base64Decode(rawMessageBase64);
|
|
805
1071
|
return extractConversationText(raw, args.messageId);
|
|
806
1072
|
}
|
|
807
|
-
/**
|
|
808
|
-
* Extract the conversation body text from a raw inbound MIME message.
|
|
809
|
-
*
|
|
810
|
-
* Three on-wire shapes are handled, matching every producer the mail
|
|
811
|
-
* bus accepts:
|
|
812
|
-
* 1. The Interchange assembler's `multipart/signed` envelope whose
|
|
813
|
-
* first part is a `multipart/mixed` body carrying the text at part
|
|
814
|
-
* path `1.1`.
|
|
815
|
-
* 2. A `multipart/signed` envelope wrapping a bare `text/plain` part
|
|
816
|
-
* (a sender that signs without the `multipart/mixed` wrapper); the
|
|
817
|
-
* text is at part path `1`.
|
|
818
|
-
* 3. A flat top-level `text/plain` message (no multipart structure at
|
|
819
|
-
* all); the body is the bytes after the header section.
|
|
820
|
-
*
|
|
821
|
-
* The top-level `Content-Type` selects the shape: only a `multipart/*`
|
|
822
|
-
* root walks into parts; anything else reads the single body directly.
|
|
823
|
-
* This mirrors the conversation branch of mail-memory's `fetchFull`
|
|
824
|
-
* while also tolerating the flat single-part case the in-process agent
|
|
825
|
-
* accepts, so a non-standard inbound mail still delivers its text to
|
|
826
|
-
* the agent rather than crashing the run.
|
|
827
|
-
*/
|
|
828
|
-
function extractConversationText(raw, messageId) {
|
|
829
|
-
const { headers, bodyOffset } = parseHeaderSection(raw);
|
|
830
|
-
const rootMime = (headers.get("content-type") ?? "")
|
|
831
|
-
.split(";")[0]
|
|
832
|
-
?.trim()
|
|
833
|
-
.toLowerCase();
|
|
834
|
-
if (rootMime === undefined || !rootMime.startsWith("multipart/")) {
|
|
835
|
-
// Flat single-part message: the body is everything after the
|
|
836
|
-
// header section.
|
|
837
|
-
return new TextDecoder("utf-8", { fatal: false }).decode(raw.subarray(bodyOffset));
|
|
838
|
-
}
|
|
839
|
-
let part1;
|
|
840
|
-
try {
|
|
841
|
-
part1 = parseMimePart(extractPartByPath(raw, "1"));
|
|
842
|
-
}
|
|
843
|
-
catch (cause) {
|
|
844
|
-
throw new Error(`workflow-child trigger.fire: cannot parse inbound mail part 1 for messageId ${messageId}`, { cause });
|
|
845
|
-
}
|
|
846
|
-
const part1Mime = (part1.contentType.split(";")[0] ?? "")
|
|
847
|
-
.trim()
|
|
848
|
-
.toLowerCase();
|
|
849
|
-
const bodyBytes = part1Mime.startsWith("multipart/")
|
|
850
|
-
? parseMimePart(extractPartByPath(raw, "1.1")).body
|
|
851
|
-
: part1.body;
|
|
852
|
-
return new TextDecoder("utf-8", { fatal: false }).decode(bodyBytes);
|
|
853
|
-
}
|
|
854
|
-
/**
|
|
855
|
-
* Load the `WorkflowDefinition` from the workflow asset repo's deploy
|
|
856
|
-
* ref. Mirrors the sibling `spawn-child` adapter's working-tree-read
|
|
857
|
-
* pattern -- the deploy orchestrator's `writeTree` materializes
|
|
858
|
-
* `workflow.json` under the substrate's repo dir, so a flat
|
|
859
|
-
* `fs.readFile` returns the bytes without round-tripping through git.
|
|
860
|
-
*/
|
|
861
|
-
async function loadWorkflowDefinition(bindings) {
|
|
862
|
-
const fs = await import("node:fs/promises");
|
|
863
|
-
const path = await import("node:path");
|
|
864
|
-
const dir = bindings.substrate.getRepoDir(bindings.workflowDefinitionRepoId);
|
|
865
|
-
const workflowPath = path.join(dir, WORKFLOW_JSON_PATH);
|
|
866
|
-
let raw;
|
|
867
|
-
try {
|
|
868
|
-
raw = await fs.readFile(workflowPath, "utf8");
|
|
869
|
-
}
|
|
870
|
-
catch (cause) {
|
|
871
|
-
throw new Error(`workflow-child: cannot read ${WORKFLOW_JSON_PATH} for ${bindings.workflowDefinitionRepoId.kind}/${bindings.workflowDefinitionRepoId.id} on ${bindings.workflowDefinitionRef}`, { cause });
|
|
872
|
-
}
|
|
873
|
-
let parsed;
|
|
874
|
-
try {
|
|
875
|
-
parsed = JSON.parse(raw);
|
|
876
|
-
}
|
|
877
|
-
catch (cause) {
|
|
878
|
-
throw new Error(`workflow-child: ${WORKFLOW_JSON_PATH} for ${bindings.workflowDefinitionRepoId.kind}/${bindings.workflowDefinitionRepoId.id} on ${bindings.workflowDefinitionRef} is not valid JSON`, { cause });
|
|
879
|
-
}
|
|
880
|
-
const validated = workflowDefinitionEnvelopeSchema(parsed);
|
|
881
|
-
if (validated instanceof type.errors) {
|
|
882
|
-
throw new Error(`workflow-child: ${WORKFLOW_JSON_PATH} for ${bindings.workflowDefinitionRepoId.kind}/${bindings.workflowDefinitionRepoId.id} on ${bindings.workflowDefinitionRef} failed envelope validation: ${validated.summary}`);
|
|
883
|
-
}
|
|
884
|
-
// The envelope schema enforces the structural shape; the
|
|
885
|
-
// discriminated narrow over every primitive variant lives downstream
|
|
886
|
-
// in the runtime body. The sibling `spawn-child` adapter follows the
|
|
887
|
-
// same pattern at the same boundary.
|
|
888
|
-
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- envelope schema enforces structural shape; primitive narrows live downstream in the runtime body
|
|
889
|
-
return validated;
|
|
890
|
-
}
|
|
891
1073
|
function defaultClock() {
|
|
892
1074
|
return new Date();
|
|
893
1075
|
}
|
|
@@ -25,5 +25,15 @@ export interface DiscoverRunsOpts {
|
|
|
25
25
|
* terminal event. Runs that already terminated are skipped because
|
|
26
26
|
* resume against a terminal log would still settle without progress
|
|
27
27
|
* but would generate spurious "resume seed" reads for no benefit.
|
|
28
|
+
*
|
|
29
|
+
* Child runs are excluded. A run spawned by another run -- a
|
|
30
|
+
* `childWorkflow` step's child or an `onTrigger` section's per-event body,
|
|
31
|
+
* whichever committed a `ChildSpawned` naming it -- is driven by its
|
|
32
|
+
* PARENT's runtime, not on its own. Resuming a child here would re-run it
|
|
33
|
+
* under the deployment definition rather than the child's own definition,
|
|
34
|
+
* and for a body approval park it would hub-register the child's internal
|
|
35
|
+
* park that the parent already proxies up on the shared correlation. A
|
|
36
|
+
* child run is identified structurally, by appearing as a
|
|
37
|
+
* `ChildSpawned.childRunId` in some log, rather than by its id shape.
|
|
28
38
|
*/
|
|
29
39
|
export declare function discoverInFlightRuns(opts: DiscoverRunsOpts): Promise<readonly DiscoveredRun[]>;
|
|
@@ -22,6 +22,16 @@ const RUNS_PREFIX = "runs";
|
|
|
22
22
|
* terminal event. Runs that already terminated are skipped because
|
|
23
23
|
* resume against a terminal log would still settle without progress
|
|
24
24
|
* but would generate spurious "resume seed" reads for no benefit.
|
|
25
|
+
*
|
|
26
|
+
* Child runs are excluded. A run spawned by another run -- a
|
|
27
|
+
* `childWorkflow` step's child or an `onTrigger` section's per-event body,
|
|
28
|
+
* whichever committed a `ChildSpawned` naming it -- is driven by its
|
|
29
|
+
* PARENT's runtime, not on its own. Resuming a child here would re-run it
|
|
30
|
+
* under the deployment definition rather than the child's own definition,
|
|
31
|
+
* and for a body approval park it would hub-register the child's internal
|
|
32
|
+
* park that the parent already proxies up on the shared correlation. A
|
|
33
|
+
* child run is identified structurally, by appearing as a
|
|
34
|
+
* `ChildSpawned.childRunId` in some log, rather than by its id shape.
|
|
25
35
|
*/
|
|
26
36
|
export async function discoverInFlightRuns(opts) {
|
|
27
37
|
const fs = await import("node:fs/promises");
|
|
@@ -37,11 +47,25 @@ export async function discoverInFlightRuns(opts) {
|
|
|
37
47
|
return [];
|
|
38
48
|
throw cause;
|
|
39
49
|
}
|
|
40
|
-
|
|
50
|
+
// Read every run's log once, and collect the set of run ids that any log
|
|
51
|
+
// spawned as a child. The scan spans every run (terminal ones included) so
|
|
52
|
+
// a child whose parent already completed is still recognized as a child.
|
|
53
|
+
const logs = new Map();
|
|
54
|
+
const childRunIds = new Set();
|
|
41
55
|
for (const runId of runDirs) {
|
|
42
56
|
const events = await opts.runtimeRepoStore.read(runId);
|
|
43
57
|
if (events.length === 0)
|
|
44
58
|
continue;
|
|
59
|
+
logs.set(runId, events);
|
|
60
|
+
for (const event of events) {
|
|
61
|
+
if (event.kind === "ChildSpawned")
|
|
62
|
+
childRunIds.add(event.childRunId);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const out = [];
|
|
66
|
+
for (const [runId, events] of logs) {
|
|
67
|
+
if (childRunIds.has(runId))
|
|
68
|
+
continue;
|
|
45
69
|
const resumed = resumeFromLog(runId, events);
|
|
46
70
|
if (isTerminalRunPhase(resumed.phase))
|
|
47
71
|
continue;
|