@intx/workflow-host 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/dist/adapters/mail-part-store.d.ts +46 -0
- package/dist/adapters/mail-part-store.js +251 -0
- package/dist/adapters/repo-store.js +5 -14
- package/dist/adapters/spawn-child.d.ts +42 -6
- package/dist/adapters/spawn-child.js +8 -18
- package/dist/adapters/step-invoker.d.ts +52 -2
- package/dist/adapters/step-invoker.js +230 -60
- package/dist/adapters/substrate-mailbox-store.d.ts +80 -0
- package/dist/adapters/substrate-mailbox-store.js +404 -0
- package/dist/child/child-mailbox-reader.d.ts +10 -0
- package/dist/child/child-mailbox-reader.js +23 -0
- package/dist/child/credential-cell.d.ts +8 -0
- package/dist/child/credential-cell.js +66 -0
- package/dist/child/from-process-env.d.ts +12 -0
- package/dist/child/from-process-env.js +6 -0
- package/dist/child/index.d.ts +4 -1
- package/dist/child/index.js +4 -1
- package/dist/child/mailbox-mutation-bridge.d.ts +61 -0
- package/dist/child/mailbox-mutation-bridge.js +101 -0
- package/dist/child/mailbox-watch-registry.d.ts +17 -0
- package/dist/child/mailbox-watch-registry.js +61 -0
- package/dist/child/outbound-mail-bridge.d.ts +3 -2
- package/dist/child/outbound-mail-bridge.js +20 -32
- package/dist/child/pending-request.d.ts +89 -0
- package/dist/child/pending-request.js +80 -0
- package/dist/child/run-child.d.ts +69 -7
- package/dist/child/run-child.js +307 -75
- package/dist/child/substrate-write-bridge.d.ts +3 -2
- package/dist/child/substrate-write-bridge.js +21 -38
- package/dist/child/supervisor-backed-transport.d.ts +52 -6
- package/dist/child/supervisor-backed-transport.js +205 -62
- package/dist/child/warm-agent-cache.d.ts +44 -4
- package/dist/child/warm-agent-cache.js +41 -10
- package/dist/index.d.ts +4 -3
- package/dist/index.js +4 -3
- package/dist/ipc/control-channel.d.ts +93 -2
- package/dist/ipc/control-channel.js +147 -47
- package/dist/ipc/index.d.ts +1 -1
- package/dist/ipc/index.js +1 -1
- package/dist/run-body-then-cleanup.d.ts +17 -0
- package/dist/run-body-then-cleanup.js +38 -0
- package/dist/seams/scheduler.d.ts +12 -0
- package/dist/seams/scheduler.js +13 -4
- package/dist/supervisor/cancel-signing.js +3 -7
- package/dist/supervisor/credentials.d.ts +17 -5
- package/dist/supervisor/recycle.d.ts +5 -1
- package/dist/supervisor/run-event-compaction.d.ts +2 -2
- package/dist/supervisor/run-event-compaction.js +11 -16
- package/dist/supervisor/run-event-recovery.d.ts +34 -0
- package/dist/supervisor/run-event-recovery.js +45 -0
- package/dist/supervisor/supervisor.d.ts +27 -4
- package/dist/supervisor/supervisor.js +644 -58
- package/dist/supervisor/terminal-commit.js +3 -7
- package/dist/supervisor/types.d.ts +30 -0
- package/dist/testing/change-notifier.d.ts +12 -0
- package/dist/testing/change-notifier.js +63 -0
- package/dist/testing/index.d.ts +8 -0
- package/dist/testing/index.js +16 -0
- package/dist/testing/log-capture.d.ts +52 -0
- package/dist/testing/log-capture.js +124 -0
- package/dist/testing/mail-bus.d.ts +22 -0
- package/dist/testing/mail-bus.js +78 -0
- package/dist/testing/memory-streams.d.ts +43 -0
- package/dist/testing/memory-streams.js +211 -0
- package/dist/testing/spawn-observer.d.ts +12 -0
- package/dist/testing/spawn-observer.js +36 -0
- package/dist/testing/stub-repo-store.d.ts +10 -0
- package/dist/testing/stub-repo-store.js +39 -0
- package/dist/testing/supervisor-reaper.d.ts +24 -0
- package/dist/testing/supervisor-reaper.js +49 -0
- package/dist/testing/upstream-frames.d.ts +47 -0
- package/dist/testing/upstream-frames.js +94 -0
- package/dist/workflow-definition-loader.d.ts +56 -0
- package/dist/workflow-definition-loader.js +106 -0
- package/package.json +17 -11
- package/dist/conversation-text.d.ts +0 -23
- package/dist/conversation-text.js +0 -56
package/dist/child/run-child.js
CHANGED
|
@@ -50,23 +50,24 @@
|
|
|
50
50
|
// respawn) and does not require a child-side control frame.
|
|
51
51
|
import { getLogger } from "@intx/log";
|
|
52
52
|
import { generateKeyPair } from "@intx/crypto";
|
|
53
|
-
import {
|
|
54
|
-
import {
|
|
55
|
-
import {
|
|
56
|
-
import { baseStepId, emptyState, runtimeRun } from "@intx/workflow";
|
|
53
|
+
import { hexEncode } from "@intx/types";
|
|
54
|
+
import { rewriteInlineOnTriggerBodies, rewriteInlineChildWorkflowBodies, enumerateInlineLoopBodies, eagerlyResolveLoopFns, } from "@intx/workflow";
|
|
55
|
+
import { baseStepId, createDefaultActionInvoker, createInMemoryEffectLedger, createLoopIterationHandle, emptyState, runtimeRun, } from "@intx/workflow";
|
|
57
56
|
import { createWorkflowHostDrainController, } from "../drain-controller.js";
|
|
58
57
|
import { createWorkflowRunRepoStore } from "../adapters/repo-store.js";
|
|
59
58
|
import { createWorkflowRunBlobSubstrate } from "../adapters/blob-substrate.js";
|
|
59
|
+
import { createMailPartReader } from "../adapters/mail-part-store.js";
|
|
60
60
|
import { createInMemorySpawnSuspendableChild, createInMemorySpawnChild, } from "../adapters/spawn-child.js";
|
|
61
61
|
import { createControlChannelSender, createEventChannelSender, receiveControlChannel, } from "../ipc/index.js";
|
|
62
|
+
import { runBodyThenCleanup } from "../run-body-then-cleanup.js";
|
|
62
63
|
import { createWorkflowHostSignalChannel } from "../seams/signal-channel.js";
|
|
63
|
-
import { extractConversationText } from "../conversation-text.js";
|
|
64
64
|
import { hashGrants } from "../supervisor/credentials.js";
|
|
65
65
|
import { loadVerifiedWorkflowDefinitionFromClosure } from "./verified-definition-loader.js";
|
|
66
|
-
import { loadWorkflowDirectorRegistryFromClosure } from "../workflow-definition-loader.js";
|
|
66
|
+
import { loadWorkflowActionHandlersFromClosure, loadWorkflowDirectorRegistryFromClosure, loadWorkflowLoopFnsFromClosure, } from "../workflow-definition-loader.js";
|
|
67
67
|
import { discoverInFlightRuns } from "./self-discovery.js";
|
|
68
68
|
import { collectParkedApprovalCorrelations, } from "./parked-correlations.js";
|
|
69
69
|
import { createWarmAgentCache } from "./warm-agent-cache.js";
|
|
70
|
+
import { mergeCredentialDelivery } from "./credential-cell.js";
|
|
70
71
|
const logger = getLogger(["workflow-host", "child"]);
|
|
71
72
|
export function createCredentialsBackedAuthorize(ref, evaluate) {
|
|
72
73
|
return async (resource, action, ctx) => {
|
|
@@ -186,6 +187,36 @@ export async function runWorkflowChild(opts) {
|
|
|
186
187
|
const childRewrite = rewriteInlineChildWorkflowBodies(definition);
|
|
187
188
|
definition = childRewrite.workflow;
|
|
188
189
|
const childBodiesMap = new Map(childRewrite.bodies.map((b) => [b.ref, b.definition]));
|
|
190
|
+
// A loop iteration runs its body as a suspendable child through the same seam
|
|
191
|
+
// an onTrigger body uses, so register each top-level loop body in `bodiesMap`
|
|
192
|
+
// under its `<workflowId>__<stepId>` ref. Unlike an onTrigger or childWorkflow
|
|
193
|
+
// body, a loop keeps its body INLINE on the primitive -- both hash layers
|
|
194
|
+
// project the body inline, so rewriting the primitive would change every
|
|
195
|
+
// existing loop's hash. `enumerateInlineLoopBodies` mints a ref-keyed COPY and
|
|
196
|
+
// leaves the primitive untouched, and a given step is exactly one primitive
|
|
197
|
+
// kind, so a loop ref never collides with an onTrigger or childWorkflow ref.
|
|
198
|
+
//
|
|
199
|
+
// A loop body may itself contain a `childWorkflow` grandchild: rewrite the
|
|
200
|
+
// COPY's inline children to `{ ref }` (the primitive, and thus the hash, is
|
|
201
|
+
// untouched) and fold the extracted grandchildren into `childBodiesMap` HERE
|
|
202
|
+
// -- before the eager loop-fn/handler resolution and the terminal-childWorkflow
|
|
203
|
+
// host selection below, both of which read `childBodiesMap`. Merging later
|
|
204
|
+
// would resolve a grandchild's refs mid-run instead of at establish, and would
|
|
205
|
+
// leave `childBodiesMap` empty for a deployment whose only child is a
|
|
206
|
+
// loop-body grandchild, so its spawn would find no wired terminal host.
|
|
207
|
+
// Keep each loop body's PRE-rewrite form (its childWorkflow grandchild still
|
|
208
|
+
// inline) keyed by ref: the grants cap for a loop iteration re-walks it, and
|
|
209
|
+
// capping the rewritten `{ ref }` form would skip -- and so under-authorize --
|
|
210
|
+
// the grandchild's declared resources.
|
|
211
|
+
const loopBodyPreRewrite = new Map();
|
|
212
|
+
for (const loopBody of enumerateInlineLoopBodies(definition)) {
|
|
213
|
+
loopBodyPreRewrite.set(loopBody.ref, loopBody.definition);
|
|
214
|
+
const bodyRewrite = rewriteInlineChildWorkflowBodies(loopBody.definition);
|
|
215
|
+
bodiesMap.set(loopBody.ref, bodyRewrite.workflow);
|
|
216
|
+
for (const grandchild of bodyRewrite.bodies) {
|
|
217
|
+
childBodiesMap.set(grandchild.ref, grandchild.definition);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
189
220
|
// Directors resolve from the pinned closure so a custom director authored in
|
|
190
221
|
// the workflow's own package runs. Loading directors OUTSIDE the
|
|
191
222
|
// definition-hash re-verify is safe: the approved hash pins each director's
|
|
@@ -196,21 +227,44 @@ export async function runWorkflowChild(opts) {
|
|
|
196
227
|
const directors = await loadWorkflowDirectorRegistryFromClosure({
|
|
197
228
|
packageDir: opts.env.closurePackageDir,
|
|
198
229
|
});
|
|
199
|
-
//
|
|
200
|
-
//
|
|
201
|
-
//
|
|
202
|
-
//
|
|
203
|
-
//
|
|
204
|
-
//
|
|
205
|
-
//
|
|
206
|
-
//
|
|
230
|
+
// Loop `while`/`carry` functions resolve from the pinned closure's
|
|
231
|
+
// `interchange.loops` module, loaded alongside the directors and OUTSIDE the
|
|
232
|
+
// definition-hash re-verify for the same reason: the approved hash pins each
|
|
233
|
+
// ref string and the closure's SRI pins the module bytes. Resolve every loop
|
|
234
|
+
// ref reachable from the definition (its own loop bodies, and the lifted
|
|
235
|
+
// onTrigger/childWorkflow bodies, which share this same registry at runtime)
|
|
236
|
+
// eagerly here, so a deployment that declares a loop whose fn the closure
|
|
237
|
+
// does not export fails at establish rather than mid-run.
|
|
238
|
+
const loopFns = await loadWorkflowLoopFnsFromClosure({
|
|
239
|
+
packageDir: opts.env.closurePackageDir,
|
|
240
|
+
});
|
|
241
|
+
eagerlyResolveLoopFns([definition, ...bodiesMap.values(), ...childBodiesMap.values()], loopFns);
|
|
242
|
+
// Action handlers resolve from the pinned closure's `interchange.actions`
|
|
243
|
+
// module, on the same terms as loop fns. Resolve every action handler ref
|
|
244
|
+
// reachable from the definition eagerly here (recursing into loop bodies,
|
|
245
|
+
// where an action body is the common case), so a deployment that declares an
|
|
246
|
+
// action whose handler the closure does not export fails at establish rather
|
|
247
|
+
// than mid-run.
|
|
248
|
+
const actionResolver = await loadWorkflowActionHandlersFromClosure({
|
|
249
|
+
packageDir: opts.env.closurePackageDir,
|
|
250
|
+
});
|
|
251
|
+
eagerlyResolveActionHandlers([definition, ...bodiesMap.values(), ...childBodiesMap.values()], actionResolver);
|
|
252
|
+
// Suspendable-child resolver (onTrigger section bodies and loop bodies),
|
|
253
|
+
// selected ONCE per deployment: the bodies map is immutable and the per-run
|
|
254
|
+
// `onEvent` is injected later in `buildRuntimeEnv`. Resolve each body from the
|
|
255
|
+
// parent's in-memory closure (already re-verified above) via the raw executor
|
|
256
|
+
// binding. A deployment that carries bodies but whose host wired no executor is
|
|
257
|
+
// a misconfiguration -- fail loud at startup rather than silently falling back
|
|
258
|
+
// to a disk read (the exact behaviour this arm exists to avoid). A deployment
|
|
259
|
+
// with no suspendable body leaves the host undefined; its slot is never
|
|
260
|
+
// invoked.
|
|
207
261
|
let suspendableChildHost;
|
|
208
262
|
if (bodiesMap.size > 0) {
|
|
209
263
|
const executor = opts.bindings.runSuspendableChild;
|
|
210
264
|
if (executor === undefined) {
|
|
211
|
-
throw new Error("workflow-child: source-ref deployment carries
|
|
212
|
-
"the host wired no
|
|
213
|
-
"bodies in-memory");
|
|
265
|
+
throw new Error("workflow-child: source-ref deployment carries suspendable bodies " +
|
|
266
|
+
"(onTrigger sections or loop bodies) but the host wired no " +
|
|
267
|
+
"runSuspendableChild executor; cannot resolve bodies in-memory");
|
|
214
268
|
}
|
|
215
269
|
suspendableChildHost = createInMemorySpawnSuspendableChild({
|
|
216
270
|
bodies: bodiesMap,
|
|
@@ -225,6 +279,9 @@ export async function runWorkflowChild(opts) {
|
|
|
225
279
|
// a misconfiguration and fails loud at startup rather than falling back to a
|
|
226
280
|
// disk read. A definition with no inline child keeps the injected binding (a
|
|
227
281
|
// test seam); its childWorkflow slot is never invoked.
|
|
282
|
+
// `HostSpawnChild` (a call-arg `onEvent`) like the suspendable host: the
|
|
283
|
+
// resolver is deployment-scoped but each run injects its own event sink in
|
|
284
|
+
// `buildRuntimeEnv`. The two fallback arms ignore the sink.
|
|
228
285
|
let spawnChild;
|
|
229
286
|
if (childBodiesMap.size > 0) {
|
|
230
287
|
const executor = opts.bindings.runChild;
|
|
@@ -238,13 +295,14 @@ export async function runWorkflowChild(opts) {
|
|
|
238
295
|
});
|
|
239
296
|
}
|
|
240
297
|
else if (opts.bindings.spawnChild !== undefined) {
|
|
241
|
-
|
|
298
|
+
const injected = opts.bindings.spawnChild;
|
|
299
|
+
spawnChild = (input, _onEvent) => injected(input);
|
|
242
300
|
}
|
|
243
301
|
else {
|
|
244
302
|
// No inline child and no injected binding: a workflow that nonetheless
|
|
245
303
|
// reaches a childWorkflow spawn fails loud here rather than silently
|
|
246
304
|
// completing against a child that never ran.
|
|
247
|
-
spawnChild = async ({ definitionRef }) => {
|
|
305
|
+
spawnChild = async ({ definitionRef }, _onEvent) => {
|
|
248
306
|
throw new Error(`workflow-child: childWorkflow ${definitionRef} reached the runtime ` +
|
|
249
307
|
`but no child executor is wired`);
|
|
250
308
|
};
|
|
@@ -303,7 +361,11 @@ export async function runWorkflowChild(opts) {
|
|
|
303
361
|
authorize,
|
|
304
362
|
directors,
|
|
305
363
|
suspendableChildHost,
|
|
364
|
+
bodiesMap,
|
|
365
|
+
loopBodyPreRewrite,
|
|
306
366
|
spawnChild,
|
|
367
|
+
loopFns,
|
|
368
|
+
actionResolver,
|
|
307
369
|
clock,
|
|
308
370
|
newId,
|
|
309
371
|
drainController,
|
|
@@ -380,7 +442,14 @@ export async function runWorkflowChild(opts) {
|
|
|
380
442
|
logger.error `workflow-child control channel crash: ${reason}`;
|
|
381
443
|
},
|
|
382
444
|
});
|
|
383
|
-
|
|
445
|
+
// Resolve the mailbox watch registry the control loop routes `mailbox.notify`
|
|
446
|
+
// frames to. Production wires it on the bindings (the substrate factory builds
|
|
447
|
+
// one instance and shares it with the warm agent's supervisor-backed
|
|
448
|
+
// transport); a test may inject one directly through the opts, which wins.
|
|
449
|
+
// Both absent leaves inbound `mailbox.notify` frames logged and dropped -- a
|
|
450
|
+
// deploy with no wired mail surface.
|
|
451
|
+
const mailboxWatchRegistry = opts.mailboxWatchRegistry ?? opts.bindings.mailboxWatchRegistry;
|
|
452
|
+
const runControlLoop = async () => {
|
|
384
453
|
for await (const payload of iter) {
|
|
385
454
|
if (await handleControlPayload(payload, {
|
|
386
455
|
env: opts.env,
|
|
@@ -391,7 +460,11 @@ export async function runWorkflowChild(opts) {
|
|
|
391
460
|
authorize,
|
|
392
461
|
directors,
|
|
393
462
|
suspendableChildHost,
|
|
463
|
+
bodiesMap,
|
|
464
|
+
loopBodyPreRewrite,
|
|
394
465
|
spawnChild,
|
|
466
|
+
loopFns,
|
|
467
|
+
actionResolver,
|
|
395
468
|
clock,
|
|
396
469
|
newId,
|
|
397
470
|
eventSender,
|
|
@@ -409,14 +482,20 @@ export async function runWorkflowChild(opts) {
|
|
|
409
482
|
...(opts.outboundMailBridge !== undefined
|
|
410
483
|
? { outboundMailBridge: opts.outboundMailBridge }
|
|
411
484
|
: {}),
|
|
485
|
+
...(opts.mailboxMutationBridge !== undefined
|
|
486
|
+
? { mailboxMutationBridge: opts.mailboxMutationBridge }
|
|
487
|
+
: {}),
|
|
488
|
+
...(mailboxWatchRegistry !== undefined
|
|
489
|
+
? { mailboxWatchRegistry }
|
|
490
|
+
: {}),
|
|
412
491
|
})) {
|
|
413
492
|
// shutdown received; the shutdown case already cancelled any
|
|
414
493
|
// pending substrate writes before returning true.
|
|
415
494
|
break;
|
|
416
495
|
}
|
|
417
496
|
}
|
|
418
|
-
}
|
|
419
|
-
|
|
497
|
+
};
|
|
498
|
+
const cleanupControlLoop = async () => {
|
|
420
499
|
// Any exit path -- clean (iterator end), dirty (thrown error),
|
|
421
500
|
// shutdown (already cancelled, repeat is a no-op on an empty map)
|
|
422
501
|
// -- cancels every still-pending substrate write so the runtime
|
|
@@ -433,6 +512,13 @@ export async function runWorkflowChild(opts) {
|
|
|
433
512
|
if (opts.outboundMailBridge !== undefined) {
|
|
434
513
|
opts.outboundMailBridge.cancelAll("workflow-child control loop exited");
|
|
435
514
|
}
|
|
515
|
+
// Same contract for mailbox mutations: a step agent's flag or
|
|
516
|
+
// `expunge` still awaiting the supervisor's `mailbox.mutate.response`
|
|
517
|
+
// when the control loop exits must surface a structured rejection
|
|
518
|
+
// rather than hang on a torn-down channel.
|
|
519
|
+
if (opts.mailboxMutationBridge !== undefined) {
|
|
520
|
+
opts.mailboxMutationBridge.cancelAll("workflow-child control loop exited");
|
|
521
|
+
}
|
|
436
522
|
// Evict the warm-agent cache (design §3b) on every exit path:
|
|
437
523
|
// graceful (shutdown frame -> iterator end), dirty (thrown error),
|
|
438
524
|
// or the control channel closing. Eviction runs the wrapped
|
|
@@ -444,7 +530,12 @@ export async function runWorkflowChild(opts) {
|
|
|
444
530
|
if (warmCache !== undefined) {
|
|
445
531
|
await warmCache.evictAll("workflow-child control loop exited");
|
|
446
532
|
}
|
|
447
|
-
}
|
|
533
|
+
};
|
|
534
|
+
// Run the control loop, then always run the cleanup above. A failing
|
|
535
|
+
// eviction (the wrapped agent close rejects when a plugin/LSP disposer
|
|
536
|
+
// fails) surfaces on a clean exit, but must not mask a control-loop
|
|
537
|
+
// error already unwinding -- so it is logged, not rethrown, in that case.
|
|
538
|
+
await runBodyThenCleanup(runControlLoop, cleanupControlLoop, (cause) => logger.error `workflow-child: warm-agent eviction failed while unwinding a control-loop error; surfacing the control-loop error, eviction failure: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
448
539
|
return {
|
|
449
540
|
resumedRunIds,
|
|
450
541
|
triggeredRunIds,
|
|
@@ -478,23 +569,15 @@ async function handleControlPayload(payload, ctx) {
|
|
|
478
569
|
ctx.triggeredRunIds.push(payload.data.runId);
|
|
479
570
|
return false;
|
|
480
571
|
}
|
|
481
|
-
//
|
|
482
|
-
//
|
|
483
|
-
//
|
|
484
|
-
// payload; the one-step workflow's first step defaults
|
|
485
|
-
// selector to `trigger.payload` (defineWorkflow's default-input
|
|
486
|
-
// convention), so the step input resolves to the inbound message
|
|
487
|
-
//
|
|
488
|
-
//
|
|
489
|
-
|
|
490
|
-
// real mailbox-ownership failure.
|
|
491
|
-
const triggerPayload = await resolveTriggerPayload({
|
|
492
|
-
substrate: ctx.bindings.substrate,
|
|
493
|
-
principal: ctx.bindings.principal,
|
|
494
|
-
workflowRunRepoId: ctx.bindings.workflowRunRepoId,
|
|
495
|
-
mailboxAddress: ctx.env.mailboxAddress,
|
|
496
|
-
messageId: payload.data.messageId,
|
|
497
|
-
});
|
|
572
|
+
// The supervisor resolved the inbound mail to the run's input (the
|
|
573
|
+
// conversation text plus references to attachment bytes it committed to
|
|
574
|
+
// the workflow-run substrate) and shipped it in the frame. It becomes
|
|
575
|
+
// the run's trigger payload; the one-step workflow's first step defaults
|
|
576
|
+
// its input selector to `trigger.payload` (defineWorkflow's default-input
|
|
577
|
+
// convention), so the step input resolves to the inbound message and
|
|
578
|
+
// `agent.send` receives it once its attachment references are resolved to
|
|
579
|
+
// bytes at send time.
|
|
580
|
+
const triggerPayload = payload.data.payload;
|
|
498
581
|
const env = buildRuntimeEnv({
|
|
499
582
|
runId: payload.data.runId,
|
|
500
583
|
bindings: ctx.bindings,
|
|
@@ -502,7 +585,11 @@ async function handleControlPayload(payload, ctx) {
|
|
|
502
585
|
authorize: ctx.authorize,
|
|
503
586
|
directors: ctx.directors,
|
|
504
587
|
suspendableChildHost: ctx.suspendableChildHost,
|
|
588
|
+
bodiesMap: ctx.bodiesMap,
|
|
589
|
+
loopBodyPreRewrite: ctx.loopBodyPreRewrite,
|
|
505
590
|
spawnChild: ctx.spawnChild,
|
|
591
|
+
loopFns: ctx.loopFns,
|
|
592
|
+
actionResolver: ctx.actionResolver,
|
|
506
593
|
clock: ctx.clock,
|
|
507
594
|
newId: ctx.newId,
|
|
508
595
|
drainController: ctx.drainController,
|
|
@@ -577,12 +664,16 @@ async function handleControlPayload(payload, ctx) {
|
|
|
577
664
|
return false;
|
|
578
665
|
}
|
|
579
666
|
case "credentials-updated": {
|
|
580
|
-
//
|
|
581
|
-
//
|
|
582
|
-
//
|
|
583
|
-
//
|
|
584
|
-
//
|
|
585
|
-
|
|
667
|
+
// Merge the delivery into the live cell (see `mergeCredentialDelivery`):
|
|
668
|
+
// materials upsert by credentialId, bindings by (consumer, handle), and
|
|
669
|
+
// `revoke` drops named credentialIds plus any binding referencing them.
|
|
670
|
+
// Merge rather than wholesale-replace because the cell has several
|
|
671
|
+
// independently-scoped producers, so a swap would evict another
|
|
672
|
+
// producer's credentials. The result is assigned in one atomic
|
|
673
|
+
// whole-object swap, so a concurrent reader never observes a torn cell.
|
|
674
|
+
// The secret stays on this ref only; nothing here copies it into a
|
|
675
|
+
// snapshot, event, or state.
|
|
676
|
+
ctx.credentialMaterialRef.current = mergeCredentialDelivery(ctx.credentialMaterialRef.current, payload.data.delivery, payload.data.revoke);
|
|
586
677
|
return false;
|
|
587
678
|
}
|
|
588
679
|
case "signal.deliver": {
|
|
@@ -755,6 +846,43 @@ async function handleControlPayload(payload, ctx) {
|
|
|
755
846
|
ctx.outboundMailBridge.handleResult(payload.data);
|
|
756
847
|
return false;
|
|
757
848
|
}
|
|
849
|
+
case "mailbox.notify": {
|
|
850
|
+
// Route the supervisor's new-mail notification to the child's watch
|
|
851
|
+
// registry so a step agent's `watch`/`mail_wait` observes the arrival.
|
|
852
|
+
// A notify that lands without a registry means no watcher on the child
|
|
853
|
+
// side asked for inbound events; log and drop rather than throwing so
|
|
854
|
+
// the runtime keeps progressing (mirrors the `outbound.result` arm).
|
|
855
|
+
if (ctx.mailboxWatchRegistry === undefined) {
|
|
856
|
+
logger.warn `workflow-child mailbox.notify received without a watch registry wired; mailbox=${payload.data.mailbox} uid=${String(payload.data.uid)} dropped`;
|
|
857
|
+
return false;
|
|
858
|
+
}
|
|
859
|
+
ctx.mailboxWatchRegistry.fire(payload.data.mailbox, {
|
|
860
|
+
type: "exists",
|
|
861
|
+
uid: payload.data.uid,
|
|
862
|
+
headers: payload.data.headers,
|
|
863
|
+
});
|
|
864
|
+
return false;
|
|
865
|
+
}
|
|
866
|
+
case "mailbox.mutate.request": {
|
|
867
|
+
// `mailbox.mutate.request` is the child->supervisor mailbox-mutation
|
|
868
|
+
// request frame; receiving one on the child's downstream side is a
|
|
869
|
+
// protocol violation in the same shape as a downstream
|
|
870
|
+
// `outbound.message`.
|
|
871
|
+
throw new Error("workflow-child received a `mailbox.mutate.request` frame on its inbound control channel; this is a child-only upstream payload");
|
|
872
|
+
}
|
|
873
|
+
case "mailbox.mutate.response": {
|
|
874
|
+
// Route the supervisor's applied-mutation result to the
|
|
875
|
+
// mailbox-mutation bridge if one is wired. A response that lands
|
|
876
|
+
// without an active bridge means a stale supervisor frame for which
|
|
877
|
+
// no awaiter exists; log and drop rather than throwing so the
|
|
878
|
+
// runtime keeps progressing (mirrors the `outbound.result` arm).
|
|
879
|
+
if (ctx.mailboxMutationBridge === undefined) {
|
|
880
|
+
logger.warn `workflow-child mailbox.mutate.response received without a bridge wired; requestId=${payload.data.requestId} dropped`;
|
|
881
|
+
return false;
|
|
882
|
+
}
|
|
883
|
+
ctx.mailboxMutationBridge.handleResult(payload.data);
|
|
884
|
+
return false;
|
|
885
|
+
}
|
|
758
886
|
case "substrate.merge.request": {
|
|
759
887
|
// Route the request to the substrate-write bridge if one is
|
|
760
888
|
// wired. A request that lands without an active bridge means a
|
|
@@ -823,6 +951,28 @@ async function handleControlPayload(payload, ctx) {
|
|
|
823
951
|
* shape; the substrate handle and per-deployment `RepoStore` adapter
|
|
824
952
|
* are shared across runs.
|
|
825
953
|
*/
|
|
954
|
+
/**
|
|
955
|
+
* Force-resolve every `action` handler ref reachable from these definitions
|
|
956
|
+
* against the resolver, so a missing action handler surfaces at establish
|
|
957
|
+
* rather than when the action is first invoked mid-run. Recurses into loop
|
|
958
|
+
* bodies (an action body is the common loop shape). The caller passes the
|
|
959
|
+
* lifted onTrigger/childWorkflow bodies separately, as with loop fns.
|
|
960
|
+
*/
|
|
961
|
+
function eagerlyResolveActionHandlers(definitions, actionResolver) {
|
|
962
|
+
const visit = (def) => {
|
|
963
|
+
for (const step of Object.values(def.steps)) {
|
|
964
|
+
if (step.kind === "action") {
|
|
965
|
+
// Throws (fail closed) if the handler names no export, or a non-function.
|
|
966
|
+
actionResolver(step.handler);
|
|
967
|
+
}
|
|
968
|
+
else if (step.kind === "loop") {
|
|
969
|
+
visit(step.body);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
};
|
|
973
|
+
for (const def of definitions)
|
|
974
|
+
visit(def);
|
|
975
|
+
}
|
|
826
976
|
function buildRuntimeEnv(args) {
|
|
827
977
|
const signalChannel = createWorkflowHostSignalChannel({
|
|
828
978
|
repoStore: args.bindings.substrate,
|
|
@@ -841,6 +991,17 @@ function buildRuntimeEnv(args) {
|
|
|
841
991
|
runId: args.runId,
|
|
842
992
|
ref: args.bindings.workflowRunRef,
|
|
843
993
|
});
|
|
994
|
+
// Reader for inbound-mail parts, a sibling of `blobs` over the same
|
|
995
|
+
// workflow-run repo. The step invoker resolves a `Mail` part's `ref` to its
|
|
996
|
+
// bytes through it at `agent.send` time; the supervisor committed the bytes
|
|
997
|
+
// before the trigger. Deployment-scoped (the ref encodes the owning run), so
|
|
998
|
+
// one reader resolves any run's parts.
|
|
999
|
+
const mailPartReader = createMailPartReader({
|
|
1000
|
+
substrate: args.bindings.substrate,
|
|
1001
|
+
repoId: args.bindings.workflowRunRepoId,
|
|
1002
|
+
principal: args.bindings.principal,
|
|
1003
|
+
ref: args.bindings.workflowRunRef,
|
|
1004
|
+
});
|
|
844
1005
|
// Wrap the step invoker so every `InferenceEvent` the harness emits
|
|
845
1006
|
// funnels through the per-run `onEvent` closure, which forwards
|
|
846
1007
|
// the event up the HMAC-authenticated event channel. The wrap is
|
|
@@ -849,19 +1010,27 @@ function buildRuntimeEnv(args) {
|
|
|
849
1010
|
// `ChildStepInvoker` shape (carries onEvent), so the workflow-
|
|
850
1011
|
// runtime never has to know an event firehose exists.
|
|
851
1012
|
const invokeStep = async (req) => {
|
|
852
|
-
return args.bindings.invokeStep(req, args.onEvent, args.authorize, args.warmCache, args.sourcesRef, args.credentialWiring);
|
|
1013
|
+
return args.bindings.invokeStep(req, args.onEvent, args.authorize, args.warmCache, args.sourcesRef, args.credentialWiring, mailPartReader);
|
|
853
1014
|
};
|
|
854
1015
|
// Adapt the host binding (which takes the run's `onEvent` sink) down to the
|
|
855
1016
|
// runtime's narrow `SpawnSuspendableChild` by injecting THIS run's event
|
|
856
1017
|
// funnel -- the same closure `invokeStep` forwards -- so a body's live
|
|
857
1018
|
// inference events ride the parent run's event channel to the hub stream
|
|
858
1019
|
// (and inherit its loud-on-failure logging), while the runtime env keeps the
|
|
859
|
-
// narrow contract with no event slot.
|
|
1020
|
+
// narrow contract with no event slot. The run's live credential-material cell
|
|
1021
|
+
// rides the same seam so the body's inference resolves its source secret
|
|
1022
|
+
// against the parent's current delivery, reached live on a rotation.
|
|
860
1023
|
const hostSuspendable = args.suspendableChildHost;
|
|
861
1024
|
const spawnSuspendableChild = hostSuspendable === undefined
|
|
862
1025
|
? undefined
|
|
863
|
-
: (spawnInput) => hostSuspendable(spawnInput, args.onEvent);
|
|
864
|
-
|
|
1026
|
+
: (spawnInput) => hostSuspendable(spawnInput, args.onEvent, args.credentialWiring.materialRef);
|
|
1027
|
+
// Same adaptation for the terminal childWorkflow seam: inject THIS run's event
|
|
1028
|
+
// funnel so a child's live inference events ride the parent run's channel, and
|
|
1029
|
+
// the live credential-material cell so the child's inference resolves its
|
|
1030
|
+
// source secret against the parent's current delivery, while the runtime env
|
|
1031
|
+
// keeps the narrow `SpawnChildWorkflow` (no event slot).
|
|
1032
|
+
const spawnChild = (spawnInput) => args.spawnChild(spawnInput, args.onEvent, args.credentialWiring.materialRef);
|
|
1033
|
+
const env = {
|
|
865
1034
|
repoStore: args.runtimeRepoStore,
|
|
866
1035
|
scheduler: args.bindings.scheduler,
|
|
867
1036
|
signalChannel,
|
|
@@ -869,7 +1038,15 @@ function buildRuntimeEnv(args) {
|
|
|
869
1038
|
directors: args.directors,
|
|
870
1039
|
authorize: args.authorize,
|
|
871
1040
|
invokeStep,
|
|
872
|
-
spawnChild
|
|
1041
|
+
spawnChild,
|
|
1042
|
+
// The deployment's addressable run. Its parks are registered with the hub
|
|
1043
|
+
// through the notify sink below, and a resolved decision is delivered back
|
|
1044
|
+
// onto this run's own channel.
|
|
1045
|
+
hasUpstreamSignalResolver: true,
|
|
1046
|
+
// Resolve a loop's `while`/`carry` refs against the closure's loop module.
|
|
1047
|
+
// Every ref was force-resolved at establish, so a lookup here cannot fail
|
|
1048
|
+
// for a definition that passed startup.
|
|
1049
|
+
loopFns: args.loopFns,
|
|
873
1050
|
// Wire the suspendable-child seam only when the host supplied it; a child
|
|
874
1051
|
// that never runs an onTrigger section omits the binding, and the runtime
|
|
875
1052
|
// body fails loud if a workflow reaches a section the env did not wire.
|
|
@@ -892,6 +1069,85 @@ function buildRuntimeEnv(args) {
|
|
|
892
1069
|
? { readParkedApprovalOps: args.bindings.readParkedApprovalOps }
|
|
893
1070
|
: {}),
|
|
894
1071
|
};
|
|
1072
|
+
// The suspendable-loop executor runs each iteration's body under THIS run's
|
|
1073
|
+
// inherited env (its real tool-bearing invokeStep, invokeAction, credentials
|
|
1074
|
+
// authorize, effect ledger, and durable shared repoStore/blobs), giving the
|
|
1075
|
+
// body only its own substrate-backed signal channel -- the inherited-env
|
|
1076
|
+
// iteration model plus park capability, distinct from an onTrigger body's
|
|
1077
|
+
// fresh capped env. Resolved from the bodies map by ref (loop
|
|
1078
|
+
// bodies were registered there at establish) and wrapped with this run's
|
|
1079
|
+
// event funnel. Assigned AFTER env construction because it closes over `env`.
|
|
1080
|
+
const loopIterationHost = createInMemorySpawnSuspendableChild({
|
|
1081
|
+
bodies: args.bodiesMap,
|
|
1082
|
+
runSuspendableChild: async (loopInput, _onEvent) => {
|
|
1083
|
+
// Materialize this iteration's own grants file BEFORE the body runs (i.e.
|
|
1084
|
+
// before `createLoopIterationHandle` drives the iteration's first event
|
|
1085
|
+
// append), so a `childWorkflow` grandchild spawned from the body reads it
|
|
1086
|
+
// as authority (the sidecar's runChild fails closed on a missing parent
|
|
1087
|
+
// grants file). This ordering is LOAD-BEARING: the grants write is
|
|
1088
|
+
// write-once and its shallow-prefix rebuild is safe only while the
|
|
1089
|
+
// iteration run's subtree is still empty -- see `capAndPersistChildGrants`.
|
|
1090
|
+
// The cap must walk the PRE-rewrite loop body (grandchild still inline);
|
|
1091
|
+
// the rewritten body in `bodiesMap` would skip the grandchild's resources.
|
|
1092
|
+
// Every loop body is registered in `loopBodyPreRewrite` at establish under
|
|
1093
|
+
// the same ref the runtime dispatches, so a miss is a defect -- fail loud
|
|
1094
|
+
// rather than silently skip, which would re-open the fail-closed
|
|
1095
|
+
// grandchild spawn. The BINDING is a separate sidecar-only seam: `runLocal`
|
|
1096
|
+
// keeps no per-run grants file (its grandchild spawn never fails closed),
|
|
1097
|
+
// so it omits the binding, and an absent binding leaves the iteration's
|
|
1098
|
+
// grants unmaterialized -- matching the in-process model with no disk
|
|
1099
|
+
// authority.
|
|
1100
|
+
const preRewriteBody = args.loopBodyPreRewrite.get(loopInput.definitionRef);
|
|
1101
|
+
if (preRewriteBody === undefined) {
|
|
1102
|
+
throw new Error(`workflow-child: loop iteration ${loopInput.childRunId} has no ` +
|
|
1103
|
+
`pre-rewrite body registered for ref ${loopInput.definitionRef}`);
|
|
1104
|
+
}
|
|
1105
|
+
await args.bindings.materializeLoopIterationGrants?.({
|
|
1106
|
+
parentRunId: loopInput.parentRunId,
|
|
1107
|
+
childRunId: loopInput.childRunId,
|
|
1108
|
+
definition: preRewriteBody,
|
|
1109
|
+
});
|
|
1110
|
+
const childSignalChannel = createWorkflowHostSignalChannel({
|
|
1111
|
+
repoStore: args.bindings.substrate,
|
|
1112
|
+
principal: args.bindings.principal,
|
|
1113
|
+
repoId: args.bindings.workflowRunRepoId,
|
|
1114
|
+
ref: args.bindings.workflowRunRef,
|
|
1115
|
+
runId: loopInput.childRunId,
|
|
1116
|
+
readState: () => emptyState(loopInput.childRunId),
|
|
1117
|
+
newId: () => args.newId("sig"),
|
|
1118
|
+
clock: args.clock,
|
|
1119
|
+
});
|
|
1120
|
+
return createLoopIterationHandle(env, {
|
|
1121
|
+
definition: loopInput.definition,
|
|
1122
|
+
childRunId: loopInput.childRunId,
|
|
1123
|
+
input: loopInput.input,
|
|
1124
|
+
depth: loopInput.depth,
|
|
1125
|
+
maxChildSpawnDepth: loopInput.maxChildSpawnDepth,
|
|
1126
|
+
...(loopInput.resumeFromEvents !== undefined
|
|
1127
|
+
? { resumeFromEvents: loopInput.resumeFromEvents }
|
|
1128
|
+
: {}),
|
|
1129
|
+
signal: loopInput.signal,
|
|
1130
|
+
signalChannel: childSignalChannel,
|
|
1131
|
+
cleanup: () => childSignalChannel.stop(),
|
|
1132
|
+
});
|
|
1133
|
+
},
|
|
1134
|
+
});
|
|
1135
|
+
env.spawnLoopIteration = (spawnInput) => loopIterationHost(spawnInput, args.onEvent);
|
|
1136
|
+
// Action handlers run against a per-run effect ledger. The ledger is
|
|
1137
|
+
// IN-MEMORY, and that is correct -- not a shortcut -- on the deployed store:
|
|
1138
|
+
// appends are immediate-durable single-ref commits, `runAction` flushes
|
|
1139
|
+
// StepStarted durably before the effect, and the runtime never re-invokes a
|
|
1140
|
+
// crashed action (a mid-action crash settles the step failed; a loop-body
|
|
1141
|
+
// action leaves a non-empty child log that fails the iteration loud rather
|
|
1142
|
+
// than re-running). So the ledger is never consulted across a crash; its
|
|
1143
|
+
// cross-crash exactly-once rests on that store-consistency invariant, which
|
|
1144
|
+
// the store layer owns. A durable ledger here would re-enforce a constraint
|
|
1145
|
+
// a lower layer already guarantees. Within a single invocation the ledger
|
|
1146
|
+
// still dedups a handler that performs the same effect twice.
|
|
1147
|
+
const effects = createInMemoryEffectLedger();
|
|
1148
|
+
env.effects = effects;
|
|
1149
|
+
env.invokeAction = createDefaultActionInvoker(args.authorize, effects, args.actionResolver);
|
|
1150
|
+
return env;
|
|
895
1151
|
}
|
|
896
1152
|
/**
|
|
897
1153
|
* Forward a control-plane suspension to the supervisor over the upstream
|
|
@@ -1046,30 +1302,6 @@ function reclaimRunStorageIfCold(opts) {
|
|
|
1046
1302
|
logger.warn `workflow-step-state cleanup failed for runId=${opts.runId}: ${message}`;
|
|
1047
1303
|
});
|
|
1048
1304
|
}
|
|
1049
|
-
/**
|
|
1050
|
-
* Resolve the run's trigger payload from the inbound mail message the
|
|
1051
|
-
* supervisor moved to the claim-check processing queue. Reads the
|
|
1052
|
-
* processing entry by messageId (a read-only snapshot of the
|
|
1053
|
-
* `refs/heads/events` tip that cannot race the supervisor's
|
|
1054
|
-
* `markConsumed` write), decodes the inlined raw MIME bytes, and
|
|
1055
|
-
* extracts the conversation text the agent's `agent.send` receives.
|
|
1056
|
-
*
|
|
1057
|
-
* Defensive: a missing processing entry, an entry with no inlined
|
|
1058
|
-
* bytes, or unparseable mail all throw. The run cannot proceed without
|
|
1059
|
-
* its input, and a placeholder would mask a mailbox-ownership failure.
|
|
1060
|
-
*/
|
|
1061
|
-
async function resolveTriggerPayload(args) {
|
|
1062
|
-
const entry = await readProcessingEntry(args.substrate, args.principal, args.workflowRunRepoId, args.mailboxAddress, args.messageId);
|
|
1063
|
-
if (entry === null) {
|
|
1064
|
-
throw new Error(`workflow-child trigger.fire: no claim-check processing entry for messageId ${args.messageId} at ${args.mailboxAddress}; the run has no input to deliver to the agent`);
|
|
1065
|
-
}
|
|
1066
|
-
const rawMessageBase64 = entry.envelope.rawMessage;
|
|
1067
|
-
if (rawMessageBase64 === undefined) {
|
|
1068
|
-
throw new Error(`workflow-child trigger.fire: processing entry for messageId ${args.messageId} carries no inlined rawMessage; the supervisor must inline the inbound mail bytes for the child to deliver them as the step input`);
|
|
1069
|
-
}
|
|
1070
|
-
const raw = base64Decode(rawMessageBase64);
|
|
1071
|
-
return extractConversationText(raw, args.messageId);
|
|
1072
|
-
}
|
|
1073
1305
|
function defaultClock() {
|
|
1074
1306
|
return new Date();
|
|
1075
1307
|
}
|
|
@@ -59,8 +59,9 @@ export interface CreateChildSubstrateWriteBridgeOpts {
|
|
|
59
59
|
}
|
|
60
60
|
/**
|
|
61
61
|
* Construct the child-side substrate-write bridge. Pending writes
|
|
62
|
-
* live in
|
|
63
|
-
* when the supervisor's matching
|
|
62
|
+
* live in the shared pending-request core keyed by `requestId`; the
|
|
63
|
+
* bridge resolves the awaiter when the supervisor's matching
|
|
64
|
+
* `substrate.write.response` lands.
|
|
64
65
|
*
|
|
65
66
|
* The supervisor may emit zero or more `substrate.merge.request`
|
|
66
67
|
* frames per pending write (the supervisor's merge callback may run
|