@acpus/runtime 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/agent-telemetry.d.ts +49 -0
  2. package/dist/agent-telemetry.d.ts.map +1 -0
  3. package/dist/agent-telemetry.js +255 -0
  4. package/dist/agent-telemetry.js.map +1 -0
  5. package/dist/artifacts.d.ts +16 -1
  6. package/dist/artifacts.d.ts.map +1 -1
  7. package/dist/artifacts.js +92 -29
  8. package/dist/artifacts.js.map +1 -1
  9. package/dist/attempt-artifacts.d.ts +35 -0
  10. package/dist/attempt-artifacts.d.ts.map +1 -0
  11. package/dist/attempt-artifacts.js +90 -0
  12. package/dist/attempt-artifacts.js.map +1 -0
  13. package/dist/client.d.ts +32 -2
  14. package/dist/client.d.ts.map +1 -1
  15. package/dist/client.js +35 -2
  16. package/dist/client.js.map +1 -1
  17. package/dist/executors/agent.d.ts +7 -3
  18. package/dist/executors/agent.d.ts.map +1 -1
  19. package/dist/executors/agent.js +58 -61
  20. package/dist/executors/agent.js.map +1 -1
  21. package/dist/executors/mock-program.d.ts +2 -0
  22. package/dist/executors/mock-program.d.ts.map +1 -1
  23. package/dist/executors/mock-program.js +27 -6
  24. package/dist/executors/mock-program.js.map +1 -1
  25. package/dist/executors/output-preview.d.ts +2 -0
  26. package/dist/executors/output-preview.d.ts.map +1 -0
  27. package/dist/executors/output-preview.js +20 -0
  28. package/dist/executors/output-preview.js.map +1 -0
  29. package/dist/executors/program.d.ts +10 -3
  30. package/dist/executors/program.d.ts.map +1 -1
  31. package/dist/executors/program.js +39 -4
  32. package/dist/executors/program.js.map +1 -1
  33. package/dist/executors/types.d.ts +2 -0
  34. package/dist/executors/types.d.ts.map +1 -1
  35. package/dist/fork.d.ts +80 -0
  36. package/dist/fork.d.ts.map +1 -0
  37. package/dist/fork.js +203 -0
  38. package/dist/fork.js.map +1 -0
  39. package/dist/index.d.ts +9 -4
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +6 -3
  42. package/dist/index.js.map +1 -1
  43. package/dist/interpreter.d.ts +7 -49
  44. package/dist/interpreter.d.ts.map +1 -1
  45. package/dist/interpreter.js +232 -461
  46. package/dist/interpreter.js.map +1 -1
  47. package/dist/keys.d.ts +50 -2
  48. package/dist/keys.d.ts.map +1 -1
  49. package/dist/keys.js +261 -1
  50. package/dist/keys.js.map +1 -1
  51. package/dist/run-control.d.ts +45 -0
  52. package/dist/run-control.d.ts.map +1 -0
  53. package/dist/run-control.js +255 -0
  54. package/dist/run-control.js.map +1 -0
  55. package/dist/state-machine.d.ts +5 -1
  56. package/dist/state-machine.d.ts.map +1 -1
  57. package/dist/state-machine.js +16 -1
  58. package/dist/state-machine.js.map +1 -1
  59. package/dist/store.d.ts +29 -2
  60. package/dist/store.d.ts.map +1 -1
  61. package/dist/store.js +129 -28
  62. package/dist/store.js.map +1 -1
  63. package/dist/supervisor-app.d.ts.map +1 -1
  64. package/dist/supervisor-app.js +145 -42
  65. package/dist/supervisor-app.js.map +1 -1
  66. package/dist/supervisor-discovery.d.ts +1 -0
  67. package/dist/supervisor-discovery.d.ts.map +1 -1
  68. package/dist/supervisor-discovery.js +9 -15
  69. package/dist/supervisor-discovery.js.map +1 -1
  70. package/dist/types.d.ts +94 -5
  71. package/dist/types.d.ts.map +1 -1
  72. package/dist/workflow-outputs.d.ts +9 -0
  73. package/dist/workflow-outputs.d.ts.map +1 -0
  74. package/dist/workflow-outputs.js +59 -0
  75. package/dist/workflow-outputs.js.map +1 -0
  76. package/package.json +3 -3
@@ -1,58 +1,18 @@
1
- import { parseDurationMs, compileWorkflow } from "@acpus/core";
1
+ import { parseDurationMs, compileWorkflow, hashIrNode } from "@acpus/core";
2
2
  import { readFileSync } from "node:fs";
3
3
  import { dirname, resolve } from "node:path";
4
4
  import { ExpressionEvaluator } from "./evaluator.js";
5
- import { resolveNodeKey, sanitizeValue } from "./keys.js";
6
- import { canTransition, transition, createInitialNodeState, resetFailedForRetry, resetRunningForCrashRecovery, resetAwaitingForCrashRecovery, resetPausedForRunResume } from "./state-machine.js";
5
+ import { resolveNodeKey, staticNodePathFromKey, withNodeKeyPrefix } from "./keys.js";
6
+ import { canTransition, transition, createInitialNodeState } from "./state-machine.js";
7
7
  import { ArtifactStore } from "./artifacts.js";
8
- import { renderAgentRequestPrompt } from "./executors/agent.js";
8
+ import { AttemptArtifactRecorder } from "./attempt-artifacts.js";
9
+ import { renderAgentRequestPrompt, renderAgentSessionKey } from "./executors/agent.js";
9
10
  import { validateInput } from "./validate-input.js";
11
+ import { RunControl, abortedNodeError, isPausedContinuationState } from "./run-control.js";
12
+ import { evaluateTemplatedValue, evaluateWorkflowOutputs } from "./workflow-outputs.js";
10
13
  import { randomBytes } from "node:crypto";
11
14
  import pLimit from "p-limit";
12
- const PAUSED_ABORT_ERROR = "Aborted: paused";
13
- function abortedNodeError(state) {
14
- return state === "paused" ? PAUSED_ABORT_ERROR : "Aborted: cancelled";
15
- }
16
- function isPausedContinuationState(state) {
17
- return state?.state === "pending" && state.attempt > 0 && state.error === PAUSED_ABORT_ERROR;
18
- }
19
- function pushUnique(target, refs) {
20
- if (!refs)
21
- return;
22
- const seen = new Set(target);
23
- for (const ref of refs) {
24
- if (!seen.has(ref)) {
25
- target.push(ref);
26
- seen.add(ref);
27
- }
28
- }
29
- target.sort(compareAttemptArtifactRefs);
30
- }
31
- function mergeStderrDiagnostics(stderr, diagnostics) {
32
- if (diagnostics.length === 0)
33
- return stderr;
34
- const diagnosticText = diagnostics.map((line) => `[acpus] ${line}`).join("\n");
35
- return stderr && stderr.length > 0 ? `${stderr}\n${diagnosticText}` : diagnosticText;
36
- }
37
- function compareAttemptArtifactRefs(a, b) {
38
- const left = attemptArtifactSortKey(a);
39
- const right = attemptArtifactSortKey(b);
40
- if (!left || !right)
41
- return 0;
42
- return left.attempt - right.attempt || left.kind - right.kind;
43
- }
44
- function attemptArtifactSortKey(ref) {
45
- const match = /attempt-(\d+)\.(prompt\.md|transcript\.jsonl|response\.md|stderr\.log)$/.exec(ref);
46
- if (!match)
47
- return undefined;
48
- const kindOrder = {
49
- "prompt.md": 0,
50
- "transcript.jsonl": 1,
51
- "response.md": 2,
52
- "stderr.log": 3
53
- };
54
- return { attempt: Number(match[1]), kind: kindOrder[match[2]] ?? 99 };
55
- }
15
+ import { AgentTelemetryAccumulator, upsertAgentAttemptTelemetry } from "./agent-telemetry.js";
56
16
  /**
57
17
  * The core IR interpreter that drives state transitions, orchestrates
58
18
  * execution, and persists state.
@@ -63,28 +23,24 @@ export class WorkflowInterpreter {
63
23
  agentExecutor;
64
24
  programExecutor;
65
25
  artifactStore;
26
+ attemptArtifacts;
27
+ runControl;
66
28
  maxConcurrency;
67
29
  allowedSourceRoots;
68
30
  sleep;
69
- /** Active abort controllers keyed by "runId:nodeKey" for pause/cancel support */
70
- abortControllers = new Map();
71
- /** Intent of an in-flight abort keyed by "runId:nodeKey" (pause vs cancel). */
72
- abortIntents = new Map();
73
31
  /** Pending human-decision resolvers for Approval Gates awaiting a signal,
74
32
  * keyed by "runId:nodeKey". An entry exists only while a node is `awaiting`. */
75
33
  approvalResolvers = new Map();
76
34
  /** Absolute paths of subworkflow specs currently on the execution stack (cycle guard). */
77
35
  subworkflowStack = new Set();
78
- /** Scheduling guards for Run-level pause/cancel, keyed by runId.
79
- * Per-runId to prevent leakage across runs sharing the same interpreter. */
80
- schedulingPaused = new Map();
81
- schedulingCancelled = new Map();
82
36
  constructor(store, agentExecutor, programExecutor, options) {
83
37
  this.store = store;
84
38
  this.evaluator = new ExpressionEvaluator({ nowTimestamp: options?.nowTimestamp });
85
39
  this.agentExecutor = agentExecutor;
86
40
  this.programExecutor = programExecutor;
87
41
  this.artifactStore = new ArtifactStore(store.getBaseDir());
42
+ this.attemptArtifacts = new AttemptArtifactRecorder(this.artifactStore);
43
+ this.runControl = new RunControl(store);
88
44
  this.maxConcurrency = options?.maxConcurrency ?? 10;
89
45
  this.allowedSourceRoots = options?.allowedSourceRoots?.map((root) => resolve(root)) ?? [];
90
46
  this.sleep = options?.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
@@ -106,7 +62,9 @@ export class WorkflowInterpreter {
106
62
  const runId = opts.runId ?? generateRunId();
107
63
  return this.store.initRun(runId, ir, validatedInput, {
108
64
  workflowRef: opts.workflowRef,
109
- workflowSourcePath: opts.workflowSourcePath
65
+ workflowSourcePath: opts.workflowSourcePath,
66
+ agentOverrides: opts.agentOverrides,
67
+ submissionWarnings: opts.submissionWarnings
110
68
  });
111
69
  }
112
70
  /**
@@ -115,7 +73,10 @@ export class WorkflowInterpreter {
115
73
  async runToCompletion(ir, opts, runId) {
116
74
  const meta = this.store.readRunMeta(runId);
117
75
  try {
118
- await this.executeNode(ir.root, this.buildContext(opts.input, runId), runId, {});
76
+ const ctx = this.buildContext(opts.input, runId);
77
+ await this.executeNode(ir.root, ctx, runId, {});
78
+ meta.output = evaluateWorkflowOutputs(ir, ctx, this.evaluator);
79
+ meta.error = undefined;
119
80
  meta.status = "completed";
120
81
  }
121
82
  catch (error) {
@@ -128,14 +89,20 @@ export class WorkflowInterpreter {
128
89
  }
129
90
  else {
130
91
  meta.status = "failed";
131
- void error;
92
+ meta.output = undefined;
93
+ meta.error = errorMessage(error);
94
+ if (rootState?.state === "completed") {
95
+ rootState.state = "failed";
96
+ rootState.error = meta.error;
97
+ rootState.completedAt = new Date().toISOString();
98
+ this.store.writeNodeState(runId, rootState);
99
+ }
132
100
  }
133
101
  }
134
102
  finally {
135
103
  // Clean up per-runId scheduling guards when the run settles,
136
104
  // so they don't leak memory across runs.
137
- this.schedulingPaused.delete(runId);
138
- this.schedulingCancelled.delete(runId);
105
+ this.runControl.clearSchedulingGuards(runId);
139
106
  }
140
107
  meta.updatedAt = new Date().toISOString();
141
108
  this.store.writeRunMeta(runId, meta);
@@ -150,16 +117,7 @@ export class WorkflowInterpreter {
150
117
  * for a fresh human decision on re-execution.
151
118
  */
152
119
  recoverStaleNodes(runId) {
153
- for (const nodeState of this.store.listNodeStates(runId)) {
154
- if (nodeState.state === "running") {
155
- nodeState.state = resetRunningForCrashRecovery(nodeState.state);
156
- this.store.writeNodeState(runId, nodeState);
157
- }
158
- else if (nodeState.state === "awaiting") {
159
- nodeState.state = resetAwaitingForCrashRecovery(nodeState.state);
160
- this.store.writeNodeState(runId, nodeState);
161
- }
162
- }
120
+ this.runControl.recoverStaleNodes(runId);
163
121
  }
164
122
  /**
165
123
  * Deterministically replay a persisted Run and verify its reconstructed
@@ -225,7 +183,7 @@ export class WorkflowInterpreter {
225
183
  */
226
184
  replayNode(node, ctx, runId, dynamic, keyPrefix, recorded, reached, evaluator) {
227
185
  const resolved = resolveNodeKey(node.keyTemplate, dynamic);
228
- const nodeKey = keyPrefix ? `${keyPrefix}/${resolved}` : resolved;
186
+ const nodeKey = withNodeKeyPrefix(keyPrefix, resolved);
229
187
  const rec = recorded.get(nodeKey);
230
188
  // A node absent from the recording was never reached on the original walk
231
189
  // (e.g. an untaken switch branch); skip it so we don't fabricate topology.
@@ -245,7 +203,7 @@ export class WorkflowInterpreter {
245
203
  break;
246
204
  case "parallel":
247
205
  (node.children ?? []).forEach((child, index) => {
248
- const branchDynamic = { ...dynamic, parallelBranchId: String(index) };
206
+ const branchDynamic = nestedParallelBranchDynamic(dynamic, index);
249
207
  this.replayNode(child, ctx, runId, branchDynamic, keyPrefix, recorded, reached, evaluator);
250
208
  });
251
209
  break;
@@ -290,7 +248,7 @@ export class WorkflowInterpreter {
290
248
  this.replayNode(child, loopCtx, runId, loopDynamic, keyPrefix, recorded, reached, evaluator);
291
249
  if (reached.size > before)
292
250
  anyChildReached = true;
293
- const childKey = `${keyPrefix ? `${keyPrefix}/` : ""}${resolveNodeKey(child.keyTemplate, loopDynamic)}`;
251
+ const childKey = withNodeKeyPrefix(keyPrefix, resolveNodeKey(child.keyTemplate, loopDynamic));
294
252
  lastOutput = recorded.get(childKey)?.output ?? lastOutput;
295
253
  }
296
254
  if (!anyChildReached)
@@ -313,81 +271,13 @@ export class WorkflowInterpreter {
313
271
  break;
314
272
  }
315
273
  }
316
- /** Pause one running node as part of a Run-level pause operation. */
317
- pauseRunningNode(runId, nodeKey) {
318
- const state = this.store.readNodeState(runId, nodeKey);
319
- if (!state || !canTransition(state.state, "paused")) {
320
- return;
321
- }
322
- this.abortIntents.set(`${runId}:${nodeKey}`, "paused");
323
- const controller = this.abortControllers.get(`${runId}:${nodeKey}`);
324
- if (controller) {
325
- controller.abort();
326
- }
327
- state.state = transition(state.state, "paused");
328
- this.store.writeNodeState(runId, state);
329
- }
330
- /** Cancel one materialized node as part of Run-level cancel or fail-fast. */
331
- cancelMaterializedNode(runId, nodeKey) {
332
- this.abortIntents.set(`${runId}:${nodeKey}`, "cancelled");
333
- const controller = this.abortControllers.get(`${runId}:${nodeKey}`);
334
- if (controller) {
335
- controller.abort();
336
- }
337
- const state = this.store.readNodeState(runId, nodeKey);
338
- if (state && canTransition(state.state, "cancelled")) {
339
- state.state = transition(state.state, "cancelled");
340
- this.store.writeNodeState(runId, state);
341
- }
342
- }
343
- /** Resolve the operator intent for an in-flight abort; defaults to paused. */
344
- abortIntent(runId, nodeKey) {
345
- return this.abortIntents.get(`${runId}:${nodeKey}`) ?? "paused";
346
- }
347
- /** True if the node has been persisted as terminal `cancelled` (e.g. by
348
- * fail-fast). Used to avoid clobbering it when a still-running leaf later
349
- * resolves/rejects. */
350
- readAbortedStateOnDisk(runId, nodeKey) {
351
- const state = this.store.readNodeState(runId, nodeKey)?.state;
352
- return state === "paused" || state === "cancelled" ? state : undefined;
353
- }
354
- isStaleAttemptOnDisk(runId, nodeKey, attempt, startedAt) {
355
- const current = this.store.readNodeState(runId, nodeKey);
356
- return current !== undefined && current.attempt > attempt && current.startedAt !== startedAt;
357
- }
358
- syncInFrameAttemptFromDisk(runId, nodeKey, state) {
359
- const current = this.store.readNodeState(runId, nodeKey);
360
- if (current !== undefined && current.startedAt === state.startedAt && current.attempt > state.attempt) {
361
- state.attempt = current.attempt;
362
- }
363
- }
364
274
  /**
365
275
  * Retry a failed executable Node. This is a local repair operation for the
366
276
  * target executable only; Run-level retry remains the operation that restores
367
277
  * Workflow progress from failed composite ancestors.
368
278
  */
369
279
  async retryNode(runId, nodeKey) {
370
- const state = this.store.readNodeState(runId, nodeKey);
371
- if (!state) {
372
- throw new Error(`Node ${nodeKey} not found in run ${runId}`);
373
- }
374
- if (state.state !== "failed") {
375
- throw new Error(`Cannot retry node ${nodeKey} in state '${state.state}': only failed executable nodes are retryable`);
376
- }
377
- if (state.kind !== "run.agent" && state.kind !== "run.program") {
378
- throw new Error(`Cannot retry node ${nodeKey}: only failed executable nodes are retryable`);
379
- }
380
- const meta = this.store.readRunMeta(runId);
381
- if (!meta)
382
- throw new Error(`Run ${runId} not found`);
383
- if (meta.status !== "failed") {
384
- throw new Error(`Cannot retry node ${nodeKey}: node retry is accepted only when the Run is failed`);
385
- }
386
- const ir = this.store.readIr(runId);
387
- const input = this.store.readInput(runId);
388
- if (!ir || !input) {
389
- throw new Error(`Cannot retry node ${nodeKey}: run ${runId} has no persisted IR or input`);
390
- }
280
+ const { state, ir, input } = this.runControl.prepareNodeRetry(runId, nodeKey);
391
281
  // Resolve the node IR *before* mutating state. Subworkflow child IR is
392
282
  // compiled on demand and never persisted, so a child key like
393
283
  // `workflow/sub/child` is unresolvable here; reject explicitly rather than
@@ -398,9 +288,7 @@ export class WorkflowInterpreter {
398
288
  }
399
289
  // Control-plane reset (failed → pending), not a business-lifecycle
400
290
  // transition. `attempt` is incremented by executeNode.
401
- state.state = resetFailedForRetry(state.state);
402
- state.error = undefined;
403
- this.store.writeNodeState(runId, state);
291
+ this.runControl.resetNodeForRetry(runId, state);
404
292
  const ctx = this.buildContext(input, runId);
405
293
  this.populateStepOutputs(runId, ctx);
406
294
  // Restore the parent dynamic value-context captured at first execution.
@@ -416,22 +304,7 @@ export class WorkflowInterpreter {
416
304
  * pauses all running nodes, and updates Run metadata to `paused`.
417
305
  */
418
306
  pauseRun(runId) {
419
- const meta = this.store.readRunMeta(runId);
420
- if (!meta)
421
- throw new Error(`Run ${runId} not found`);
422
- if (meta.status !== "running") {
423
- throw new Error(`Cannot pause a run in state '${meta.status}'`);
424
- }
425
- this.schedulingPaused.set(runId, true);
426
- // Pause all currently running nodes
427
- for (const nodeState of this.store.listNodeStates(runId)) {
428
- if (nodeState.state === "running") {
429
- this.pauseRunningNode(runId, nodeState.nodeKey);
430
- }
431
- }
432
- meta.status = "paused";
433
- meta.updatedAt = new Date().toISOString();
434
- this.store.writeRunMeta(runId, meta);
307
+ this.runControl.pauseRun(runId);
435
308
  }
436
309
  /**
437
310
  * Cancel an entire Run. Validates Run is `running` or `paused`, sets
@@ -439,78 +312,14 @@ export class WorkflowInterpreter {
439
312
  * and updates Run metadata to `cancelled`.
440
313
  */
441
314
  cancelRun(runId) {
442
- const meta = this.store.readRunMeta(runId);
443
- if (!meta)
444
- throw new Error(`Run ${runId} not found`);
445
- if (meta.status !== "running" && meta.status !== "paused") {
446
- throw new Error(`Cannot cancel a run in state '${meta.status}'`);
447
- }
448
- this.schedulingCancelled.set(runId, true);
449
- // Override any prior pause abort intents to "cancelled" so that
450
- // runToCompletion's catch block writes "cancelled" (not "paused").
451
- for (const [key, intent] of this.abortIntents) {
452
- if (key.startsWith(`${runId}:`) && intent === "paused") {
453
- this.abortIntents.set(key, "cancelled");
454
- }
455
- }
456
- // Cancel all currently running or awaiting nodes (abort the in-flight leaf
457
- // or the pending approval wait, then transition to cancelled).
458
- for (const nodeState of this.store.listNodeStates(runId)) {
459
- if (nodeState.state === "running" || nodeState.state === "awaiting") {
460
- this.cancelMaterializedNode(runId, nodeState.nodeKey);
461
- }
462
- }
463
- // Also transition paused nodes to cancelled (a prior pauseRun may have
464
- // left nodes in "paused" state; cancel supersedes pause).
465
- for (const nodeState of this.store.listNodeStates(runId)) {
466
- if (nodeState.state === "paused") {
467
- if (canTransition(nodeState.state, "cancelled")) {
468
- nodeState.state = transition(nodeState.state, "cancelled");
469
- this.store.writeNodeState(runId, nodeState);
470
- }
471
- }
472
- }
473
- // Mark all pending nodes as cancelled (do NOT materialize unvisited nodes)
474
- for (const nodeState of this.store.listNodeStates(runId)) {
475
- if (nodeState.state === "pending") {
476
- if (canTransition(nodeState.state, "cancelled")) {
477
- nodeState.state = transition(nodeState.state, "cancelled");
478
- this.store.writeNodeState(runId, nodeState);
479
- }
480
- }
481
- }
482
- meta.status = "cancelled";
483
- meta.updatedAt = new Date().toISOString();
484
- this.store.writeRunMeta(runId, meta);
315
+ this.runControl.cancelRun(runId);
485
316
  }
486
317
  /**
487
318
  * Resume an entire paused Run. Validates Run is `paused`, clears scheduling
488
319
  * guards, recovers stale nodes, and re-executes from root.
489
320
  */
490
321
  async resumeRun(runId) {
491
- const meta = this.store.readRunMeta(runId);
492
- if (!meta)
493
- throw new Error(`Run ${runId} not found`);
494
- if (meta.status !== "paused") {
495
- throw new Error(`Cannot resume a run in state '${meta.status}'`);
496
- }
497
- // Clear scheduling guards for this run
498
- this.schedulingPaused.delete(runId);
499
- this.schedulingCancelled.delete(runId);
500
- // Recover stale running nodes back to pending
501
- this.recoverStaleNodes(runId);
502
- // Also reset paused nodes back to pending so runToCompletion can re-execute them.
503
- // Without this, executeNode sees the 'paused' state and throws NodeAbortedError,
504
- // making the run permanently stuck.
505
- for (const nodeState of this.store.listNodeStates(runId)) {
506
- if (nodeState.state === "paused") {
507
- nodeState.state = resetPausedForRunResume(nodeState.state);
508
- this.store.writeNodeState(runId, nodeState);
509
- }
510
- }
511
- meta.status = "running";
512
- meta.updatedAt = new Date().toISOString();
513
- this.store.writeRunMeta(runId, meta);
322
+ this.runControl.resumeRun(runId);
514
323
  }
515
324
  /**
516
325
  * Retry a failed Run. Validates Run is `failed`, resets failed materialized
@@ -518,33 +327,7 @@ export class WorkflowInterpreter {
518
327
  * re-executes from root. Same Run ID, no new Run.
519
328
  */
520
329
  retryRun(runId) {
521
- const meta = this.store.readRunMeta(runId);
522
- if (!meta)
523
- throw new Error(`Run ${runId} not found`);
524
- if (meta.status !== "failed") {
525
- throw new Error(`Cannot retry a run in state '${meta.status}'`);
526
- }
527
- // Clear scheduling guards for this run
528
- this.schedulingPaused.delete(runId);
529
- this.schedulingCancelled.delete(runId);
530
- // Reset failed and paused materialized nodes to pending (preserve completed).
531
- // Paused nodes can exist in a "failed" run if it was paused before a sibling
532
- // failed (e.g. parallel lane failure while another lane was paused).
533
- for (const nodeState of this.store.listNodeStates(runId)) {
534
- if (nodeState.state === "failed") {
535
- nodeState.state = resetFailedForRetry(nodeState.state);
536
- nodeState.error = undefined;
537
- this.store.writeNodeState(runId, nodeState);
538
- }
539
- else if (nodeState.state === "paused") {
540
- nodeState.state = resetPausedForRunResume(nodeState.state);
541
- this.store.writeNodeState(runId, nodeState);
542
- }
543
- }
544
- meta.status = "running";
545
- meta.runAttempt++;
546
- meta.updatedAt = new Date().toISOString();
547
- this.store.writeRunMeta(runId, meta);
330
+ this.runControl.retryRun(runId);
548
331
  }
549
332
  // ─── Node execution dispatch ──────────────────────────────────
550
333
  async executeNode(node, ctx, runId, dynamic, keyPrefix, isContinuation, overrideNodeKey) {
@@ -552,7 +335,7 @@ export class WorkflowInterpreter {
552
335
  // node's stable identity (and thus the agent's acpx session name) survives
553
336
  // across loop/fanout/lane/subworkflow dynamics that are not re-derived here.
554
337
  const resolved = resolveNodeKey(node.keyTemplate, dynamic);
555
- const nodeKey = overrideNodeKey ?? (keyPrefix ? `${keyPrefix}/${resolved}` : resolved);
338
+ const nodeKey = overrideNodeKey ?? withNodeKeyPrefix(keyPrefix, resolved);
556
339
  // Check if already completed (from prior run)
557
340
  const existing = this.store.readNodeState(runId, nodeKey);
558
341
  if (existing?.state === "completed") {
@@ -564,25 +347,20 @@ export class WorkflowInterpreter {
564
347
  }
565
348
  const continuation = Boolean(isContinuation || isPausedContinuationState(existing));
566
349
  // Initialize state
567
- const state = existing ?? createInitialNodeState(nodeKey, node.id, node.kind);
350
+ const definitionHash = hashIrNode(node);
351
+ const state = existing ?? createInitialNodeState(nodeKey, node.id, node.kind, definitionHash);
352
+ if (!state.definitionHash)
353
+ state.definitionHash = definitionHash;
568
354
  if (state.state === "failed") {
569
355
  throw new Error(`Node ${nodeKey} is in failed state`);
570
356
  }
571
- // Run-level scheduling guards: if this Run is paused or cancelled,
572
- // don't schedule new work.
573
- if (this.schedulingPaused.get(runId) && state.state === "pending") {
574
- throw new NodeAbortedError(nodeKey, "paused");
575
- }
576
- if (this.schedulingCancelled.get(runId) && state.state === "pending") {
577
- if (canTransition(state.state, "cancelled")) {
578
- state.state = transition(state.state, "cancelled");
579
- this.store.writeNodeState(runId, state);
580
- }
581
- throw new NodeAbortedError(nodeKey, "cancelled");
357
+ const schedulingAbort = this.runControl.applySchedulingGuard(runId, state);
358
+ if (schedulingAbort) {
359
+ throw new NodeAbortedError(nodeKey, schedulingAbort);
582
360
  }
583
361
  // Set up abort controller
584
362
  const controller = new AbortController();
585
- this.abortControllers.set(`${runId}:${nodeKey}`, controller);
363
+ this.runControl.registerAbortController(runId, nodeKey, controller);
586
364
  // Transition to running
587
365
  if (canTransition(state.state, "running")) {
588
366
  state.state = transition(state.state, "running");
@@ -614,8 +392,6 @@ export class WorkflowInterpreter {
614
392
  const leaf = await this.executeAgent(node, ctx, runId, controller.signal, nodeKey, continuation);
615
393
  output = leaf.output;
616
394
  artifactRefs = leaf.artifactRefs;
617
- if (leaf.renderedPrompt)
618
- state.renderedPrompt = leaf.renderedPrompt;
619
395
  break;
620
396
  }
621
397
  case "run.program": {
@@ -625,10 +401,10 @@ export class WorkflowInterpreter {
625
401
  break;
626
402
  }
627
403
  case "parallel":
628
- output = await this.executeParallel(node, ctx, runId, dynamic, keyPrefix);
404
+ output = await this.executeParallel(node, ctx, runId, dynamic, nodeKey, keyPrefix);
629
405
  break;
630
406
  case "fanout":
631
- output = await this.executeFanout(node, ctx, runId, dynamic, keyPrefix);
407
+ output = await this.executeFanout(node, ctx, runId, dynamic, nodeKey, keyPrefix);
632
408
  break;
633
409
  case "switch":
634
410
  output = await this.executeSwitch(node, ctx, runId, dynamic, keyPrefix);
@@ -653,15 +429,16 @@ export class WorkflowInterpreter {
653
429
  }
654
430
  // If a Run-level resume already re-entered this node, a late result from
655
431
  // the old attempt must not overwrite the newer attempt's state.
656
- if (this.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
432
+ if (this.runControl.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
657
433
  return output;
658
434
  }
659
- this.syncInFrameAttemptFromDisk(runId, nodeKey, state);
435
+ this.runControl.syncInFrameAttemptFromDisk(runId, nodeKey, state);
436
+ this.syncAgentTelemetryFromDisk(runId, nodeKey, state);
660
437
  // Transition to completed unless an external pause/cancel already changed
661
438
  // this node's state on disk while we were awaiting the leaf.
662
- const abortedState = this.readAbortedStateOnDisk(runId, nodeKey);
439
+ const abortedState = this.runControl.readAbortedStateOnDisk(runId, nodeKey);
663
440
  if (abortedState) {
664
- throw new NodeAbortedError(nodeKey, abortedState, artifactRefs, output, state.renderedPrompt);
441
+ throw new NodeAbortedError(nodeKey, abortedState, artifactRefs, output);
665
442
  }
666
443
  state.state = "completed";
667
444
  state.error = undefined;
@@ -682,29 +459,29 @@ export class WorkflowInterpreter {
682
459
  throw error;
683
460
  }
684
461
  if (error instanceof NodeAbortedError) {
685
- if (this.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
462
+ if (this.runControl.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
686
463
  throw error;
687
464
  }
688
- this.syncInFrameAttemptFromDisk(runId, nodeKey, state);
465
+ this.runControl.syncInFrameAttemptFromDisk(runId, nodeKey, state);
689
466
  // Transition this node to the same state as the child that was aborted.
690
467
  state.state = error.state === "paused" ? "paused" : "cancelled";
691
468
  state.error = abortedNodeError(error.state);
692
- // Preserve any output + partial transcript artifacts from the aborted leaf.
469
+ // Preserve any output + partial Agent artifacts from the aborted leaf.
693
470
  if (error.output !== undefined)
694
471
  state.output = error.output;
695
472
  if (error.artifactRefs)
696
473
  state.artifactRefs = error.artifactRefs;
697
- state.renderedPrompt = error.renderedPrompt ?? state.renderedPrompt ?? this.store.readNodeState(runId, nodeKey)?.renderedPrompt;
474
+ this.syncAgentTelemetryFromDisk(runId, nodeKey, state);
698
475
  this.store.writeNodeState(runId, state);
699
476
  throw error;
700
477
  }
701
478
  // Don't clobber an external pause/cancel that landed while this node's
702
479
  // leaf was failing; keep the control-plane abort as the observed outcome.
703
- if (this.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
480
+ if (this.runControl.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
704
481
  throw error;
705
482
  }
706
- this.syncInFrameAttemptFromDisk(runId, nodeKey, state);
707
- const abortedState = this.readAbortedStateOnDisk(runId, nodeKey);
483
+ this.runControl.syncInFrameAttemptFromDisk(runId, nodeKey, state);
484
+ const abortedState = this.runControl.readAbortedStateOnDisk(runId, nodeKey);
708
485
  if (abortedState) {
709
486
  throw new NodeAbortedError(nodeKey, abortedState);
710
487
  }
@@ -720,13 +497,13 @@ export class WorkflowInterpreter {
720
497
  if (error instanceof GuardFailureError) {
721
498
  state.output = error.output;
722
499
  }
500
+ this.syncAgentTelemetryFromDisk(runId, nodeKey, state);
723
501
  state.completedAt = new Date().toISOString();
724
502
  this.store.writeNodeState(runId, state);
725
503
  throw error;
726
504
  }
727
505
  finally {
728
- this.abortControllers.delete(`${runId}:${nodeKey}`);
729
- this.abortIntents.delete(`${runId}:${nodeKey}`);
506
+ this.runControl.clearInFlightNode(runId, nodeKey);
730
507
  }
731
508
  }
732
509
  // ─── Kind-specific execution ───────────────────────────────────
@@ -738,12 +515,12 @@ export class WorkflowInterpreter {
738
515
  }
739
516
  catch (error) {
740
517
  if (error instanceof ScopeCompleted)
741
- return error.output;
518
+ return { output: error.output };
742
519
  throw error;
743
520
  }
744
521
  }
745
522
  // Pipeline output: map of step outputs
746
- return { ...ctx.steps };
523
+ return { output: { ...ctx.steps } };
747
524
  }
748
525
  async executeAgent(node, ctx, runId, signal, nodeKey, continuation) {
749
526
  const retry = node.metadata.retry;
@@ -757,52 +534,100 @@ export class WorkflowInterpreter {
757
534
  for (let attempt = 0;; attempt++) {
758
535
  const attemptNo = this.store.readNodeState(runId, nodeKey)?.attempt ?? attempt + 1;
759
536
  let preparedPrompt;
537
+ let renderedSessionKey;
760
538
  try {
761
539
  preparedPrompt = renderAgentRequestPrompt(node, ctx, this.evaluator, Boolean(continuation), attempt > 0);
540
+ renderedSessionKey = renderAgentSessionKey(node, ctx, this.evaluator);
762
541
  }
763
542
  catch (error) {
764
543
  const use = node.metadata.agent?.use ?? "?";
765
544
  throw new LeafExecutionError(`Agent step '${node.id}' (use: ${use}) failed (config): Failed to evaluate agent configuration template: ${error instanceof Error ? error.message : String(error)}`);
766
545
  }
767
- const liveRefs = this.startAgentAttemptArtifacts(runId, nodeKey, attemptNo, preparedPrompt);
768
- pushUnique(allArtifactRefs, liveRefs);
769
- this.publishRunningAgentAttempt(runId, nodeKey, preparedPrompt, allArtifactRefs);
546
+ const rawAcpDebug = process.env.ACPUS_AGENT_RAW_ACP_DEBUG === "1";
547
+ const liveArtifacts = this.attemptArtifacts.startAgentAttempt(runId, nodeKey, attemptNo, preparedPrompt, { rawAcpDebug });
548
+ this.attemptArtifacts.mergeAttemptRefs(allArtifactRefs, liveArtifacts.artifactRefs);
549
+ this.publishRenderedAgentPrompt(runId, nodeKey, preparedPrompt);
550
+ if (attempt === 0 && renderedSessionKey !== undefined) {
551
+ this.publishRenderedAgentSessionKey(runId, nodeKey, renderedSessionKey);
552
+ }
770
553
  const streamDiagnostics = [];
554
+ let sawStdoutStream = false;
555
+ const activity = new AgentTelemetryAccumulator({
556
+ attempt: attemptNo,
557
+ inputText: preparedPrompt,
558
+ inputArtifactRef: liveArtifacts.promptRef,
559
+ onTelemetry: (attemptTelemetry) => {
560
+ this.publishAgentTelemetry(runId, nodeKey, attemptTelemetry);
561
+ }
562
+ });
563
+ this.publishRunningAgentAttempt(runId, nodeKey, allArtifactRefs, activity.snapshot("running"));
771
564
  const result = await this.agentExecutor.execute({
772
565
  node,
773
566
  context: ctx,
774
567
  signal,
775
568
  nodeKey,
776
569
  prompt: preparedPrompt,
570
+ sessionKey: renderedSessionKey,
777
571
  continuation,
778
572
  retry: attempt > 0,
779
573
  onStream: (stream, chunk) => {
780
574
  if (stream === "stdout" && chunk.length > 0) {
575
+ sawStdoutStream = true;
576
+ if (rawAcpDebug) {
577
+ try {
578
+ this.attemptArtifacts.appendAgentRawAcpDebug(runId, nodeKey, attemptNo, chunk);
579
+ }
580
+ catch (error) {
581
+ streamDiagnostics.push(`failed to append raw ACP debug stream: ${error instanceof Error ? error.message : String(error)}`);
582
+ }
583
+ }
781
584
  try {
782
- this.appendAgentTranscript(runId, nodeKey, attemptNo, chunk);
585
+ activity.append(chunk);
783
586
  }
784
587
  catch (error) {
785
- streamDiagnostics.push(`failed to append live agent transcript: ${error instanceof Error ? error.message : String(error)}`);
588
+ streamDiagnostics.push(`failed to parse live agent telemetry: ${error instanceof Error ? error.message : String(error)}`);
786
589
  }
787
590
  }
788
591
  }
789
592
  });
790
- // Persist per-attempt human-readable agent IO plus raw protocol artifacts
593
+ try {
594
+ if (!sawStdoutStream && result.stdout !== undefined)
595
+ activity.append(result.stdout);
596
+ activity.flush();
597
+ }
598
+ catch (error) {
599
+ streamDiagnostics.push(`failed to parse live agent telemetry: ${error instanceof Error ? error.message : String(error)}`);
600
+ }
601
+ if (result.responseText !== undefined)
602
+ activity.setResponseText(result.responseText);
603
+ const abortIntent = result.partial ? this.runControl.abortIntent(runId, nodeKey) : undefined;
604
+ const finalAttemptState = abortIntent
605
+ ?? (result.failureKind || (result.error && !result.partial) ? "failed" : "completed");
606
+ const completedAt = new Date().toISOString();
607
+ // Persist per-attempt human-readable agent IO plus diagnostics
791
608
  // when the executor exposed them. Refs flow back to executeNode via the
792
609
  // return value / thrown error (no shared mutable field).
793
- const artifactRefs = result.responseText !== undefined || result.stdout !== undefined || result.stderr !== undefined
794
- ? this.writeAgentArtifacts(runId, nodeKey, attemptNo, {
610
+ const finalized = result.responseText !== undefined || result.stderr !== undefined
611
+ ? this.attemptArtifacts.finalizeAgentAttempt(runId, nodeKey, attemptNo, {
795
612
  responseText: result.responseText,
796
- transcript: result.stdout,
797
- stderr: mergeStderrDiagnostics(result.stderr, streamDiagnostics)
613
+ stderr: result.stderr,
614
+ diagnostics: streamDiagnostics
798
615
  })
799
616
  : result.artifactRefs;
617
+ const artifactRefs = Array.isArray(finalized) ? finalized : finalized?.artifactRefs;
618
+ const responseRef = !Array.isArray(finalized) ? finalized?.responseRef : undefined;
619
+ if (responseRef)
620
+ activity.setOutputArtifactRef(responseRef);
621
+ const finalTelemetry = activity.snapshot(finalAttemptState, completedAt);
622
+ const telemetryRef = this.attemptArtifacts.writeAgentTelemetry(runId, nodeKey, attemptNo, finalTelemetry);
623
+ this.publishAgentTelemetry(runId, nodeKey, finalTelemetry);
800
624
  if (artifactRefs)
801
- pushUnique(allArtifactRefs, artifactRefs);
625
+ this.attemptArtifacts.mergeAttemptRefs(allArtifactRefs, artifactRefs);
626
+ this.attemptArtifacts.mergeAttemptRefs(allArtifactRefs, [telemetryRef]);
802
627
  if (result.partial) {
803
- // Operator abort → carry output + transcript refs on the abort error;
628
+ // Operator abort → carry output + partial Agent artifact refs on the abort error;
804
629
  // executeNode persists the paused/cancelled state.
805
- throw new NodeAbortedError(nodeKey, this.abortIntent(runId, nodeKey), allArtifactRefs.length > 0 ? allArtifactRefs : undefined, result.output, preparedPrompt);
630
+ throw new NodeAbortedError(nodeKey, abortIntent ?? "paused", allArtifactRefs.length > 0 ? allArtifactRefs : undefined, result.output);
806
631
  }
807
632
  // parse/schema failures are retryable while attempts remain.
808
633
  const retryable = result.failureKind === "parse" || result.failureKind === "schema";
@@ -823,8 +648,7 @@ export class WorkflowInterpreter {
823
648
  // Agent output is wrapped in an envelope for parity with program steps.
824
649
  return {
825
650
  output: { output: result.output },
826
- artifactRefs: allArtifactRefs.length > 0 ? allArtifactRefs : artifactRefs,
827
- renderedPrompt: result.prompt ?? result.renderedPrompt
651
+ artifactRefs: allArtifactRefs.length > 0 ? allArtifactRefs : artifactRefs
828
652
  };
829
653
  }
830
654
  }
@@ -832,73 +656,66 @@ export class WorkflowInterpreter {
832
656
  const result = await this.programExecutor.execute({ node, context: ctx, signal, nodeKey });
833
657
  // Operator abort → paused/cancelled (carry output on the abort error).
834
658
  if (result.partial) {
835
- throw new NodeAbortedError(nodeKey, this.abortIntent(runId, nodeKey), undefined, result.output);
659
+ throw new NodeAbortedError(nodeKey, this.runControl.abortIntent(runId, nodeKey), undefined, result.output);
836
660
  }
837
661
  // Always persist stdout/stderr as artifacts (even when empty). An artifact
838
662
  // write failure is itself non-recoverable.
839
- const artifactRefs = this.writeProgramArtifacts(runId, nodeKey, result.stdout ?? "", result.stderr ?? "");
663
+ const artifactRefs = this.attemptArtifacts.writeProgramArtifacts(runId, nodeKey, result.stdout ?? "", result.stderr ?? "");
840
664
  // Non-recoverable failures fail the node.
841
665
  if (result.failureKind) {
842
666
  throw new LeafExecutionError(`Program step '${node.id}' failed (${result.failureKind}): ${result.error ?? "unknown"}`, artifactRefs);
843
667
  }
844
- // A non-zero exit code is step data. Expose output + exit_code envelope.
668
+ // exit code is allow-listed by `expect.exit_code` (default `[0]`); other
845
669
  return { output: { output: result.output, exit_code: result.exitCode ?? 0 }, artifactRefs };
846
670
  }
847
- /** Write stdout.log/stderr.log artifacts; returns their URIs. */
848
- writeProgramArtifacts(runId, nodeKey, stdout, stderr) {
849
- const out = this.artifactStore.write(runId, nodeKey, "stdout.log", stdout);
850
- const err = this.artifactStore.write(runId, nodeKey, "stderr.log", stderr);
851
- return [out.uri, err.uri];
852
- }
853
- /** Write per-attempt agent response/protocol artifacts; returns their URIs. */
854
- writeAgentArtifacts(runId, nodeKey, attemptNo, content) {
855
- const prefix = this.agentAttemptPrefix(attemptNo);
856
- const refs = [];
857
- if (content.prompt !== undefined) {
858
- refs.push(this.artifactStore.write(runId, nodeKey, `${prefix}.prompt.md`, content.prompt).uri);
859
- }
860
- if (content.responseText !== undefined) {
861
- refs.push(this.artifactStore.write(runId, nodeKey, `${prefix}.response.md`, content.responseText).uri);
862
- }
863
- if (content.transcript !== undefined) {
864
- // Transcript is pre-created and append-built while acpx runs. Finalization
865
- // returns the live artifact ref without overwriting accumulated NDJSON.
866
- refs.push(this.artifactStore.append(runId, nodeKey, `${prefix}.transcript.jsonl`, "").uri);
867
- }
868
- if (content.stderr !== undefined) {
869
- refs.push(this.artifactStore.write(runId, nodeKey, `${prefix}.stderr.log`, content.stderr).uri);
870
- }
871
- return refs;
671
+ publishRunningAgentAttempt(runId, nodeKey, artifactRefs, attemptTelemetry) {
672
+ const state = this.store.readNodeState(runId, nodeKey);
673
+ if (!state)
674
+ return;
675
+ state.artifactRefs = [...artifactRefs];
676
+ state.agentTelemetry = upsertAgentAttemptTelemetry(state.agentTelemetry, attemptTelemetry);
677
+ this.store.writeNodeState(runId, state);
872
678
  }
873
- /** Pre-create live Agent attempt artifacts and return their refs immediately. */
874
- startAgentAttemptArtifacts(runId, nodeKey, attemptNo, prompt) {
875
- const prefix = this.agentAttemptPrefix(attemptNo);
876
- const promptRef = this.artifactStore.write(runId, nodeKey, `${prefix}.prompt.md`, prompt);
877
- const transcriptRef = this.artifactStore.create(runId, nodeKey, `${prefix}.transcript.jsonl`);
878
- return [promptRef.uri, transcriptRef.uri];
679
+ publishRenderedAgentSessionKey(runId, nodeKey, renderedSessionKey) {
680
+ const state = this.store.readNodeState(runId, nodeKey);
681
+ if (!state)
682
+ return;
683
+ state.renderedSessionKey = renderedSessionKey;
684
+ this.store.writeNodeState(runId, state);
879
685
  }
880
- appendAgentTranscript(runId, nodeKey, attemptNo, chunk) {
881
- this.artifactStore.append(runId, nodeKey, `${this.agentAttemptPrefix(attemptNo)}.transcript.jsonl`, chunk);
686
+ publishRenderedAgentPrompt(runId, nodeKey, renderedPrompt) {
687
+ const state = this.store.readNodeState(runId, nodeKey);
688
+ if (!state)
689
+ return;
690
+ state.renderedPrompt = renderedPrompt;
691
+ this.store.writeNodeState(runId, state);
882
692
  }
883
- publishRunningAgentAttempt(runId, nodeKey, prompt, artifactRefs) {
693
+ publishAgentTelemetry(runId, nodeKey, attemptTelemetry) {
884
694
  const state = this.store.readNodeState(runId, nodeKey);
885
695
  if (!state)
886
696
  return;
887
- state.renderedPrompt = prompt;
888
- state.artifactRefs = [...artifactRefs];
697
+ state.agentTelemetry = upsertAgentAttemptTelemetry(state.agentTelemetry, attemptTelemetry);
889
698
  this.store.writeNodeState(runId, state);
890
699
  }
891
- agentAttemptPrefix(attemptNo) {
892
- return `attempt-${String(Math.max(0, attemptNo)).padStart(3, "0")}`;
700
+ syncAgentTelemetryFromDisk(runId, nodeKey, state) {
701
+ const persisted = this.store.readNodeState(runId, nodeKey);
702
+ state.agentTelemetry = persisted?.agentTelemetry ?? state.agentTelemetry;
703
+ state.renderedSessionKey = persisted?.renderedSessionKey ?? state.renderedSessionKey;
704
+ state.renderedPrompt = persisted?.renderedPrompt ?? state.renderedPrompt;
893
705
  }
894
- async executeParallel(node, ctx, runId, dynamic, keyPrefix) {
706
+ async executeParallel(node, ctx, runId, dynamic, nodeKey, keyPrefix) {
895
707
  const children = node.children ?? [];
896
708
  const maxConcurrency = node.metadata.max_concurrency ?? this.maxConcurrency;
897
709
  const join = node.metadata.join ?? "all";
710
+ if (join === "all") {
711
+ children.forEach((child, index) => {
712
+ this.materializePendingNode(runId, child, nestedParallelBranchDynamic(dynamic, index), keyPrefix);
713
+ });
714
+ }
898
715
  const limit = pLimit(maxConcurrency);
899
716
  // Each branch resolves to { child, output } so race can identify the winner.
900
717
  const branchPromises = children.map((child, index) => limit(async () => {
901
- const branchDynamic = { ...dynamic, parallelBranchId: String(index) };
718
+ const branchDynamic = nestedParallelBranchDynamic(dynamic, index);
902
719
  let output;
903
720
  try {
904
721
  output = await this.executeNode(child, { ...ctx, steps: { ...ctx.steps } }, runId, branchDynamic, keyPrefix);
@@ -921,11 +738,11 @@ export class WorkflowInterpreter {
921
738
  branchPromises.forEach((p) => void p.catch(() => undefined));
922
739
  const mapOutput = { [winner.child.id]: winner.output };
923
740
  ctx.steps[winner.child.id] = winner.output;
924
- return mapOutput;
741
+ return { output: mapOutput };
925
742
  }
926
743
  catch (error) {
927
744
  if (!(error instanceof NodeAbortedError)) {
928
- this.cancelDescendantsInScope(runId, node, dynamic);
745
+ this.runControl.cancelDescendantsInScope(runId, nodeKey);
929
746
  }
930
747
  branchPromises.forEach((p) => void p.catch(() => undefined));
931
748
  throw error;
@@ -943,7 +760,7 @@ export class WorkflowInterpreter {
943
760
  catch (error) {
944
761
  if (!(error instanceof NodeAbortedError)) {
945
762
  // Genuine failure or cancel — fast-stop: cancel still-running siblings
946
- this.cancelDescendantsInScope(runId, node, dynamic);
763
+ this.runControl.cancelDescendantsInScope(runId, nodeKey);
947
764
  }
948
765
  branchPromises.forEach((p) => void p.catch(() => undefined));
949
766
  throw error;
@@ -953,65 +770,9 @@ export class WorkflowInterpreter {
953
770
  mapOutput[child.id] = output;
954
771
  ctx.steps[child.id] = output;
955
772
  }
956
- return mapOutput;
773
+ return { output: mapOutput };
957
774
  }
958
- /**
959
- * Cancel still-running/pending/awaiting descendant nodes of a failed
960
- * composite (parallel or fanout), scoped to the given dynamic context. A
961
- * descendant must (a) be one of the composite's descendant IR node ids and
962
- * (b) share every dynamic dimension (item/lane/round) the composite already
963
- * carries — so concurrent sibling fanout lanes outside this scope are not
964
- * affected.
965
- *
966
- * NOTE: pass the composite's OWN dynamic. For a parallel this scopes to the
967
- * current lane (item/lane present). For a fanout, the dynamic does NOT yet
968
- * carry this fanout's lane (lanes are introduced for its children), so the
969
- * scope spans ALL of the fanout's lanes — exactly the fail-fast intent.
970
- */
971
- descendantsInScope(runId, node, dynamic, action) {
972
- const descendantIds = new Set();
973
- const collect = (n) => {
974
- for (const c of n.children ?? []) {
975
- descendantIds.add(c.id);
976
- collect(c);
977
- }
978
- for (const b of n.branches ?? []) {
979
- for (const c of b.children) {
980
- descendantIds.add(c.id);
981
- collect(c);
982
- }
983
- }
984
- };
985
- collect(node);
986
- // Key segments that must be present for an instance to share this scope.
987
- // Values are sanitized to match resolveNodeKey's on-disk encoding (e.g. an
988
- // item id "file:alpha" is stored as "item:file_alpha").
989
- const scopeSegments = [];
990
- if (dynamic.fanoutItemId !== undefined)
991
- scopeSegments.push(`item:${sanitizeValue(String(dynamic.fanoutItemId))}`);
992
- if (dynamic.laneId !== undefined)
993
- scopeSegments.push(`lane:${sanitizeValue(String(dynamic.laneId))}`);
994
- if (dynamic.loopRound !== undefined)
995
- scopeSegments.push(`round:${dynamic.loopRound}`);
996
- for (const ns of this.store.listNodeStates(runId)) {
997
- if (ns.state !== "running" && ns.state !== "awaiting" && ns.state !== "pending")
998
- continue;
999
- if (!descendantIds.has(ns.nodeId))
1000
- continue;
1001
- const segs = ns.nodeKey.split("/");
1002
- if (!scopeSegments.every((s) => segs.includes(s)))
1003
- continue;
1004
- action(runId, ns.nodeKey);
1005
- }
1006
- }
1007
- /**
1008
- * Cancel still-running/pending descendant nodes of a failed composite
1009
- * (parallel `join: all`, or fanout `join: all` fail-fast).
1010
- */
1011
- cancelDescendantsInScope(runId, node, dynamic) {
1012
- this.descendantsInScope(runId, node, dynamic, (rid, key) => this.cancelMaterializedNode(rid, key));
1013
- }
1014
- async executeFanout(node, ctx, runId, dynamic, keyPrefix) {
775
+ async executeFanout(node, ctx, runId, dynamic, nodeKey, keyPrefix) {
1015
776
  const overExpr = node.metadata.over;
1016
777
  if (!overExpr) {
1017
778
  throw new Error(`fanout node ${node.id} missing 'over' expression`);
@@ -1026,6 +787,24 @@ export class WorkflowInterpreter {
1026
787
  // Success target (how many lanes must succeed). Default follows join.
1027
788
  const defaultMinSuccess = join === "race" ? 1 : join === "quorum" ? (quorum ?? 1) : items.length;
1028
789
  const minSuccess = successCriteria?.min_success ?? defaultMinSuccess;
790
+ const lanePlan = items.map((item, index) => {
791
+ const keyCtx = { ...ctx, item, item_index: index };
792
+ const itemId = this.extractItemId(item, node.metadata.key, index, keyCtx, this.evaluator);
793
+ const laneDynamic = { fanoutItemId: itemId, laneId: String(index) };
794
+ return {
795
+ itemId,
796
+ keyCtx,
797
+ itemDynamic: laneDynamic
798
+ };
799
+ });
800
+ if (join === "all") {
801
+ for (const lane of lanePlan) {
802
+ const itemDynamic = { ...dynamic, ...lane.itemDynamic };
803
+ for (const child of children) {
804
+ this.materializePendingNode(runId, child, itemDynamic, keyPrefix);
805
+ }
806
+ }
807
+ }
1029
808
  // Fail-fast: once enough lanes have failed that the success target is
1030
809
  // unreachable (failures > total - minSuccess), abort the whole fanout —
1031
810
  // cancel every still-running/pending lane subtree and reject to short
@@ -1037,14 +816,12 @@ export class WorkflowInterpreter {
1037
816
  // Each lane resolves to a LaneResult. A tolerable failure is captured (does
1038
817
  // not reject) so the join/min_success logic can run. A NodeAbortedError
1039
818
  // (operator pause/cancel) re-throws so it propagates to the parent.
1040
- const lanePromises = items.map((item, index) => limit(async () => {
1041
- const keyCtx = { ...ctx, item, item_index: index };
1042
- const itemId = this.extractItemId(item, node.metadata.key, index, keyCtx, this.evaluator);
1043
- const itemDynamic = { ...dynamic, fanoutItemId: itemId, laneId: String(index) };
819
+ const lanePromises = lanePlan.map((lane) => limit(async () => {
820
+ const itemDynamic = { ...dynamic, ...lane.itemDynamic };
1044
821
  const itemCtx = {
1045
- ...keyCtx,
822
+ ...lane.keyCtx,
1046
823
  steps: { ...ctx.steps },
1047
- item_id: itemId
824
+ item_id: lane.itemId
1048
825
  };
1049
826
  try {
1050
827
  let laneOutput;
@@ -1071,7 +848,7 @@ export class WorkflowInterpreter {
1071
848
  // so it spans every lane), then reject to short-circuit the wait.
1072
849
  if (!failFastTriggered) {
1073
850
  failFastTriggered = true;
1074
- this.cancelDescendantsInScope(runId, node, dynamic);
851
+ this.runControl.cancelDescendantsInScope(runId, nodeKey);
1075
852
  }
1076
853
  throw error;
1077
854
  }
@@ -1085,7 +862,7 @@ export class WorkflowInterpreter {
1085
862
  throw new Error(`fanout ${node.id}: ${successes.length} successful lanes, requires ${minSuccess}`);
1086
863
  }
1087
864
  // outputMerge: "array" of successful lane outputs.
1088
- return successes.map((r) => r.output);
865
+ return { output: successes.map((r) => r.output) };
1089
866
  }
1090
867
  /**
1091
868
  * Resolve fanout lanes per the wait strategy. Lanes never reject on normal
@@ -1116,6 +893,13 @@ export class WorkflowInterpreter {
1116
893
  // and the rejection propagates to fail the fanout immediately.
1117
894
  return Promise.all(lanePromises);
1118
895
  }
896
+ materializePendingNode(runId, node, dynamic, keyPrefix) {
897
+ const resolved = resolveNodeKey(node.keyTemplate, dynamic);
898
+ const nodeKey = withNodeKeyPrefix(keyPrefix, resolved);
899
+ if (this.store.readNodeState(runId, nodeKey))
900
+ return;
901
+ this.store.writeNodeState(runId, createInitialNodeState(nodeKey, node.id, node.kind, hashIrNode(node)));
902
+ }
1119
903
  async executeSwitch(node, ctx, runId, dynamic, keyPrefix) {
1120
904
  const branches = node.branches ?? [];
1121
905
  for (const branch of branches) {
@@ -1129,11 +913,11 @@ export class WorkflowInterpreter {
1129
913
  }
1130
914
  catch (error) {
1131
915
  if (error instanceof ScopeCompleted)
1132
- return error.output;
916
+ return { output: error.output };
1133
917
  throw error;
1134
918
  }
1135
919
  }
1136
- return lastOutput;
920
+ return { output: lastOutput };
1137
921
  }
1138
922
  }
1139
923
  else {
@@ -1145,11 +929,11 @@ export class WorkflowInterpreter {
1145
929
  }
1146
930
  catch (error) {
1147
931
  if (error instanceof ScopeCompleted)
1148
- return error.output;
932
+ return { output: error.output };
1149
933
  throw error;
1150
934
  }
1151
935
  }
1152
- return lastOutput;
936
+ return { output: lastOutput };
1153
937
  }
1154
938
  }
1155
939
  throw new Error(`Switch node ${node.id}: no branch matched and no default`);
@@ -1178,13 +962,13 @@ export class WorkflowInterpreter {
1178
962
  }
1179
963
  catch (error) {
1180
964
  if (error instanceof ScopeCompleted)
1181
- return error.output;
965
+ return { output: error.output };
1182
966
  throw error;
1183
967
  }
1184
968
  }
1185
969
  }
1186
970
  // outputMerge: "last"
1187
- return lastOutput;
971
+ return { output: lastOutput };
1188
972
  }
1189
973
  executeGuard(node, ctx) {
1190
974
  const when = node.metadata.when;
@@ -1196,15 +980,15 @@ export class WorkflowInterpreter {
1196
980
  if (action !== "continue" && action !== "fail" && action !== "complete") {
1197
981
  throw new Error(`guard node ${node.id}: action must be continue, fail, or complete`);
1198
982
  }
1199
- const messageTemplate = node.metadata.message;
1200
- const message = typeof messageTemplate === "string" ? this.evaluator.evaluateTemplate(messageTemplate, ctx) : undefined;
1201
- const output = { matched, action };
1202
- if (message !== undefined)
1203
- output.message = message;
983
+ const guardOutput = { matched, action };
1204
984
  if (action === "fail") {
1205
- throw new GuardFailureError(message ?? `Guard '${node.id}' failed`, output);
985
+ const messageTemplate = node.metadata.message;
986
+ const message = typeof messageTemplate === "string" ? this.evaluator.evaluateTemplate(messageTemplate, ctx) : undefined;
987
+ if (message !== undefined)
988
+ guardOutput.message = message;
989
+ throw new GuardFailureError(message ?? `Guard '${node.id}' failed`, { output: guardOutput });
1206
990
  }
1207
- return { output, completeScope: action === "complete" };
991
+ return { output: { output: guardOutput }, completeScope: action === "complete" };
1208
992
  }
1209
993
  async executeApproval(node, ctx, runId, signal, nodeKey) {
1210
994
  const timeout = node.metadata.timeout;
@@ -1225,7 +1009,7 @@ export class WorkflowInterpreter {
1225
1009
  cleanup();
1226
1010
  // Honor the operator intent recorded by Run-level pause/cancel. An
1227
1011
  // awaiting gate is normally cancelled (awaiting → cancelled).
1228
- reject(new NodeAbortedError(nodeKey, this.abortIntent(runId, nodeKey)));
1012
+ reject(new NodeAbortedError(nodeKey, this.runControl.abortIntent(runId, nodeKey)));
1229
1013
  };
1230
1014
  const timer = timeoutMs
1231
1015
  ? setTimeout(() => {
@@ -1234,10 +1018,10 @@ export class WorkflowInterpreter {
1234
1018
  // `approve`/`reject` resolve; `fail`/`escalate` fail the node
1235
1019
  // (escalate has no runtime channel yet — see roadmap).
1236
1020
  if (onTimeout === "approve") {
1237
- resolve({ approved: true, decision: "timeout", at });
1021
+ resolve({ output: { approved: true, decision: "timeout", at } });
1238
1022
  }
1239
1023
  else if (onTimeout === "reject") {
1240
- resolve({ approved: false, decision: "timeout", at });
1024
+ resolve({ output: { approved: false, decision: "timeout", at } });
1241
1025
  }
1242
1026
  else {
1243
1027
  reject(new Error(`Approval timed out after ${timeout} (on_timeout: ${onTimeout ?? "fail"})`));
@@ -1257,7 +1041,7 @@ export class WorkflowInterpreter {
1257
1041
  // cleanup().
1258
1042
  this.approvalResolvers.set(resolverKey, (approved) => {
1259
1043
  cleanup();
1260
- resolve({ approved, decision: approved ? "approved" : "rejected", at });
1044
+ resolve({ output: { approved, decision: approved ? "approved" : "rejected", at } });
1261
1045
  });
1262
1046
  });
1263
1047
  }
@@ -1305,7 +1089,7 @@ export class WorkflowInterpreter {
1305
1089
  // template string.
1306
1090
  const childInput = {};
1307
1091
  for (const [key, value] of Object.entries(inputSpec ?? {})) {
1308
- childInput[key] = this.evaluateInputValue(value, ctx);
1092
+ childInput[key] = evaluateTemplatedValue(value, ctx, this.evaluator);
1309
1093
  }
1310
1094
  // Validate subworkflow input against the child IR's compiled input schema.
1311
1095
  const validatedChildInput = validateInput(compiled.ir.input, childInput);
@@ -1315,7 +1099,7 @@ export class WorkflowInterpreter {
1315
1099
  try {
1316
1100
  const childCtx = this.buildContext(validatedChildInput, runId);
1317
1101
  await this.executeNode(compiled.ir.root, childCtx, runId, dynamic, nodeKey);
1318
- return { ...childCtx.steps };
1102
+ return { output: evaluateWorkflowOutputs(compiled.ir, childCtx, this.evaluator) };
1319
1103
  }
1320
1104
  finally {
1321
1105
  this.subworkflowStack.delete(childAbs);
@@ -1328,16 +1112,6 @@ export class WorkflowInterpreter {
1328
1112
  return;
1329
1113
  throw new Error(message);
1330
1114
  }
1331
- /** Evaluate a subworkflow input value, preserving native type for single expressions. */
1332
- evaluateInputValue(value, ctx) {
1333
- if (typeof value !== "string")
1334
- return value;
1335
- const single = value.match(/^\s*\$\{\{(.+)\}\}\s*$/s);
1336
- if (single) {
1337
- return this.evaluator.evaluateExpression(single[1].trim(), ctx);
1338
- }
1339
- return this.evaluator.evaluateTemplate(value, ctx);
1340
- }
1341
1115
  // ─── Helpers ────────────────────────────────────────────────────
1342
1116
  buildContext(input, runId) {
1343
1117
  return { input, steps: {}, run_id: runId };
@@ -1395,17 +1169,8 @@ export class WorkflowInterpreter {
1395
1169
  }
1396
1170
  /** Extract the step ID from a resolved node key. The ID is the last path segment before dynamic dims. */
1397
1171
  extractNodeIdFromKey(nodeKey) {
1398
- // Key format: "workflow/step-a" or "workflow/mapped/item:0/lane:0"
1399
- // The node id is the second segment (after "workflow/") for top-level steps
1400
- // For nested, it's the segment before any "type:value" segments
1401
- const segments = nodeKey.split("/");
1402
- // Find the last segment that is NOT a "type:value" dynamic segment
1403
- for (let i = segments.length - 1; i >= 0; i--) {
1404
- if (!segments[i].includes(":")) {
1405
- return segments[i];
1406
- }
1407
- }
1408
- return segments[segments.length - 1];
1172
+ const staticPath = staticNodePathFromKey(nodeKey);
1173
+ return staticPath.split("/").at(-1) ?? nodeKey;
1409
1174
  }
1410
1175
  findNodeById(root, nodeId) {
1411
1176
  if (root.id === nodeId)
@@ -1446,25 +1211,28 @@ export function generateRunId(now = new Date()) {
1446
1211
  ].join("");
1447
1212
  return `${timestamp}${randomBytes(10).toString("hex").toUpperCase()}`;
1448
1213
  }
1214
+ function nestedParallelBranchDynamic(dynamic, branchIndex) {
1215
+ const branchId = String(branchIndex);
1216
+ return {
1217
+ ...dynamic,
1218
+ parallelBranchId: dynamic.parallelBranchId === undefined ? branchId : `${dynamic.parallelBranchId}.${branchId}`
1219
+ };
1220
+ }
1449
1221
  class NodeAbortedError extends Error {
1450
1222
  nodeKey;
1451
1223
  state;
1452
1224
  artifactRefs;
1453
1225
  output;
1454
- renderedPrompt;
1455
1226
  constructor(nodeKey, state,
1456
- /** Artifact refs (e.g. partial transcript) to persist on the aborted node. */
1227
+ /** Artifact refs (e.g. partial Agent response/telemetry) to persist on the aborted node. */
1457
1228
  artifactRefs,
1458
1229
  /** Partial output captured before the abort. */
1459
- output,
1460
- /** Rendered Agent prompt to preserve when abort overwrites node state. */
1461
- renderedPrompt) {
1230
+ output) {
1462
1231
  super(`Node ${nodeKey} aborted: ${state}`);
1463
1232
  this.nodeKey = nodeKey;
1464
1233
  this.state = state;
1465
1234
  this.artifactRefs = artifactRefs;
1466
1235
  this.output = output;
1467
- this.renderedPrompt = renderedPrompt;
1468
1236
  this.name = "NodeAbortedError";
1469
1237
  }
1470
1238
  }
@@ -1476,6 +1244,9 @@ class ScopeCompleted extends Error {
1476
1244
  this.name = "ScopeCompleted";
1477
1245
  }
1478
1246
  }
1247
+ function errorMessage(error) {
1248
+ return error instanceof Error ? error.message : String(error);
1249
+ }
1479
1250
  class GuardFailureError extends Error {
1480
1251
  output;
1481
1252
  constructor(message, output) {