@acpus/runtime 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/agent-telemetry.d.ts +49 -0
  2. package/dist/agent-telemetry.d.ts.map +1 -0
  3. package/dist/agent-telemetry.js +255 -0
  4. package/dist/agent-telemetry.js.map +1 -0
  5. package/dist/artifacts.d.ts +16 -1
  6. package/dist/artifacts.d.ts.map +1 -1
  7. package/dist/artifacts.js +92 -29
  8. package/dist/artifacts.js.map +1 -1
  9. package/dist/attempt-artifacts.d.ts +35 -0
  10. package/dist/attempt-artifacts.d.ts.map +1 -0
  11. package/dist/attempt-artifacts.js +90 -0
  12. package/dist/attempt-artifacts.js.map +1 -0
  13. package/dist/client.d.ts +27 -0
  14. package/dist/client.d.ts.map +1 -1
  15. package/dist/client.js +33 -0
  16. package/dist/client.js.map +1 -1
  17. package/dist/executors/agent.d.ts +7 -3
  18. package/dist/executors/agent.d.ts.map +1 -1
  19. package/dist/executors/agent.js +58 -61
  20. package/dist/executors/agent.js.map +1 -1
  21. package/dist/executors/mock-program.d.ts +2 -0
  22. package/dist/executors/mock-program.d.ts.map +1 -1
  23. package/dist/executors/mock-program.js +27 -6
  24. package/dist/executors/mock-program.js.map +1 -1
  25. package/dist/executors/output-preview.d.ts +2 -0
  26. package/dist/executors/output-preview.d.ts.map +1 -0
  27. package/dist/executors/output-preview.js +20 -0
  28. package/dist/executors/output-preview.js.map +1 -0
  29. package/dist/executors/program.d.ts +10 -3
  30. package/dist/executors/program.d.ts.map +1 -1
  31. package/dist/executors/program.js +39 -4
  32. package/dist/executors/program.js.map +1 -1
  33. package/dist/executors/types.d.ts +2 -0
  34. package/dist/executors/types.d.ts.map +1 -1
  35. package/dist/fork.d.ts +76 -0
  36. package/dist/fork.d.ts.map +1 -0
  37. package/dist/fork.js +201 -0
  38. package/dist/fork.js.map +1 -0
  39. package/dist/index.d.ts +9 -4
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +6 -3
  42. package/dist/index.js.map +1 -1
  43. package/dist/interpreter.d.ts +5 -49
  44. package/dist/interpreter.d.ts.map +1 -1
  45. package/dist/interpreter.js +210 -462
  46. package/dist/interpreter.js.map +1 -1
  47. package/dist/keys.d.ts +50 -2
  48. package/dist/keys.d.ts.map +1 -1
  49. package/dist/keys.js +261 -1
  50. package/dist/keys.js.map +1 -1
  51. package/dist/run-control.d.ts +45 -0
  52. package/dist/run-control.d.ts.map +1 -0
  53. package/dist/run-control.js +253 -0
  54. package/dist/run-control.js.map +1 -0
  55. package/dist/state-machine.d.ts +5 -1
  56. package/dist/state-machine.d.ts.map +1 -1
  57. package/dist/state-machine.js +16 -1
  58. package/dist/state-machine.js.map +1 -1
  59. package/dist/store.d.ts +26 -1
  60. package/dist/store.d.ts.map +1 -1
  61. package/dist/store.js +127 -28
  62. package/dist/store.js.map +1 -1
  63. package/dist/supervisor-app.d.ts.map +1 -1
  64. package/dist/supervisor-app.js +110 -40
  65. package/dist/supervisor-app.js.map +1 -1
  66. package/dist/supervisor-discovery.d.ts +1 -0
  67. package/dist/supervisor-discovery.d.ts.map +1 -1
  68. package/dist/supervisor-discovery.js +9 -15
  69. package/dist/supervisor-discovery.js.map +1 -1
  70. package/dist/types.d.ts +81 -4
  71. package/dist/types.d.ts.map +1 -1
  72. package/dist/workflow-outputs.d.ts +9 -0
  73. package/dist/workflow-outputs.d.ts.map +1 -0
  74. package/dist/workflow-outputs.js +59 -0
  75. package/dist/workflow-outputs.js.map +1 -0
  76. package/package.json +3 -3
@@ -1,58 +1,18 @@
1
- import { parseDurationMs, compileWorkflow } from "@acpus/core";
1
+ import { parseDurationMs, compileWorkflow, hashIrNode } from "@acpus/core";
2
2
  import { readFileSync } from "node:fs";
3
3
  import { dirname, resolve } from "node:path";
4
4
  import { ExpressionEvaluator } from "./evaluator.js";
5
- import { resolveNodeKey, sanitizeValue } from "./keys.js";
6
- import { canTransition, transition, createInitialNodeState, resetFailedForRetry, resetRunningForCrashRecovery, resetAwaitingForCrashRecovery, resetPausedForRunResume } from "./state-machine.js";
5
+ import { resolveNodeKey, staticNodePathFromKey, withNodeKeyPrefix } from "./keys.js";
6
+ import { canTransition, transition, createInitialNodeState } from "./state-machine.js";
7
7
  import { ArtifactStore } from "./artifacts.js";
8
- import { renderAgentRequestPrompt } from "./executors/agent.js";
8
+ import { AttemptArtifactRecorder } from "./attempt-artifacts.js";
9
+ import { renderAgentRequestPrompt, renderAgentSessionKey } from "./executors/agent.js";
9
10
  import { validateInput } from "./validate-input.js";
11
+ import { RunControl, abortedNodeError, isPausedContinuationState } from "./run-control.js";
12
+ import { evaluateTemplatedValue, evaluateWorkflowOutputs } from "./workflow-outputs.js";
10
13
  import { randomBytes } from "node:crypto";
11
14
  import pLimit from "p-limit";
12
- const PAUSED_ABORT_ERROR = "Aborted: paused";
13
- function abortedNodeError(state) {
14
- return state === "paused" ? PAUSED_ABORT_ERROR : "Aborted: cancelled";
15
- }
16
- function isPausedContinuationState(state) {
17
- return state?.state === "pending" && state.attempt > 0 && state.error === PAUSED_ABORT_ERROR;
18
- }
19
- function pushUnique(target, refs) {
20
- if (!refs)
21
- return;
22
- const seen = new Set(target);
23
- for (const ref of refs) {
24
- if (!seen.has(ref)) {
25
- target.push(ref);
26
- seen.add(ref);
27
- }
28
- }
29
- target.sort(compareAttemptArtifactRefs);
30
- }
31
- function mergeStderrDiagnostics(stderr, diagnostics) {
32
- if (diagnostics.length === 0)
33
- return stderr;
34
- const diagnosticText = diagnostics.map((line) => `[acpus] ${line}`).join("\n");
35
- return stderr && stderr.length > 0 ? `${stderr}\n${diagnosticText}` : diagnosticText;
36
- }
37
- function compareAttemptArtifactRefs(a, b) {
38
- const left = attemptArtifactSortKey(a);
39
- const right = attemptArtifactSortKey(b);
40
- if (!left || !right)
41
- return 0;
42
- return left.attempt - right.attempt || left.kind - right.kind;
43
- }
44
- function attemptArtifactSortKey(ref) {
45
- const match = /attempt-(\d+)\.(prompt\.md|transcript\.jsonl|response\.md|stderr\.log)$/.exec(ref);
46
- if (!match)
47
- return undefined;
48
- const kindOrder = {
49
- "prompt.md": 0,
50
- "transcript.jsonl": 1,
51
- "response.md": 2,
52
- "stderr.log": 3
53
- };
54
- return { attempt: Number(match[1]), kind: kindOrder[match[2]] ?? 99 };
55
- }
15
+ import { AgentTelemetryAccumulator, upsertAgentAttemptTelemetry } from "./agent-telemetry.js";
56
16
  /**
57
17
  * The core IR interpreter that drives state transitions, orchestrates
58
18
  * execution, and persists state.
@@ -63,28 +23,24 @@ export class WorkflowInterpreter {
63
23
  agentExecutor;
64
24
  programExecutor;
65
25
  artifactStore;
26
+ attemptArtifacts;
27
+ runControl;
66
28
  maxConcurrency;
67
29
  allowedSourceRoots;
68
30
  sleep;
69
- /** Active abort controllers keyed by "runId:nodeKey" for pause/cancel support */
70
- abortControllers = new Map();
71
- /** Intent of an in-flight abort keyed by "runId:nodeKey" (pause vs cancel). */
72
- abortIntents = new Map();
73
31
  /** Pending human-decision resolvers for Approval Gates awaiting a signal,
74
32
  * keyed by "runId:nodeKey". An entry exists only while a node is `awaiting`. */
75
33
  approvalResolvers = new Map();
76
34
  /** Absolute paths of subworkflow specs currently on the execution stack (cycle guard). */
77
35
  subworkflowStack = new Set();
78
- /** Scheduling guards for Run-level pause/cancel, keyed by runId.
79
- * Per-runId to prevent leakage across runs sharing the same interpreter. */
80
- schedulingPaused = new Map();
81
- schedulingCancelled = new Map();
82
36
  constructor(store, agentExecutor, programExecutor, options) {
83
37
  this.store = store;
84
38
  this.evaluator = new ExpressionEvaluator({ nowTimestamp: options?.nowTimestamp });
85
39
  this.agentExecutor = agentExecutor;
86
40
  this.programExecutor = programExecutor;
87
41
  this.artifactStore = new ArtifactStore(store.getBaseDir());
42
+ this.attemptArtifacts = new AttemptArtifactRecorder(this.artifactStore);
43
+ this.runControl = new RunControl(store);
88
44
  this.maxConcurrency = options?.maxConcurrency ?? 10;
89
45
  this.allowedSourceRoots = options?.allowedSourceRoots?.map((root) => resolve(root)) ?? [];
90
46
  this.sleep = options?.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
@@ -115,7 +71,10 @@ export class WorkflowInterpreter {
115
71
  async runToCompletion(ir, opts, runId) {
116
72
  const meta = this.store.readRunMeta(runId);
117
73
  try {
118
- await this.executeNode(ir.root, this.buildContext(opts.input, runId), runId, {});
74
+ const ctx = this.buildContext(opts.input, runId);
75
+ await this.executeNode(ir.root, ctx, runId, {});
76
+ meta.output = evaluateWorkflowOutputs(ir, ctx, this.evaluator);
77
+ meta.error = undefined;
119
78
  meta.status = "completed";
120
79
  }
121
80
  catch (error) {
@@ -128,14 +87,20 @@ export class WorkflowInterpreter {
128
87
  }
129
88
  else {
130
89
  meta.status = "failed";
131
- void error;
90
+ meta.output = undefined;
91
+ meta.error = errorMessage(error);
92
+ if (rootState?.state === "completed") {
93
+ rootState.state = "failed";
94
+ rootState.error = meta.error;
95
+ rootState.completedAt = new Date().toISOString();
96
+ this.store.writeNodeState(runId, rootState);
97
+ }
132
98
  }
133
99
  }
134
100
  finally {
135
101
  // Clean up per-runId scheduling guards when the run settles,
136
102
  // so they don't leak memory across runs.
137
- this.schedulingPaused.delete(runId);
138
- this.schedulingCancelled.delete(runId);
103
+ this.runControl.clearSchedulingGuards(runId);
139
104
  }
140
105
  meta.updatedAt = new Date().toISOString();
141
106
  this.store.writeRunMeta(runId, meta);
@@ -150,16 +115,7 @@ export class WorkflowInterpreter {
150
115
  * for a fresh human decision on re-execution.
151
116
  */
152
117
  recoverStaleNodes(runId) {
153
- for (const nodeState of this.store.listNodeStates(runId)) {
154
- if (nodeState.state === "running") {
155
- nodeState.state = resetRunningForCrashRecovery(nodeState.state);
156
- this.store.writeNodeState(runId, nodeState);
157
- }
158
- else if (nodeState.state === "awaiting") {
159
- nodeState.state = resetAwaitingForCrashRecovery(nodeState.state);
160
- this.store.writeNodeState(runId, nodeState);
161
- }
162
- }
118
+ this.runControl.recoverStaleNodes(runId);
163
119
  }
164
120
  /**
165
121
  * Deterministically replay a persisted Run and verify its reconstructed
@@ -225,7 +181,7 @@ export class WorkflowInterpreter {
225
181
  */
226
182
  replayNode(node, ctx, runId, dynamic, keyPrefix, recorded, reached, evaluator) {
227
183
  const resolved = resolveNodeKey(node.keyTemplate, dynamic);
228
- const nodeKey = keyPrefix ? `${keyPrefix}/${resolved}` : resolved;
184
+ const nodeKey = withNodeKeyPrefix(keyPrefix, resolved);
229
185
  const rec = recorded.get(nodeKey);
230
186
  // A node absent from the recording was never reached on the original walk
231
187
  // (e.g. an untaken switch branch); skip it so we don't fabricate topology.
@@ -245,7 +201,7 @@ export class WorkflowInterpreter {
245
201
  break;
246
202
  case "parallel":
247
203
  (node.children ?? []).forEach((child, index) => {
248
- const branchDynamic = { ...dynamic, parallelBranchId: String(index) };
204
+ const branchDynamic = nestedParallelBranchDynamic(dynamic, index);
249
205
  this.replayNode(child, ctx, runId, branchDynamic, keyPrefix, recorded, reached, evaluator);
250
206
  });
251
207
  break;
@@ -290,7 +246,7 @@ export class WorkflowInterpreter {
290
246
  this.replayNode(child, loopCtx, runId, loopDynamic, keyPrefix, recorded, reached, evaluator);
291
247
  if (reached.size > before)
292
248
  anyChildReached = true;
293
- const childKey = `${keyPrefix ? `${keyPrefix}/` : ""}${resolveNodeKey(child.keyTemplate, loopDynamic)}`;
249
+ const childKey = withNodeKeyPrefix(keyPrefix, resolveNodeKey(child.keyTemplate, loopDynamic));
294
250
  lastOutput = recorded.get(childKey)?.output ?? lastOutput;
295
251
  }
296
252
  if (!anyChildReached)
@@ -313,81 +269,13 @@ export class WorkflowInterpreter {
313
269
  break;
314
270
  }
315
271
  }
316
- /** Pause one running node as part of a Run-level pause operation. */
317
- pauseRunningNode(runId, nodeKey) {
318
- const state = this.store.readNodeState(runId, nodeKey);
319
- if (!state || !canTransition(state.state, "paused")) {
320
- return;
321
- }
322
- this.abortIntents.set(`${runId}:${nodeKey}`, "paused");
323
- const controller = this.abortControllers.get(`${runId}:${nodeKey}`);
324
- if (controller) {
325
- controller.abort();
326
- }
327
- state.state = transition(state.state, "paused");
328
- this.store.writeNodeState(runId, state);
329
- }
330
- /** Cancel one materialized node as part of Run-level cancel or fail-fast. */
331
- cancelMaterializedNode(runId, nodeKey) {
332
- this.abortIntents.set(`${runId}:${nodeKey}`, "cancelled");
333
- const controller = this.abortControllers.get(`${runId}:${nodeKey}`);
334
- if (controller) {
335
- controller.abort();
336
- }
337
- const state = this.store.readNodeState(runId, nodeKey);
338
- if (state && canTransition(state.state, "cancelled")) {
339
- state.state = transition(state.state, "cancelled");
340
- this.store.writeNodeState(runId, state);
341
- }
342
- }
343
- /** Resolve the operator intent for an in-flight abort; defaults to paused. */
344
- abortIntent(runId, nodeKey) {
345
- return this.abortIntents.get(`${runId}:${nodeKey}`) ?? "paused";
346
- }
347
- /** True if the node has been persisted as terminal `cancelled` (e.g. by
348
- * fail-fast). Used to avoid clobbering it when a still-running leaf later
349
- * resolves/rejects. */
350
- readAbortedStateOnDisk(runId, nodeKey) {
351
- const state = this.store.readNodeState(runId, nodeKey)?.state;
352
- return state === "paused" || state === "cancelled" ? state : undefined;
353
- }
354
- isStaleAttemptOnDisk(runId, nodeKey, attempt, startedAt) {
355
- const current = this.store.readNodeState(runId, nodeKey);
356
- return current !== undefined && current.attempt > attempt && current.startedAt !== startedAt;
357
- }
358
- syncInFrameAttemptFromDisk(runId, nodeKey, state) {
359
- const current = this.store.readNodeState(runId, nodeKey);
360
- if (current !== undefined && current.startedAt === state.startedAt && current.attempt > state.attempt) {
361
- state.attempt = current.attempt;
362
- }
363
- }
364
272
  /**
365
273
  * Retry a failed executable Node. This is a local repair operation for the
366
274
  * target executable only; Run-level retry remains the operation that restores
367
275
  * Workflow progress from failed composite ancestors.
368
276
  */
369
277
  async retryNode(runId, nodeKey) {
370
- const state = this.store.readNodeState(runId, nodeKey);
371
- if (!state) {
372
- throw new Error(`Node ${nodeKey} not found in run ${runId}`);
373
- }
374
- if (state.state !== "failed") {
375
- throw new Error(`Cannot retry node ${nodeKey} in state '${state.state}': only failed executable nodes are retryable`);
376
- }
377
- if (state.kind !== "run.agent" && state.kind !== "run.program") {
378
- throw new Error(`Cannot retry node ${nodeKey}: only failed executable nodes are retryable`);
379
- }
380
- const meta = this.store.readRunMeta(runId);
381
- if (!meta)
382
- throw new Error(`Run ${runId} not found`);
383
- if (meta.status !== "failed") {
384
- throw new Error(`Cannot retry node ${nodeKey}: node retry is accepted only when the Run is failed`);
385
- }
386
- const ir = this.store.readIr(runId);
387
- const input = this.store.readInput(runId);
388
- if (!ir || !input) {
389
- throw new Error(`Cannot retry node ${nodeKey}: run ${runId} has no persisted IR or input`);
390
- }
278
+ const { state, ir, input } = this.runControl.prepareNodeRetry(runId, nodeKey);
391
279
  // Resolve the node IR *before* mutating state. Subworkflow child IR is
392
280
  // compiled on demand and never persisted, so a child key like
393
281
  // `workflow/sub/child` is unresolvable here; reject explicitly rather than
@@ -398,9 +286,7 @@ export class WorkflowInterpreter {
398
286
  }
399
287
  // Control-plane reset (failed → pending), not a business-lifecycle
400
288
  // transition. `attempt` is incremented by executeNode.
401
- state.state = resetFailedForRetry(state.state);
402
- state.error = undefined;
403
- this.store.writeNodeState(runId, state);
289
+ this.runControl.resetNodeForRetry(runId, state);
404
290
  const ctx = this.buildContext(input, runId);
405
291
  this.populateStepOutputs(runId, ctx);
406
292
  // Restore the parent dynamic value-context captured at first execution.
@@ -416,22 +302,7 @@ export class WorkflowInterpreter {
416
302
  * pauses all running nodes, and updates Run metadata to `paused`.
417
303
  */
418
304
  pauseRun(runId) {
419
- const meta = this.store.readRunMeta(runId);
420
- if (!meta)
421
- throw new Error(`Run ${runId} not found`);
422
- if (meta.status !== "running") {
423
- throw new Error(`Cannot pause a run in state '${meta.status}'`);
424
- }
425
- this.schedulingPaused.set(runId, true);
426
- // Pause all currently running nodes
427
- for (const nodeState of this.store.listNodeStates(runId)) {
428
- if (nodeState.state === "running") {
429
- this.pauseRunningNode(runId, nodeState.nodeKey);
430
- }
431
- }
432
- meta.status = "paused";
433
- meta.updatedAt = new Date().toISOString();
434
- this.store.writeRunMeta(runId, meta);
305
+ this.runControl.pauseRun(runId);
435
306
  }
436
307
  /**
437
308
  * Cancel an entire Run. Validates Run is `running` or `paused`, sets
@@ -439,78 +310,14 @@ export class WorkflowInterpreter {
439
310
  * and updates Run metadata to `cancelled`.
440
311
  */
441
312
  cancelRun(runId) {
442
- const meta = this.store.readRunMeta(runId);
443
- if (!meta)
444
- throw new Error(`Run ${runId} not found`);
445
- if (meta.status !== "running" && meta.status !== "paused") {
446
- throw new Error(`Cannot cancel a run in state '${meta.status}'`);
447
- }
448
- this.schedulingCancelled.set(runId, true);
449
- // Override any prior pause abort intents to "cancelled" so that
450
- // runToCompletion's catch block writes "cancelled" (not "paused").
451
- for (const [key, intent] of this.abortIntents) {
452
- if (key.startsWith(`${runId}:`) && intent === "paused") {
453
- this.abortIntents.set(key, "cancelled");
454
- }
455
- }
456
- // Cancel all currently running or awaiting nodes (abort the in-flight leaf
457
- // or the pending approval wait, then transition to cancelled).
458
- for (const nodeState of this.store.listNodeStates(runId)) {
459
- if (nodeState.state === "running" || nodeState.state === "awaiting") {
460
- this.cancelMaterializedNode(runId, nodeState.nodeKey);
461
- }
462
- }
463
- // Also transition paused nodes to cancelled (a prior pauseRun may have
464
- // left nodes in "paused" state; cancel supersedes pause).
465
- for (const nodeState of this.store.listNodeStates(runId)) {
466
- if (nodeState.state === "paused") {
467
- if (canTransition(nodeState.state, "cancelled")) {
468
- nodeState.state = transition(nodeState.state, "cancelled");
469
- this.store.writeNodeState(runId, nodeState);
470
- }
471
- }
472
- }
473
- // Mark all pending nodes as cancelled (do NOT materialize unvisited nodes)
474
- for (const nodeState of this.store.listNodeStates(runId)) {
475
- if (nodeState.state === "pending") {
476
- if (canTransition(nodeState.state, "cancelled")) {
477
- nodeState.state = transition(nodeState.state, "cancelled");
478
- this.store.writeNodeState(runId, nodeState);
479
- }
480
- }
481
- }
482
- meta.status = "cancelled";
483
- meta.updatedAt = new Date().toISOString();
484
- this.store.writeRunMeta(runId, meta);
313
+ this.runControl.cancelRun(runId);
485
314
  }
486
315
  /**
487
316
  * Resume an entire paused Run. Validates Run is `paused`, clears scheduling
488
317
  * guards, recovers stale nodes, and re-executes from root.
489
318
  */
490
319
  async resumeRun(runId) {
491
- const meta = this.store.readRunMeta(runId);
492
- if (!meta)
493
- throw new Error(`Run ${runId} not found`);
494
- if (meta.status !== "paused") {
495
- throw new Error(`Cannot resume a run in state '${meta.status}'`);
496
- }
497
- // Clear scheduling guards for this run
498
- this.schedulingPaused.delete(runId);
499
- this.schedulingCancelled.delete(runId);
500
- // Recover stale running nodes back to pending
501
- this.recoverStaleNodes(runId);
502
- // Also reset paused nodes back to pending so runToCompletion can re-execute them.
503
- // Without this, executeNode sees the 'paused' state and throws NodeAbortedError,
504
- // making the run permanently stuck.
505
- for (const nodeState of this.store.listNodeStates(runId)) {
506
- if (nodeState.state === "paused") {
507
- nodeState.state = resetPausedForRunResume(nodeState.state);
508
- this.store.writeNodeState(runId, nodeState);
509
- }
510
- }
511
- meta.status = "running";
512
- meta.updatedAt = new Date().toISOString();
513
- this.store.writeRunMeta(runId, meta);
320
+ this.runControl.resumeRun(runId);
514
321
  }
515
322
  /**
516
323
  * Retry a failed Run. Validates Run is `failed`, resets failed materialized
@@ -518,33 +325,7 @@ export class WorkflowInterpreter {
518
325
  * re-executes from root. Same Run ID, no new Run.
519
326
  */
520
327
  retryRun(runId) {
521
- const meta = this.store.readRunMeta(runId);
522
- if (!meta)
523
- throw new Error(`Run ${runId} not found`);
524
- if (meta.status !== "failed") {
525
- throw new Error(`Cannot retry a run in state '${meta.status}'`);
526
- }
527
- // Clear scheduling guards for this run
528
- this.schedulingPaused.delete(runId);
529
- this.schedulingCancelled.delete(runId);
530
- // Reset failed and paused materialized nodes to pending (preserve completed).
531
- // Paused nodes can exist in a "failed" run if it was paused before a sibling
532
- // failed (e.g. parallel lane failure while another lane was paused).
533
- for (const nodeState of this.store.listNodeStates(runId)) {
534
- if (nodeState.state === "failed") {
535
- nodeState.state = resetFailedForRetry(nodeState.state);
536
- nodeState.error = undefined;
537
- this.store.writeNodeState(runId, nodeState);
538
- }
539
- else if (nodeState.state === "paused") {
540
- nodeState.state = resetPausedForRunResume(nodeState.state);
541
- this.store.writeNodeState(runId, nodeState);
542
- }
543
- }
544
- meta.status = "running";
545
- meta.runAttempt++;
546
- meta.updatedAt = new Date().toISOString();
547
- this.store.writeRunMeta(runId, meta);
328
+ this.runControl.retryRun(runId);
548
329
  }
549
330
  // ─── Node execution dispatch ──────────────────────────────────
550
331
  async executeNode(node, ctx, runId, dynamic, keyPrefix, isContinuation, overrideNodeKey) {
@@ -552,7 +333,7 @@ export class WorkflowInterpreter {
552
333
  // node's stable identity (and thus the agent's acpx session name) survives
553
334
  // across loop/fanout/lane/subworkflow dynamics that are not re-derived here.
554
335
  const resolved = resolveNodeKey(node.keyTemplate, dynamic);
555
- const nodeKey = overrideNodeKey ?? (keyPrefix ? `${keyPrefix}/${resolved}` : resolved);
336
+ const nodeKey = overrideNodeKey ?? withNodeKeyPrefix(keyPrefix, resolved);
556
337
  // Check if already completed (from prior run)
557
338
  const existing = this.store.readNodeState(runId, nodeKey);
558
339
  if (existing?.state === "completed") {
@@ -564,25 +345,20 @@ export class WorkflowInterpreter {
564
345
  }
565
346
  const continuation = Boolean(isContinuation || isPausedContinuationState(existing));
566
347
  // Initialize state
567
- const state = existing ?? createInitialNodeState(nodeKey, node.id, node.kind);
348
+ const definitionHash = hashIrNode(node);
349
+ const state = existing ?? createInitialNodeState(nodeKey, node.id, node.kind, definitionHash);
350
+ if (!state.definitionHash)
351
+ state.definitionHash = definitionHash;
568
352
  if (state.state === "failed") {
569
353
  throw new Error(`Node ${nodeKey} is in failed state`);
570
354
  }
571
- // Run-level scheduling guards: if this Run is paused or cancelled,
572
- // don't schedule new work.
573
- if (this.schedulingPaused.get(runId) && state.state === "pending") {
574
- throw new NodeAbortedError(nodeKey, "paused");
575
- }
576
- if (this.schedulingCancelled.get(runId) && state.state === "pending") {
577
- if (canTransition(state.state, "cancelled")) {
578
- state.state = transition(state.state, "cancelled");
579
- this.store.writeNodeState(runId, state);
580
- }
581
- throw new NodeAbortedError(nodeKey, "cancelled");
355
+ const schedulingAbort = this.runControl.applySchedulingGuard(runId, state);
356
+ if (schedulingAbort) {
357
+ throw new NodeAbortedError(nodeKey, schedulingAbort);
582
358
  }
583
359
  // Set up abort controller
584
360
  const controller = new AbortController();
585
- this.abortControllers.set(`${runId}:${nodeKey}`, controller);
361
+ this.runControl.registerAbortController(runId, nodeKey, controller);
586
362
  // Transition to running
587
363
  if (canTransition(state.state, "running")) {
588
364
  state.state = transition(state.state, "running");
@@ -614,8 +390,6 @@ export class WorkflowInterpreter {
614
390
  const leaf = await this.executeAgent(node, ctx, runId, controller.signal, nodeKey, continuation);
615
391
  output = leaf.output;
616
392
  artifactRefs = leaf.artifactRefs;
617
- if (leaf.renderedPrompt)
618
- state.renderedPrompt = leaf.renderedPrompt;
619
393
  break;
620
394
  }
621
395
  case "run.program": {
@@ -625,10 +399,10 @@ export class WorkflowInterpreter {
625
399
  break;
626
400
  }
627
401
  case "parallel":
628
- output = await this.executeParallel(node, ctx, runId, dynamic, keyPrefix);
402
+ output = await this.executeParallel(node, ctx, runId, dynamic, nodeKey, keyPrefix);
629
403
  break;
630
404
  case "fanout":
631
- output = await this.executeFanout(node, ctx, runId, dynamic, keyPrefix);
405
+ output = await this.executeFanout(node, ctx, runId, dynamic, nodeKey, keyPrefix);
632
406
  break;
633
407
  case "switch":
634
408
  output = await this.executeSwitch(node, ctx, runId, dynamic, keyPrefix);
@@ -653,15 +427,16 @@ export class WorkflowInterpreter {
653
427
  }
654
428
  // If a Run-level resume already re-entered this node, a late result from
655
429
  // the old attempt must not overwrite the newer attempt's state.
656
- if (this.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
430
+ if (this.runControl.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
657
431
  return output;
658
432
  }
659
- this.syncInFrameAttemptFromDisk(runId, nodeKey, state);
433
+ this.runControl.syncInFrameAttemptFromDisk(runId, nodeKey, state);
434
+ this.syncAgentTelemetryFromDisk(runId, nodeKey, state);
660
435
  // Transition to completed unless an external pause/cancel already changed
661
436
  // this node's state on disk while we were awaiting the leaf.
662
- const abortedState = this.readAbortedStateOnDisk(runId, nodeKey);
437
+ const abortedState = this.runControl.readAbortedStateOnDisk(runId, nodeKey);
663
438
  if (abortedState) {
664
- throw new NodeAbortedError(nodeKey, abortedState, artifactRefs, output, state.renderedPrompt);
439
+ throw new NodeAbortedError(nodeKey, abortedState, artifactRefs, output);
665
440
  }
666
441
  state.state = "completed";
667
442
  state.error = undefined;
@@ -682,29 +457,29 @@ export class WorkflowInterpreter {
682
457
  throw error;
683
458
  }
684
459
  if (error instanceof NodeAbortedError) {
685
- if (this.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
460
+ if (this.runControl.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
686
461
  throw error;
687
462
  }
688
- this.syncInFrameAttemptFromDisk(runId, nodeKey, state);
463
+ this.runControl.syncInFrameAttemptFromDisk(runId, nodeKey, state);
689
464
  // Transition this node to the same state as the child that was aborted.
690
465
  state.state = error.state === "paused" ? "paused" : "cancelled";
691
466
  state.error = abortedNodeError(error.state);
692
- // Preserve any output + partial transcript artifacts from the aborted leaf.
467
+ // Preserve any output + partial Agent artifacts from the aborted leaf.
693
468
  if (error.output !== undefined)
694
469
  state.output = error.output;
695
470
  if (error.artifactRefs)
696
471
  state.artifactRefs = error.artifactRefs;
697
- state.renderedPrompt = error.renderedPrompt ?? state.renderedPrompt ?? this.store.readNodeState(runId, nodeKey)?.renderedPrompt;
472
+ this.syncAgentTelemetryFromDisk(runId, nodeKey, state);
698
473
  this.store.writeNodeState(runId, state);
699
474
  throw error;
700
475
  }
701
476
  // Don't clobber an external pause/cancel that landed while this node's
702
477
  // leaf was failing; keep the control-plane abort as the observed outcome.
703
- if (this.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
478
+ if (this.runControl.isStaleAttemptOnDisk(runId, nodeKey, state.attempt, state.startedAt)) {
704
479
  throw error;
705
480
  }
706
- this.syncInFrameAttemptFromDisk(runId, nodeKey, state);
707
- const abortedState = this.readAbortedStateOnDisk(runId, nodeKey);
481
+ this.runControl.syncInFrameAttemptFromDisk(runId, nodeKey, state);
482
+ const abortedState = this.runControl.readAbortedStateOnDisk(runId, nodeKey);
708
483
  if (abortedState) {
709
484
  throw new NodeAbortedError(nodeKey, abortedState);
710
485
  }
@@ -720,13 +495,13 @@ export class WorkflowInterpreter {
720
495
  if (error instanceof GuardFailureError) {
721
496
  state.output = error.output;
722
497
  }
498
+ this.syncAgentTelemetryFromDisk(runId, nodeKey, state);
723
499
  state.completedAt = new Date().toISOString();
724
500
  this.store.writeNodeState(runId, state);
725
501
  throw error;
726
502
  }
727
503
  finally {
728
- this.abortControllers.delete(`${runId}:${nodeKey}`);
729
- this.abortIntents.delete(`${runId}:${nodeKey}`);
504
+ this.runControl.clearInFlightNode(runId, nodeKey);
730
505
  }
731
506
  }
732
507
  // ─── Kind-specific execution ───────────────────────────────────
@@ -738,12 +513,12 @@ export class WorkflowInterpreter {
738
513
  }
739
514
  catch (error) {
740
515
  if (error instanceof ScopeCompleted)
741
- return error.output;
516
+ return { output: error.output };
742
517
  throw error;
743
518
  }
744
519
  }
745
520
  // Pipeline output: map of step outputs
746
- return { ...ctx.steps };
521
+ return { output: { ...ctx.steps } };
747
522
  }
748
523
  async executeAgent(node, ctx, runId, signal, nodeKey, continuation) {
749
524
  const retry = node.metadata.retry;
@@ -757,52 +532,96 @@ export class WorkflowInterpreter {
757
532
  for (let attempt = 0;; attempt++) {
758
533
  const attemptNo = this.store.readNodeState(runId, nodeKey)?.attempt ?? attempt + 1;
759
534
  let preparedPrompt;
535
+ let renderedSessionKey;
760
536
  try {
761
537
  preparedPrompt = renderAgentRequestPrompt(node, ctx, this.evaluator, Boolean(continuation), attempt > 0);
538
+ renderedSessionKey = renderAgentSessionKey(node, ctx, this.evaluator);
762
539
  }
763
540
  catch (error) {
764
541
  const use = node.metadata.agent?.use ?? "?";
765
542
  throw new LeafExecutionError(`Agent step '${node.id}' (use: ${use}) failed (config): Failed to evaluate agent configuration template: ${error instanceof Error ? error.message : String(error)}`);
766
543
  }
767
- const liveRefs = this.startAgentAttemptArtifacts(runId, nodeKey, attemptNo, preparedPrompt);
768
- pushUnique(allArtifactRefs, liveRefs);
769
- this.publishRunningAgentAttempt(runId, nodeKey, preparedPrompt, allArtifactRefs);
544
+ const rawAcpDebug = process.env.ACPUS_AGENT_RAW_ACP_DEBUG === "1";
545
+ const liveArtifacts = this.attemptArtifacts.startAgentAttempt(runId, nodeKey, attemptNo, preparedPrompt, { rawAcpDebug });
546
+ this.attemptArtifacts.mergeAttemptRefs(allArtifactRefs, liveArtifacts.artifactRefs);
770
547
  const streamDiagnostics = [];
548
+ let sawStdoutStream = false;
549
+ const activity = new AgentTelemetryAccumulator({
550
+ attempt: attemptNo,
551
+ inputText: preparedPrompt,
552
+ inputArtifactRef: liveArtifacts.promptRef,
553
+ onTelemetry: (attemptTelemetry) => {
554
+ this.publishAgentTelemetry(runId, nodeKey, attemptTelemetry);
555
+ }
556
+ });
557
+ this.publishRunningAgentAttempt(runId, nodeKey, allArtifactRefs, activity.snapshot("running"));
771
558
  const result = await this.agentExecutor.execute({
772
559
  node,
773
560
  context: ctx,
774
561
  signal,
775
562
  nodeKey,
776
563
  prompt: preparedPrompt,
564
+ sessionKey: renderedSessionKey,
777
565
  continuation,
778
566
  retry: attempt > 0,
779
567
  onStream: (stream, chunk) => {
780
568
  if (stream === "stdout" && chunk.length > 0) {
569
+ sawStdoutStream = true;
570
+ if (rawAcpDebug) {
571
+ try {
572
+ this.attemptArtifacts.appendAgentRawAcpDebug(runId, nodeKey, attemptNo, chunk);
573
+ }
574
+ catch (error) {
575
+ streamDiagnostics.push(`failed to append raw ACP debug stream: ${error instanceof Error ? error.message : String(error)}`);
576
+ }
577
+ }
781
578
  try {
782
- this.appendAgentTranscript(runId, nodeKey, attemptNo, chunk);
579
+ activity.append(chunk);
783
580
  }
784
581
  catch (error) {
785
- streamDiagnostics.push(`failed to append live agent transcript: ${error instanceof Error ? error.message : String(error)}`);
582
+ streamDiagnostics.push(`failed to parse live agent telemetry: ${error instanceof Error ? error.message : String(error)}`);
786
583
  }
787
584
  }
788
585
  }
789
586
  });
790
- // Persist per-attempt human-readable agent IO plus raw protocol artifacts
587
+ try {
588
+ if (!sawStdoutStream && result.stdout !== undefined)
589
+ activity.append(result.stdout);
590
+ activity.flush();
591
+ }
592
+ catch (error) {
593
+ streamDiagnostics.push(`failed to parse live agent telemetry: ${error instanceof Error ? error.message : String(error)}`);
594
+ }
595
+ if (result.responseText !== undefined)
596
+ activity.setResponseText(result.responseText);
597
+ const abortIntent = result.partial ? this.runControl.abortIntent(runId, nodeKey) : undefined;
598
+ const finalAttemptState = abortIntent
599
+ ?? (result.failureKind || (result.error && !result.partial) ? "failed" : "completed");
600
+ const completedAt = new Date().toISOString();
601
+ // Persist per-attempt human-readable agent IO plus diagnostics
791
602
  // when the executor exposed them. Refs flow back to executeNode via the
792
603
  // return value / thrown error (no shared mutable field).
793
- const artifactRefs = result.responseText !== undefined || result.stdout !== undefined || result.stderr !== undefined
794
- ? this.writeAgentArtifacts(runId, nodeKey, attemptNo, {
604
+ const finalized = result.responseText !== undefined || result.stderr !== undefined
605
+ ? this.attemptArtifacts.finalizeAgentAttempt(runId, nodeKey, attemptNo, {
795
606
  responseText: result.responseText,
796
- transcript: result.stdout,
797
- stderr: mergeStderrDiagnostics(result.stderr, streamDiagnostics)
607
+ stderr: result.stderr,
608
+ diagnostics: streamDiagnostics
798
609
  })
799
610
  : result.artifactRefs;
611
+ const artifactRefs = Array.isArray(finalized) ? finalized : finalized?.artifactRefs;
612
+ const responseRef = !Array.isArray(finalized) ? finalized?.responseRef : undefined;
613
+ if (responseRef)
614
+ activity.setOutputArtifactRef(responseRef);
615
+ const finalTelemetry = activity.snapshot(finalAttemptState, completedAt);
616
+ const telemetryRef = this.attemptArtifacts.writeAgentTelemetry(runId, nodeKey, attemptNo, finalTelemetry);
617
+ this.publishAgentTelemetry(runId, nodeKey, finalTelemetry);
800
618
  if (artifactRefs)
801
- pushUnique(allArtifactRefs, artifactRefs);
619
+ this.attemptArtifacts.mergeAttemptRefs(allArtifactRefs, artifactRefs);
620
+ this.attemptArtifacts.mergeAttemptRefs(allArtifactRefs, [telemetryRef]);
802
621
  if (result.partial) {
803
- // Operator abort → carry output + transcript refs on the abort error;
622
+ // Operator abort → carry output + partial Agent artifact refs on the abort error;
804
623
  // executeNode persists the paused/cancelled state.
805
- throw new NodeAbortedError(nodeKey, this.abortIntent(runId, nodeKey), allArtifactRefs.length > 0 ? allArtifactRefs : undefined, result.output, preparedPrompt);
624
+ throw new NodeAbortedError(nodeKey, abortIntent ?? "paused", allArtifactRefs.length > 0 ? allArtifactRefs : undefined, result.output);
806
625
  }
807
626
  // parse/schema failures are retryable while attempts remain.
808
627
  const retryable = result.failureKind === "parse" || result.failureKind === "schema";
@@ -823,8 +642,7 @@ export class WorkflowInterpreter {
823
642
  // Agent output is wrapped in an envelope for parity with program steps.
824
643
  return {
825
644
  output: { output: result.output },
826
- artifactRefs: allArtifactRefs.length > 0 ? allArtifactRefs : artifactRefs,
827
- renderedPrompt: result.prompt ?? result.renderedPrompt
645
+ artifactRefs: allArtifactRefs.length > 0 ? allArtifactRefs : artifactRefs
828
646
  };
829
647
  }
830
648
  }
@@ -832,73 +650,49 @@ export class WorkflowInterpreter {
832
650
  const result = await this.programExecutor.execute({ node, context: ctx, signal, nodeKey });
833
651
  // Operator abort → paused/cancelled (carry output on the abort error).
834
652
  if (result.partial) {
835
- throw new NodeAbortedError(nodeKey, this.abortIntent(runId, nodeKey), undefined, result.output);
653
+ throw new NodeAbortedError(nodeKey, this.runControl.abortIntent(runId, nodeKey), undefined, result.output);
836
654
  }
837
655
  // Always persist stdout/stderr as artifacts (even when empty). An artifact
838
656
  // write failure is itself non-recoverable.
839
- const artifactRefs = this.writeProgramArtifacts(runId, nodeKey, result.stdout ?? "", result.stderr ?? "");
657
+ const artifactRefs = this.attemptArtifacts.writeProgramArtifacts(runId, nodeKey, result.stdout ?? "", result.stderr ?? "");
840
658
  // Non-recoverable failures fail the node.
841
659
  if (result.failureKind) {
842
660
  throw new LeafExecutionError(`Program step '${node.id}' failed (${result.failureKind}): ${result.error ?? "unknown"}`, artifactRefs);
843
661
  }
844
- // A non-zero exit code is step data. Expose output + exit_code envelope.
662
+ // exit code is allow-listed by `expect.exit_code` (default `[0]`); other
845
663
  return { output: { output: result.output, exit_code: result.exitCode ?? 0 }, artifactRefs };
846
664
  }
847
- /** Write stdout.log/stderr.log artifacts; returns their URIs. */
848
- writeProgramArtifacts(runId, nodeKey, stdout, stderr) {
849
- const out = this.artifactStore.write(runId, nodeKey, "stdout.log", stdout);
850
- const err = this.artifactStore.write(runId, nodeKey, "stderr.log", stderr);
851
- return [out.uri, err.uri];
852
- }
853
- /** Write per-attempt agent response/protocol artifacts; returns their URIs. */
854
- writeAgentArtifacts(runId, nodeKey, attemptNo, content) {
855
- const prefix = this.agentAttemptPrefix(attemptNo);
856
- const refs = [];
857
- if (content.prompt !== undefined) {
858
- refs.push(this.artifactStore.write(runId, nodeKey, `${prefix}.prompt.md`, content.prompt).uri);
859
- }
860
- if (content.responseText !== undefined) {
861
- refs.push(this.artifactStore.write(runId, nodeKey, `${prefix}.response.md`, content.responseText).uri);
862
- }
863
- if (content.transcript !== undefined) {
864
- // Transcript is pre-created and append-built while acpx runs. Finalization
865
- // returns the live artifact ref without overwriting accumulated NDJSON.
866
- refs.push(this.artifactStore.append(runId, nodeKey, `${prefix}.transcript.jsonl`, "").uri);
867
- }
868
- if (content.stderr !== undefined) {
869
- refs.push(this.artifactStore.write(runId, nodeKey, `${prefix}.stderr.log`, content.stderr).uri);
870
- }
871
- return refs;
872
- }
873
- /** Pre-create live Agent attempt artifacts and return their refs immediately. */
874
- startAgentAttemptArtifacts(runId, nodeKey, attemptNo, prompt) {
875
- const prefix = this.agentAttemptPrefix(attemptNo);
876
- const promptRef = this.artifactStore.write(runId, nodeKey, `${prefix}.prompt.md`, prompt);
877
- const transcriptRef = this.artifactStore.create(runId, nodeKey, `${prefix}.transcript.jsonl`);
878
- return [promptRef.uri, transcriptRef.uri];
879
- }
880
- appendAgentTranscript(runId, nodeKey, attemptNo, chunk) {
881
- this.artifactStore.append(runId, nodeKey, `${this.agentAttemptPrefix(attemptNo)}.transcript.jsonl`, chunk);
882
- }
883
- publishRunningAgentAttempt(runId, nodeKey, prompt, artifactRefs) {
665
+ publishRunningAgentAttempt(runId, nodeKey, artifactRefs, attemptTelemetry) {
884
666
  const state = this.store.readNodeState(runId, nodeKey);
885
667
  if (!state)
886
668
  return;
887
- state.renderedPrompt = prompt;
888
669
  state.artifactRefs = [...artifactRefs];
670
+ state.agentTelemetry = upsertAgentAttemptTelemetry(state.agentTelemetry, attemptTelemetry);
889
671
  this.store.writeNodeState(runId, state);
890
672
  }
891
- agentAttemptPrefix(attemptNo) {
892
- return `attempt-${String(Math.max(0, attemptNo)).padStart(3, "0")}`;
673
+ publishAgentTelemetry(runId, nodeKey, attemptTelemetry) {
674
+ const state = this.store.readNodeState(runId, nodeKey);
675
+ if (!state)
676
+ return;
677
+ state.agentTelemetry = upsertAgentAttemptTelemetry(state.agentTelemetry, attemptTelemetry);
678
+ this.store.writeNodeState(runId, state);
679
+ }
680
+ syncAgentTelemetryFromDisk(runId, nodeKey, state) {
681
+ state.agentTelemetry = this.store.readNodeState(runId, nodeKey)?.agentTelemetry ?? state.agentTelemetry;
893
682
  }
894
- async executeParallel(node, ctx, runId, dynamic, keyPrefix) {
683
+ async executeParallel(node, ctx, runId, dynamic, nodeKey, keyPrefix) {
895
684
  const children = node.children ?? [];
896
685
  const maxConcurrency = node.metadata.max_concurrency ?? this.maxConcurrency;
897
686
  const join = node.metadata.join ?? "all";
687
+ if (join === "all") {
688
+ children.forEach((child, index) => {
689
+ this.materializePendingNode(runId, child, nestedParallelBranchDynamic(dynamic, index), keyPrefix);
690
+ });
691
+ }
898
692
  const limit = pLimit(maxConcurrency);
899
693
  // Each branch resolves to { child, output } so race can identify the winner.
900
694
  const branchPromises = children.map((child, index) => limit(async () => {
901
- const branchDynamic = { ...dynamic, parallelBranchId: String(index) };
695
+ const branchDynamic = nestedParallelBranchDynamic(dynamic, index);
902
696
  let output;
903
697
  try {
904
698
  output = await this.executeNode(child, { ...ctx, steps: { ...ctx.steps } }, runId, branchDynamic, keyPrefix);
@@ -921,11 +715,11 @@ export class WorkflowInterpreter {
921
715
  branchPromises.forEach((p) => void p.catch(() => undefined));
922
716
  const mapOutput = { [winner.child.id]: winner.output };
923
717
  ctx.steps[winner.child.id] = winner.output;
924
- return mapOutput;
718
+ return { output: mapOutput };
925
719
  }
926
720
  catch (error) {
927
721
  if (!(error instanceof NodeAbortedError)) {
928
- this.cancelDescendantsInScope(runId, node, dynamic);
722
+ this.runControl.cancelDescendantsInScope(runId, nodeKey);
929
723
  }
930
724
  branchPromises.forEach((p) => void p.catch(() => undefined));
931
725
  throw error;
@@ -943,7 +737,7 @@ export class WorkflowInterpreter {
943
737
  catch (error) {
944
738
  if (!(error instanceof NodeAbortedError)) {
945
739
  // Genuine failure or cancel — fast-stop: cancel still-running siblings
946
- this.cancelDescendantsInScope(runId, node, dynamic);
740
+ this.runControl.cancelDescendantsInScope(runId, nodeKey);
947
741
  }
948
742
  branchPromises.forEach((p) => void p.catch(() => undefined));
949
743
  throw error;
@@ -953,65 +747,9 @@ export class WorkflowInterpreter {
953
747
  mapOutput[child.id] = output;
954
748
  ctx.steps[child.id] = output;
955
749
  }
956
- return mapOutput;
750
+ return { output: mapOutput };
957
751
  }
958
- /**
959
- * Cancel still-running/pending/awaiting descendant nodes of a failed
960
- * composite (parallel or fanout), scoped to the given dynamic context. A
961
- * descendant must (a) be one of the composite's descendant IR node ids and
962
- * (b) share every dynamic dimension (item/lane/round) the composite already
963
- * carries — so concurrent sibling fanout lanes outside this scope are not
964
- * affected.
965
- *
966
- * NOTE: pass the composite's OWN dynamic. For a parallel this scopes to the
967
- * current lane (item/lane present). For a fanout, the dynamic does NOT yet
968
- * carry this fanout's lane (lanes are introduced for its children), so the
969
- * scope spans ALL of the fanout's lanes — exactly the fail-fast intent.
970
- */
971
- descendantsInScope(runId, node, dynamic, action) {
972
- const descendantIds = new Set();
973
- const collect = (n) => {
974
- for (const c of n.children ?? []) {
975
- descendantIds.add(c.id);
976
- collect(c);
977
- }
978
- for (const b of n.branches ?? []) {
979
- for (const c of b.children) {
980
- descendantIds.add(c.id);
981
- collect(c);
982
- }
983
- }
984
- };
985
- collect(node);
986
- // Key segments that must be present for an instance to share this scope.
987
- // Values are sanitized to match resolveNodeKey's on-disk encoding (e.g. an
988
- // item id "file:alpha" is stored as "item:file_alpha").
989
- const scopeSegments = [];
990
- if (dynamic.fanoutItemId !== undefined)
991
- scopeSegments.push(`item:${sanitizeValue(String(dynamic.fanoutItemId))}`);
992
- if (dynamic.laneId !== undefined)
993
- scopeSegments.push(`lane:${sanitizeValue(String(dynamic.laneId))}`);
994
- if (dynamic.loopRound !== undefined)
995
- scopeSegments.push(`round:${dynamic.loopRound}`);
996
- for (const ns of this.store.listNodeStates(runId)) {
997
- if (ns.state !== "running" && ns.state !== "awaiting" && ns.state !== "pending")
998
- continue;
999
- if (!descendantIds.has(ns.nodeId))
1000
- continue;
1001
- const segs = ns.nodeKey.split("/");
1002
- if (!scopeSegments.every((s) => segs.includes(s)))
1003
- continue;
1004
- action(runId, ns.nodeKey);
1005
- }
1006
- }
1007
- /**
1008
- * Cancel still-running/pending descendant nodes of a failed composite
1009
- * (parallel `join: all`, or fanout `join: all` fail-fast).
1010
- */
1011
- cancelDescendantsInScope(runId, node, dynamic) {
1012
- this.descendantsInScope(runId, node, dynamic, (rid, key) => this.cancelMaterializedNode(rid, key));
1013
- }
1014
- async executeFanout(node, ctx, runId, dynamic, keyPrefix) {
752
+ async executeFanout(node, ctx, runId, dynamic, nodeKey, keyPrefix) {
1015
753
  const overExpr = node.metadata.over;
1016
754
  if (!overExpr) {
1017
755
  throw new Error(`fanout node ${node.id} missing 'over' expression`);
@@ -1026,6 +764,24 @@ export class WorkflowInterpreter {
1026
764
  // Success target (how many lanes must succeed). Default follows join.
1027
765
  const defaultMinSuccess = join === "race" ? 1 : join === "quorum" ? (quorum ?? 1) : items.length;
1028
766
  const minSuccess = successCriteria?.min_success ?? defaultMinSuccess;
767
+ const lanePlan = items.map((item, index) => {
768
+ const keyCtx = { ...ctx, item, item_index: index };
769
+ const itemId = this.extractItemId(item, node.metadata.key, index, keyCtx, this.evaluator);
770
+ const laneDynamic = { fanoutItemId: itemId, laneId: String(index) };
771
+ return {
772
+ itemId,
773
+ keyCtx,
774
+ itemDynamic: laneDynamic
775
+ };
776
+ });
777
+ if (join === "all") {
778
+ for (const lane of lanePlan) {
779
+ const itemDynamic = { ...dynamic, ...lane.itemDynamic };
780
+ for (const child of children) {
781
+ this.materializePendingNode(runId, child, itemDynamic, keyPrefix);
782
+ }
783
+ }
784
+ }
1029
785
  // Fail-fast: once enough lanes have failed that the success target is
1030
786
  // unreachable (failures > total - minSuccess), abort the whole fanout —
1031
787
  // cancel every still-running/pending lane subtree and reject to short
@@ -1037,14 +793,12 @@ export class WorkflowInterpreter {
1037
793
  // Each lane resolves to a LaneResult. A tolerable failure is captured (does
1038
794
  // not reject) so the join/min_success logic can run. A NodeAbortedError
1039
795
  // (operator pause/cancel) re-throws so it propagates to the parent.
1040
- const lanePromises = items.map((item, index) => limit(async () => {
1041
- const keyCtx = { ...ctx, item, item_index: index };
1042
- const itemId = this.extractItemId(item, node.metadata.key, index, keyCtx, this.evaluator);
1043
- const itemDynamic = { ...dynamic, fanoutItemId: itemId, laneId: String(index) };
796
+ const lanePromises = lanePlan.map((lane) => limit(async () => {
797
+ const itemDynamic = { ...dynamic, ...lane.itemDynamic };
1044
798
  const itemCtx = {
1045
- ...keyCtx,
799
+ ...lane.keyCtx,
1046
800
  steps: { ...ctx.steps },
1047
- item_id: itemId
801
+ item_id: lane.itemId
1048
802
  };
1049
803
  try {
1050
804
  let laneOutput;
@@ -1071,7 +825,7 @@ export class WorkflowInterpreter {
1071
825
  // so it spans every lane), then reject to short-circuit the wait.
1072
826
  if (!failFastTriggered) {
1073
827
  failFastTriggered = true;
1074
- this.cancelDescendantsInScope(runId, node, dynamic);
828
+ this.runControl.cancelDescendantsInScope(runId, nodeKey);
1075
829
  }
1076
830
  throw error;
1077
831
  }
@@ -1085,7 +839,7 @@ export class WorkflowInterpreter {
1085
839
  throw new Error(`fanout ${node.id}: ${successes.length} successful lanes, requires ${minSuccess}`);
1086
840
  }
1087
841
  // outputMerge: "array" of successful lane outputs.
1088
- return successes.map((r) => r.output);
842
+ return { output: successes.map((r) => r.output) };
1089
843
  }
1090
844
  /**
1091
845
  * Resolve fanout lanes per the wait strategy. Lanes never reject on normal
@@ -1116,6 +870,13 @@ export class WorkflowInterpreter {
1116
870
  // and the rejection propagates to fail the fanout immediately.
1117
871
  return Promise.all(lanePromises);
1118
872
  }
873
+ materializePendingNode(runId, node, dynamic, keyPrefix) {
874
+ const resolved = resolveNodeKey(node.keyTemplate, dynamic);
875
+ const nodeKey = withNodeKeyPrefix(keyPrefix, resolved);
876
+ if (this.store.readNodeState(runId, nodeKey))
877
+ return;
878
+ this.store.writeNodeState(runId, createInitialNodeState(nodeKey, node.id, node.kind, hashIrNode(node)));
879
+ }
1119
880
  async executeSwitch(node, ctx, runId, dynamic, keyPrefix) {
1120
881
  const branches = node.branches ?? [];
1121
882
  for (const branch of branches) {
@@ -1129,11 +890,11 @@ export class WorkflowInterpreter {
1129
890
  }
1130
891
  catch (error) {
1131
892
  if (error instanceof ScopeCompleted)
1132
- return error.output;
893
+ return { output: error.output };
1133
894
  throw error;
1134
895
  }
1135
896
  }
1136
- return lastOutput;
897
+ return { output: lastOutput };
1137
898
  }
1138
899
  }
1139
900
  else {
@@ -1145,11 +906,11 @@ export class WorkflowInterpreter {
1145
906
  }
1146
907
  catch (error) {
1147
908
  if (error instanceof ScopeCompleted)
1148
- return error.output;
909
+ return { output: error.output };
1149
910
  throw error;
1150
911
  }
1151
912
  }
1152
- return lastOutput;
913
+ return { output: lastOutput };
1153
914
  }
1154
915
  }
1155
916
  throw new Error(`Switch node ${node.id}: no branch matched and no default`);
@@ -1178,13 +939,13 @@ export class WorkflowInterpreter {
1178
939
  }
1179
940
  catch (error) {
1180
941
  if (error instanceof ScopeCompleted)
1181
- return error.output;
942
+ return { output: error.output };
1182
943
  throw error;
1183
944
  }
1184
945
  }
1185
946
  }
1186
947
  // outputMerge: "last"
1187
- return lastOutput;
948
+ return { output: lastOutput };
1188
949
  }
1189
950
  executeGuard(node, ctx) {
1190
951
  const when = node.metadata.when;
@@ -1196,15 +957,15 @@ export class WorkflowInterpreter {
1196
957
  if (action !== "continue" && action !== "fail" && action !== "complete") {
1197
958
  throw new Error(`guard node ${node.id}: action must be continue, fail, or complete`);
1198
959
  }
1199
- const messageTemplate = node.metadata.message;
1200
- const message = typeof messageTemplate === "string" ? this.evaluator.evaluateTemplate(messageTemplate, ctx) : undefined;
1201
- const output = { matched, action };
1202
- if (message !== undefined)
1203
- output.message = message;
960
+ const guardOutput = { matched, action };
1204
961
  if (action === "fail") {
1205
- throw new GuardFailureError(message ?? `Guard '${node.id}' failed`, output);
962
+ const messageTemplate = node.metadata.message;
963
+ const message = typeof messageTemplate === "string" ? this.evaluator.evaluateTemplate(messageTemplate, ctx) : undefined;
964
+ if (message !== undefined)
965
+ guardOutput.message = message;
966
+ throw new GuardFailureError(message ?? `Guard '${node.id}' failed`, { output: guardOutput });
1206
967
  }
1207
- return { output, completeScope: action === "complete" };
968
+ return { output: { output: guardOutput }, completeScope: action === "complete" };
1208
969
  }
1209
970
  async executeApproval(node, ctx, runId, signal, nodeKey) {
1210
971
  const timeout = node.metadata.timeout;
@@ -1225,7 +986,7 @@ export class WorkflowInterpreter {
1225
986
  cleanup();
1226
987
  // Honor the operator intent recorded by Run-level pause/cancel. An
1227
988
  // awaiting gate is normally cancelled (awaiting → cancelled).
1228
- reject(new NodeAbortedError(nodeKey, this.abortIntent(runId, nodeKey)));
989
+ reject(new NodeAbortedError(nodeKey, this.runControl.abortIntent(runId, nodeKey)));
1229
990
  };
1230
991
  const timer = timeoutMs
1231
992
  ? setTimeout(() => {
@@ -1234,10 +995,10 @@ export class WorkflowInterpreter {
1234
995
  // `approve`/`reject` resolve; `fail`/`escalate` fail the node
1235
996
  // (escalate has no runtime channel yet — see roadmap).
1236
997
  if (onTimeout === "approve") {
1237
- resolve({ approved: true, decision: "timeout", at });
998
+ resolve({ output: { approved: true, decision: "timeout", at } });
1238
999
  }
1239
1000
  else if (onTimeout === "reject") {
1240
- resolve({ approved: false, decision: "timeout", at });
1001
+ resolve({ output: { approved: false, decision: "timeout", at } });
1241
1002
  }
1242
1003
  else {
1243
1004
  reject(new Error(`Approval timed out after ${timeout} (on_timeout: ${onTimeout ?? "fail"})`));
@@ -1257,7 +1018,7 @@ export class WorkflowInterpreter {
1257
1018
  // cleanup().
1258
1019
  this.approvalResolvers.set(resolverKey, (approved) => {
1259
1020
  cleanup();
1260
- resolve({ approved, decision: approved ? "approved" : "rejected", at });
1021
+ resolve({ output: { approved, decision: approved ? "approved" : "rejected", at } });
1261
1022
  });
1262
1023
  });
1263
1024
  }
@@ -1305,7 +1066,7 @@ export class WorkflowInterpreter {
1305
1066
  // template string.
1306
1067
  const childInput = {};
1307
1068
  for (const [key, value] of Object.entries(inputSpec ?? {})) {
1308
- childInput[key] = this.evaluateInputValue(value, ctx);
1069
+ childInput[key] = evaluateTemplatedValue(value, ctx, this.evaluator);
1309
1070
  }
1310
1071
  // Validate subworkflow input against the child IR's compiled input schema.
1311
1072
  const validatedChildInput = validateInput(compiled.ir.input, childInput);
@@ -1315,7 +1076,7 @@ export class WorkflowInterpreter {
1315
1076
  try {
1316
1077
  const childCtx = this.buildContext(validatedChildInput, runId);
1317
1078
  await this.executeNode(compiled.ir.root, childCtx, runId, dynamic, nodeKey);
1318
- return { ...childCtx.steps };
1079
+ return { output: evaluateWorkflowOutputs(compiled.ir, childCtx, this.evaluator) };
1319
1080
  }
1320
1081
  finally {
1321
1082
  this.subworkflowStack.delete(childAbs);
@@ -1328,16 +1089,6 @@ export class WorkflowInterpreter {
1328
1089
  return;
1329
1090
  throw new Error(message);
1330
1091
  }
1331
- /** Evaluate a subworkflow input value, preserving native type for single expressions. */
1332
- evaluateInputValue(value, ctx) {
1333
- if (typeof value !== "string")
1334
- return value;
1335
- const single = value.match(/^\s*\$\{\{(.+)\}\}\s*$/s);
1336
- if (single) {
1337
- return this.evaluator.evaluateExpression(single[1].trim(), ctx);
1338
- }
1339
- return this.evaluator.evaluateTemplate(value, ctx);
1340
- }
1341
1092
  // ─── Helpers ────────────────────────────────────────────────────
1342
1093
  buildContext(input, runId) {
1343
1094
  return { input, steps: {}, run_id: runId };
@@ -1395,17 +1146,8 @@ export class WorkflowInterpreter {
1395
1146
  }
1396
1147
  /** Extract the step ID from a resolved node key. The ID is the last path segment before dynamic dims. */
1397
1148
  extractNodeIdFromKey(nodeKey) {
1398
- // Key format: "workflow/step-a" or "workflow/mapped/item:0/lane:0"
1399
- // The node id is the second segment (after "workflow/") for top-level steps
1400
- // For nested, it's the segment before any "type:value" segments
1401
- const segments = nodeKey.split("/");
1402
- // Find the last segment that is NOT a "type:value" dynamic segment
1403
- for (let i = segments.length - 1; i >= 0; i--) {
1404
- if (!segments[i].includes(":")) {
1405
- return segments[i];
1406
- }
1407
- }
1408
- return segments[segments.length - 1];
1149
+ const staticPath = staticNodePathFromKey(nodeKey);
1150
+ return staticPath.split("/").at(-1) ?? nodeKey;
1409
1151
  }
1410
1152
  findNodeById(root, nodeId) {
1411
1153
  if (root.id === nodeId)
@@ -1446,25 +1188,28 @@ export function generateRunId(now = new Date()) {
1446
1188
  ].join("");
1447
1189
  return `${timestamp}${randomBytes(10).toString("hex").toUpperCase()}`;
1448
1190
  }
1191
+ function nestedParallelBranchDynamic(dynamic, branchIndex) {
1192
+ const branchId = String(branchIndex);
1193
+ return {
1194
+ ...dynamic,
1195
+ parallelBranchId: dynamic.parallelBranchId === undefined ? branchId : `${dynamic.parallelBranchId}.${branchId}`
1196
+ };
1197
+ }
1449
1198
  class NodeAbortedError extends Error {
1450
1199
  nodeKey;
1451
1200
  state;
1452
1201
  artifactRefs;
1453
1202
  output;
1454
- renderedPrompt;
1455
1203
  constructor(nodeKey, state,
1456
- /** Artifact refs (e.g. partial transcript) to persist on the aborted node. */
1204
+ /** Artifact refs (e.g. partial Agent response/telemetry) to persist on the aborted node. */
1457
1205
  artifactRefs,
1458
1206
  /** Partial output captured before the abort. */
1459
- output,
1460
- /** Rendered Agent prompt to preserve when abort overwrites node state. */
1461
- renderedPrompt) {
1207
+ output) {
1462
1208
  super(`Node ${nodeKey} aborted: ${state}`);
1463
1209
  this.nodeKey = nodeKey;
1464
1210
  this.state = state;
1465
1211
  this.artifactRefs = artifactRefs;
1466
1212
  this.output = output;
1467
- this.renderedPrompt = renderedPrompt;
1468
1213
  this.name = "NodeAbortedError";
1469
1214
  }
1470
1215
  }
@@ -1476,6 +1221,9 @@ class ScopeCompleted extends Error {
1476
1221
  this.name = "ScopeCompleted";
1477
1222
  }
1478
1223
  }
1224
+ function errorMessage(error) {
1225
+ return error instanceof Error ? error.message : String(error);
1226
+ }
1479
1227
  class GuardFailureError extends Error {
1480
1228
  output;
1481
1229
  constructor(message, output) {