@xzzpig/pi-subagents 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +96 -0
  2. package/README.md +2 -0
  3. package/agents/oracle.md +7 -5
  4. package/agents/researcher.md +1 -1
  5. package/agents/reviewer.md +2 -2
  6. package/agents/scout.md +1 -1
  7. package/agents/worker.md +1 -1
  8. package/docs/agents.md +3 -0
  9. package/docs/configuration.md +121 -10
  10. package/docs/extension-api.md +36 -0
  11. package/docs/missions.md +5 -3
  12. package/docs/observability.md +43 -3
  13. package/docs/tool-reference.md +20 -3
  14. package/docs/workflows.md +2 -2
  15. package/package.json +4 -4
  16. package/skills/pi-subagents/references/constraints-and-recipes.md +3 -2
  17. package/skills/pi-subagents/references/execution-controls.md +5 -4
  18. package/skills/pi-subagents/references/management-authoring-rpc.md +2 -2
  19. package/skills/pi-subagents/references/prompting-and-roles.md +31 -15
  20. package/src/agents/agent-serializer.ts +2 -0
  21. package/src/agents/agents.ts +46 -15
  22. package/src/api/external-runs.ts +174 -84
  23. package/src/api/preflight.ts +13 -7
  24. package/src/extension/config.ts +59 -0
  25. package/src/extension/doctor.ts +40 -0
  26. package/src/extension/index.ts +83 -9
  27. package/src/extension/public-execution.ts +37 -4
  28. package/src/extension/rpc.ts +8 -10
  29. package/src/extension/schemas.ts +12 -10
  30. package/src/extension/tool-description.ts +13 -9
  31. package/src/inspectors/herdr/actions.ts +13 -8
  32. package/src/inspectors/herdr/inspector-runner.ts +16 -3
  33. package/src/inspectors/herdr/project-panes.ts +2 -6
  34. package/src/inspectors/herdr/shell-command.ts +16 -0
  35. package/src/intercom/intercom-bridge.ts +9 -5
  36. package/src/intercom/native-supervisor-channel.ts +19 -42
  37. package/src/missions/goal-driver.ts +3 -1
  38. package/src/missions/lifecycle.ts +4 -7
  39. package/src/missions/store.ts +12 -7
  40. package/src/missions/workflow-state.ts +6 -2
  41. package/src/runs/background/active-async-capacity.ts +431 -0
  42. package/src/runs/background/active-run-index.ts +77 -3
  43. package/src/runs/background/async-execution.ts +190 -72
  44. package/src/runs/background/async-job-tracker.ts +5 -0
  45. package/src/runs/background/async-resume.ts +21 -7
  46. package/src/runs/background/async-status-snapshot.ts +277 -0
  47. package/src/runs/background/async-status.ts +17 -6
  48. package/src/runs/background/chain-append.ts +33 -15
  49. package/src/runs/background/chain-root-attachment.ts +2 -2
  50. package/src/runs/background/completion-replay.ts +11 -1
  51. package/src/runs/background/fleet-view.ts +21 -6
  52. package/src/runs/background/owned-process-tree.ts +104 -0
  53. package/src/runs/background/process-terminal.ts +17 -3
  54. package/src/runs/background/result-files.ts +437 -0
  55. package/src/runs/background/result-watcher.ts +188 -41
  56. package/src/runs/background/resume-guidance.ts +27 -7
  57. package/src/runs/background/retained-children.ts +75 -18
  58. package/src/runs/background/run-id-resolver.ts +30 -24
  59. package/src/runs/background/run-status.ts +106 -5
  60. package/src/runs/background/scheduled-runs.ts +54 -28
  61. package/src/runs/background/stale-run-reconciler.ts +30 -16
  62. package/src/runs/background/subagent-runner.ts +352 -62
  63. package/src/runs/background/subagent-wait.ts +2 -0
  64. package/src/runs/background/wait-completions.ts +5 -2
  65. package/src/runs/foreground/async-dismiss-action.ts +2 -1
  66. package/src/runs/foreground/chain-execution.ts +53 -2
  67. package/src/runs/foreground/execution.ts +309 -34
  68. package/src/runs/foreground/foreground-control.ts +12 -0
  69. package/src/runs/foreground/prompt-audit.ts +172 -0
  70. package/src/runs/foreground/subagent-executor.ts +936 -224
  71. package/src/runs/shared/acceptance.ts +13 -4
  72. package/src/runs/shared/completion-guard.ts +107 -1
  73. package/src/runs/shared/external-cli-runner.ts +4 -0
  74. package/src/runs/shared/llm-intent-arbiter.ts +302 -0
  75. package/src/runs/shared/model-fallback.ts +16 -2
  76. package/src/runs/shared/nested-events.ts +66 -62
  77. package/src/runs/shared/orca-progress-tabs.ts +375 -0
  78. package/src/runs/shared/parallel-utils.ts +4 -0
  79. package/src/runs/shared/pi-args.ts +44 -1
  80. package/src/runs/shared/run-fanout-budget.ts +280 -0
  81. package/src/runs/shared/single-output.ts +4 -2
  82. package/src/runs/shared/subagent-control.ts +15 -0
  83. package/src/runs/shared/subagent-prompt-runtime.ts +1 -9
  84. package/src/runs/shared/subagent-startup-retry.ts +12 -0
  85. package/src/runs/shared/task-intent.ts +19 -3
  86. package/src/runs/shared/tool-timeout.ts +93 -0
  87. package/src/runs/shared/worktree.ts +17 -5
  88. package/src/shared/agent-stream-options.ts +5 -0
  89. package/src/shared/artifacts.ts +2 -6
  90. package/src/shared/display-text.ts +50 -0
  91. package/src/shared/node-executable.ts +21 -0
  92. package/src/shared/types.ts +143 -5
  93. package/src/shared/utils.ts +3 -1
  94. package/src/slash/slash-commands.ts +34 -25
  95. package/src/slash/slash-live-state.ts +3 -0
  96. package/src/tui/fleet-status.ts +164 -47
  97. package/src/tui/fleet-transcript.ts +1 -48
  98. package/src/tui/fleet.ts +352 -27
  99. package/src/tui/render.ts +122 -44
  100. package/src/watchdog/permission-arbiter.ts +2 -1
  101. package/src/watchdog/review.ts +4 -3
  102. package/src/workflows/chat-progress.ts +10 -2
  103. package/src/workflows/scripted-workflow.ts +292 -79
@@ -5,6 +5,7 @@ import * as path from "node:path";
5
5
  import { pathToFileURL } from "node:url";
6
6
  import type { Message } from "@earendil-works/pi-ai";
7
7
  import { writeAtomicJson } from "../../shared/atomic-json.ts";
8
+ import { writeAsyncResultFile, writePendingAsyncResultFile } from "./result-files.ts";
8
9
  import { createFileCoalescer } from "../../shared/file-coalescer.ts";
9
10
  import { isActiveAsyncState, updateActiveRunIndex } from "./active-run-index.ts";
10
11
  import { createChildTranscriptWriter, type ChildTranscriptWriter } from "../../shared/child-transcript.ts";
@@ -25,11 +26,14 @@ import {
25
26
  type LaunchResolvedChildExtensionsV1,
26
27
  type RuntimeAcknowledgedChildExtensionsV1,
27
28
  type ModelAttempt,
29
+ type PiWriterProcessInstanceExitV1,
30
+ type ProcessTreeTerminalV1,
28
31
  type NestedRouteInfo,
29
32
  type NestedRunSummary,
30
33
  type ResolvedControlConfig,
31
34
  type ResolvedTurnBudget,
32
35
  type ResolvedToolBudget,
36
+ type RunFanoutBudgetDescriptor,
33
37
  type SubagentRunMode,
34
38
  type SubagentOutputState,
35
39
  type UsageBudgetConfig,
@@ -54,6 +58,7 @@ import {
54
58
  claimControlNotification,
55
59
  formatControlIntercomMessage,
56
60
  formatControlNoticeMessage,
61
+ shouldEmitOpenToolAttention,
57
62
  } from "../shared/subagent-control.ts";
58
63
  import {
59
64
  type RunnerSubagentStep as SubagentStep,
@@ -68,26 +73,29 @@ import {
68
73
  DEFAULT_GLOBAL_CONCURRENCY_LIMIT,
69
74
  Semaphore,
70
75
  } from "../shared/parallel-utils.ts";
71
- import { applyThinkingSuffix, buildPiArgs, cleanupTempDir, projectLaunchResolvedChildExtensions, resolvePiLaunchToolPlan } from "../shared/pi-args.ts";
76
+ import { applyThinkingSuffix, buildPiArgs, cleanupTempDir, projectLaunchResolvedChildExtensions, resolvePiLaunchToolPlan, type SubagentTaskDelivery } from "../shared/pi-args.ts";
72
77
  import { readRuntimeAcknowledgedExtensions } from "../shared/runtime-acknowledged-extensions.ts";
73
78
  import { outputEntryFromAsyncResult, resolveOutputReferences } from "../shared/chain-outputs.ts";
74
79
  import { createStructuredOutputRuntime, MISSING_STRUCTURED_OUTPUT_CALL_ERROR, readStructuredOutput } from "../shared/structured-output.ts";
75
80
  import { formatProcessSignalError, isUnexplainedProcessSignal } from "../shared/process-signal.ts";
76
81
  import { readChildToolDiagnosticError } from "../shared/tool-availability.ts";
77
82
  import { collectDynamicResults, DynamicFanoutError, materializeDynamicParallelStep, validateDynamicCollection } from "../shared/dynamic-fanout.ts";
83
+ import { claimRunFanoutBatch, getRunFanoutBudgetSnapshot } from "../shared/run-fanout-budget.ts";
78
84
  import { nestedSummaryFromAsyncStatus, projectNestedEvents, resolveNestedAsyncDir, writeNestedEvent } from "../shared/nested-events.ts";
79
85
  import { formatModelAttemptNote, isRetryableModelFailure } from "../shared/model-fallback.ts";
80
86
  import {
81
87
  SUBAGENT_STARTUP_RETRY_DELAYS_MS,
88
+ formatSubagentExtensionConflictError,
82
89
  formatSubagentStartupRetryExhaustedError,
83
90
  formatSubagentStartupRetryNote,
84
91
  isRetryableSubagentStartupFailure,
85
92
  waitForSubagentStartupRetry,
86
93
  } from "../shared/subagent-startup-retry.ts";
87
94
  import { markProcessTerminalCandidateLeaseRelease, writeProcessTerminalCandidate, type ProcessTerminalCandidate } from "./process-terminal.ts";
95
+ import { createOwnedProcessTreeController, type OwnedProcessTreeController } from "./owned-process-tree.ts";
88
96
  import { createSteeringStatus, recordSteeringRequest, steeringStatus, terminalSteeringNoticeState, updateSteeringTarget } from "./steering.ts";
89
97
  import { attachPostExitStdioGuard, trySignalChild } from "../../shared/post-exit-stdio-guard.ts";
90
- import { detectSubagentError, extractTextFromContent, extractToolArgsPreview, getFinalOutput, hasEmptyTerminalAssistantResponse, readStatus } from "../../shared/utils.ts";
98
+ import { PROMPT_REDACTED, detectSubagentError, extractTextFromContent, extractToolArgsPreview, getFinalOutput, hasEmptyTerminalAssistantResponse, readStatus } from "../../shared/utils.ts";
91
99
  import { evaluateCompletionMutationGuard } from "../shared/completion-guard.ts";
92
100
  import {
93
101
  createMutatingFailureState,
@@ -121,12 +129,14 @@ import { waitForImportedAsyncRoot } from "./chain-root-attachment.ts";
121
129
  import { appendRunnerStepsToStatus, consumeChainAppendRequests, countPendingChainAppendRequests, statusStepDescription } from "./chain-append.ts";
122
130
  import { appendTurnBudgetSystemPrompt, formatTurnBudgetOutput, initialTurnBudgetState, turnBudgetDecision, turnBudgetDeferredNote, turnBudgetDeferredState, turnBudgetExceededMessage, turnBudgetSoftNote, turnBudgetState } from "../shared/turn-budget.ts";
123
131
  import { initialToolBudgetState, toolBudgetState } from "../shared/tool-budget.ts";
132
+ import { effectiveToolTimeoutMs, formatToolTimeoutMessage, toolTimeoutCallKey } from "../shared/tool-timeout.ts";
124
133
  import { usageBudgetExceededMessage, usageBudgetState } from "../shared/usage-budget.ts";
125
134
  import { formatParallelHandoffError, formatParallelHandoffReference, parallelHandoffPath, writeParallelHandoffGroup, writePendingParallelHandoff } from "../shared/parallel-handoff.ts";
126
135
  import { resolveWatchdogConfig } from "../../watchdog/settings.ts";
127
136
  import { createBoundedByteTail, createBoundedLineReader, formatProtocolOutputLimit, MAX_CHILD_STDERR_BYTES, PI_AGGREGATE_EVENT_PROJECTOR, projectChildLifecycle, type ChildLifecycleAction, type ProtocolOutputLimit } from "../shared/child-protocol.ts";
128
137
  import { acquireSessionLease, type SessionLeaseRequest } from "../shared/session-lease.ts";
129
138
  import { buildExternalCliPrompt, runExternalCli } from "../shared/external-cli-runner.ts";
139
+ import { createOrcaProgressTab, type OrcaProgressTab } from "../shared/orca-progress-tabs.ts";
130
140
  import { decodeSubagentCapabilityCeiling, SUBAGENT_CAPABILITY_CEILING_ENV, type ResolvedSubagentCapabilityCeiling } from "../shared/capability-ceiling.ts";
131
141
  import {
132
142
  CHILD_WATCHDOG_CONFIG_ENV,
@@ -138,6 +148,8 @@ import {
138
148
  type ChildWatchdogStateSnapshot,
139
149
  } from "../../watchdog/child-status.ts";
140
150
 
151
+ const INTERCOM_DETACH_RECEIPT = "Detached for intercom coordination before task completion.";
152
+
141
153
  interface SubagentRunConfig {
142
154
  id: string;
143
155
  steps: RunnerStep[];
@@ -169,6 +181,8 @@ interface SubagentRunConfig {
169
181
  nestedSelf?: { parentRunId: string; parentStepIndex?: number; depth: number; path?: Array<{ runId: string; stepIndex?: number; agent?: string }> };
170
182
  timeoutMs?: number;
171
183
  deadlineAt?: number;
184
+ /** Resolved configured hard per-tool-call timeout (ms); fast tools still have a default when undefined. */
185
+ toolTimeoutMs?: number;
172
186
  turnBudget?: ResolvedTurnBudget;
173
187
  toolBudget?: ResolvedToolBudget;
174
188
  usageBudget?: UsageBudgetConfig;
@@ -177,6 +191,7 @@ interface SubagentRunConfig {
177
191
  /** Global cap on simultaneously-running subagent tasks within this run. */
178
192
  globalConcurrencyLimit?: number;
179
193
  capabilityCeiling?: ResolvedSubagentCapabilityCeiling;
194
+ runFanoutBudget?: RunFanoutBudgetDescriptor;
180
195
  launchContractDigest?: string;
181
196
  launchResolvedExtensions?: LaunchResolvedChildExtensionsV1;
182
197
  runtimeAcknowledgedExtensions?: RuntimeAcknowledgedChildExtensionsV1;
@@ -232,7 +247,7 @@ interface StepResult {
232
247
  structuredOutputSchemaPath?: string;
233
248
  acceptance?: import("../../shared/types.ts").AcceptanceLedger;
234
249
  watchdog?: import("../../shared/types.ts").ChildWatchdogProgress;
235
- writerProcesses?: Array<{ processInstanceId: string; kind: "pi-writer"; attempt: number; closeObservedAt: number; exitCode: number | null; signal: string | null }>;
250
+ writerProcesses?: PiWriterProcessInstanceExitV1[];
236
251
  writerAttemptCount?: number;
237
252
  runner?: ExternalCliRunnerStatus;
238
253
  externalProcess?: ExternalProcessStatus;
@@ -512,6 +527,7 @@ interface RunPiStreamingResult {
512
527
  processInstanceId: string;
513
528
  processCloseObservedAt?: number;
514
529
  processSignal?: string | null;
530
+ processTree: ProcessTreeTerminalV1;
515
531
  }
516
532
 
517
533
  function runPiStreaming(
@@ -532,6 +548,9 @@ function runPiStreaming(
532
548
  stopMessage?: string,
533
549
  registerTurnBudgetAbort?: (abort: ((message: string, state?: TurnBudgetState) => void) | undefined) => void,
534
550
  onWriterProcess?: (writer: { state: "none" | "spawning" } | { state: "running"; pid: number }) => void,
551
+ toolTimeoutMs?: number,
552
+ runDeadlineAt?: number,
553
+ orcaProgressTab?: OrcaProgressTab,
535
554
  ): Promise<RunPiStreamingResult> {
536
555
  return new Promise((resolve) => {
537
556
  const startedAt = Date.now();
@@ -548,7 +567,9 @@ function runPiStreaming(
548
567
  stdio: ["ignore", "pipe", "pipe"],
549
568
  env: spawnEnv,
550
569
  windowsHide: true,
570
+ detached: process.platform !== "win32",
551
571
  });
572
+ let processTreeController: OwnedProcessTreeController | undefined;
552
573
  const stderrTail = createBoundedByteTail();
553
574
  const rawStdoutTail = createBoundedByteTail();
554
575
  const messages: Message[] = [];
@@ -556,6 +577,7 @@ function runPiStreaming(
556
577
  let model: string | undefined;
557
578
  let writerRegistrationError: string | undefined;
558
579
  if (typeof child.pid === "number") {
580
+ processTreeController = createOwnedProcessTreeController(child.pid);
559
581
  try {
560
582
  onWriterProcess?.({ state: "running", pid: child.pid });
561
583
  } catch (writerError) {
@@ -585,6 +607,7 @@ function runPiStreaming(
585
607
  const writeOutputLine = (line: string) => {
586
608
  if (!line.trim()) return;
587
609
  outputStream.write(`${line}\n`);
610
+ orcaProgressTab?.append(`${line}\n`);
588
611
  };
589
612
 
590
613
  const writeOutputText = (text: string) => {
@@ -662,8 +685,14 @@ function runPiStreaming(
662
685
 
663
686
  onChildEvent?.(event);
664
687
 
688
+ if (event.type === "tool_execution_end") {
689
+ clearActiveToolTimeout(event);
690
+ return;
691
+ }
692
+
665
693
  if (event.type === "tool_execution_start" && event.toolName) {
666
694
  toolCount += 1;
695
+ armToolTimeout({ toolCallId: (event as { toolCallId?: unknown }).toolCallId, toolName: event.toolName });
667
696
  if (event.toolName === "structured_output") {
668
697
  structuredOutputToolInvoked = true;
669
698
  structuredOutputMessageStartIndex = messages.length;
@@ -703,7 +732,6 @@ function runPiStreaming(
703
732
  // a lingering stdio holder after `exit`, or a child that never exits.
704
733
  const FINAL_STOP_GRACE_MS = 1000;
705
734
  const HARD_KILL_MS = 3000;
706
- const TIMEOUT_HARD_KILL_MS = 3000;
707
735
  let childExited = false;
708
736
  let forcedTerminationSignal = false;
709
737
  let cleanTerminalAssistantStopReceived = false;
@@ -711,7 +739,6 @@ function runPiStreaming(
711
739
  let finalDrainTimer: NodeJS.Timeout | undefined;
712
740
  let finalHardKillTimer: NodeJS.Timeout | undefined;
713
741
  let watchdogTailTimer: NodeJS.Timeout | undefined;
714
- let timeoutHardKillTimer: NodeJS.Timeout | undefined;
715
742
  let turnBudgetTerminationTimer: NodeJS.Timeout | undefined;
716
743
  let turnBudgetHardKillTimer: NodeJS.Timeout | undefined;
717
744
  let protocolHardKillTimer: NodeJS.Timeout | undefined;
@@ -761,6 +788,7 @@ function runPiStreaming(
761
788
  stderrTail.push(chunk);
762
789
  stderrReader.push(chunk);
763
790
  outputStream.write(chunk);
791
+ orcaProgressTab?.append(chunk.toString("utf-8"));
764
792
  });
765
793
  registerInterrupt?.(() => {
766
794
  if (settled || timedOut || stopped) return;
@@ -771,27 +799,67 @@ function runPiStreaming(
771
799
  if (!settled && !timedOut && !stopped) trySignalChild(child, "SIGTERM");
772
800
  }, 1000).unref?.();
773
801
  });
774
- registerTimeout?.(() => {
802
+ const terminateForTimeout = (message: string): void => {
775
803
  if (settled || timedOut || stopped) return;
776
804
  timedOut = true;
805
+ // runPiStreaming's terminal result derives the timeout error from this
806
+ // message, so retain the tool-specific reason through finalization.
807
+ timeoutMessage = message;
777
808
  interrupted = false;
778
- error = timeoutMessage ?? "Subagent timed out.";
779
- trySignalChild(child, "SIGTERM");
780
- timeoutHardKillTimer = setTimeout(() => {
781
- if (!settled) trySignalChild(child, "SIGKILL");
782
- }, TIMEOUT_HARD_KILL_MS);
783
- timeoutHardKillTimer.unref?.();
784
- });
809
+ error = message;
810
+ if (processTreeController) void processTreeController.terminate();
811
+ else trySignalChild(child, "SIGTERM");
812
+ };
813
+ let toolTimeoutSequence = 0;
814
+ const activeToolTimeouts = new Map<string, { toolName: string; timer: ReturnType<typeof setTimeout> }>();
815
+ const activeToolTimeoutKeysByName = new Map<string, string[]>();
816
+ const removeToolTimeoutKey = (key: string): void => {
817
+ const active = activeToolTimeouts.get(key);
818
+ if (!active) return;
819
+ clearTimeout(active.timer);
820
+ activeToolTimeouts.delete(key);
821
+ const keys = activeToolTimeoutKeysByName.get(active.toolName)?.filter((candidate) => candidate !== key) ?? [];
822
+ if (keys.length > 0) activeToolTimeoutKeysByName.set(active.toolName, keys);
823
+ else activeToolTimeoutKeysByName.delete(active.toolName);
824
+ };
825
+ const clearActiveToolTimeout = (event: { toolCallId?: unknown; toolName?: unknown }): void => {
826
+ const key = typeof event.toolCallId === "string" && event.toolCallId.length > 0
827
+ ? `id:${event.toolCallId}`
828
+ : typeof event.toolName === "string"
829
+ ? activeToolTimeoutKeysByName.get(event.toolName)?.[0]
830
+ : activeToolTimeouts.size === 1
831
+ ? [...activeToolTimeouts.keys()][0]
832
+ : undefined;
833
+ if (key) removeToolTimeoutKey(key);
834
+ };
835
+ const clearAllToolTimeouts = (): void => {
836
+ for (const key of [...activeToolTimeouts.keys()]) removeToolTimeoutKey(key);
837
+ };
838
+ const armToolTimeout = (event: { toolCallId?: unknown; toolName: string }): void => {
839
+ const timeoutForTool = effectiveToolTimeoutMs(event.toolName, toolTimeoutMs);
840
+ if (timeoutForTool === undefined) return;
841
+ const runRemaining = runDeadlineAt === undefined ? undefined : Math.max(0, runDeadlineAt - Date.now());
842
+ if (runRemaining !== undefined && timeoutForTool >= runRemaining) return;
843
+ const key = toolTimeoutCallKey(event, ++toolTimeoutSequence);
844
+ const toolName = event.toolName;
845
+ const timer = setTimeout(() => {
846
+ removeToolTimeoutKey(key);
847
+ terminateForTimeout(formatToolTimeoutMessage(toolName, timeoutForTool));
848
+ }, timeoutForTool);
849
+ timer.unref?.();
850
+ activeToolTimeouts.set(key, { toolName, timer });
851
+ const keys = activeToolTimeoutKeysByName.get(toolName) ?? [];
852
+ keys.push(key);
853
+ activeToolTimeoutKeysByName.set(toolName, keys);
854
+ };
855
+ registerTimeout?.(() => terminateForTimeout(timeoutMessage ?? "Subagent timed out."));
785
856
  registerStop?.(() => {
786
857
  if (settled || timedOut || stopped) return;
787
858
  stopped = true;
788
859
  interrupted = false;
789
860
  error = stopMessage ?? "Subagent stopped by user.";
790
- trySignalChild(child, "SIGTERM");
791
- timeoutHardKillTimer = setTimeout(() => {
792
- if (!settled) trySignalChild(child, "SIGKILL");
793
- }, TIMEOUT_HARD_KILL_MS);
794
- timeoutHardKillTimer.unref?.();
861
+ if (processTreeController) void processTreeController.terminate();
862
+ else trySignalChild(child, "SIGTERM");
795
863
  });
796
864
  registerTurnBudgetAbort?.((message, state) => {
797
865
  if (settled || timedOut || stopped || turnBudgetExceeded) return;
@@ -811,6 +879,7 @@ function runPiStreaming(
811
879
  turnBudgetHardKillTimer.unref?.();
812
880
  });
813
881
  const clearDrainTimers = () => {
882
+ clearAllToolTimeouts();
814
883
  if (finalDrainTimer) {
815
884
  clearTimeout(finalDrainTimer);
816
885
  finalDrainTimer = undefined;
@@ -820,10 +889,6 @@ function runPiStreaming(
820
889
  finalHardKillTimer = undefined;
821
890
  }
822
891
  clearWatchdogTailTimer();
823
- if (timeoutHardKillTimer) {
824
- clearTimeout(timeoutHardKillTimer);
825
- timeoutHardKillTimer = undefined;
826
- }
827
892
  if (turnBudgetTerminationTimer) {
828
893
  clearTimeout(turnBudgetTerminationTimer);
829
894
  turnBudgetTerminationTimer = undefined;
@@ -885,9 +950,12 @@ function runPiStreaming(
885
950
  childExited = true;
886
951
  clearDrainTimers();
887
952
  });
888
- child.on("close", (exitCode, signal) => {
953
+ child.on("close", async (exitCode, signal) => {
889
954
  settled = true;
890
955
  const processCloseObservedAt = Date.now();
956
+ const processTree = processTreeController
957
+ ? await processTreeController.finishAfterWriterClose()
958
+ : { state: "unknown" as const, reason: "verification-failed" as const, diagnostic: "Writer PID was unavailable." };
891
959
  try {
892
960
  onWriterProcess?.({ state: "none" });
893
961
  } catch {
@@ -939,6 +1007,7 @@ function runPiStreaming(
939
1007
  processInstanceId,
940
1008
  processCloseObservedAt,
941
1009
  processSignal: signal,
1010
+ processTree,
942
1011
  }));
943
1012
  });
944
1013
 
@@ -961,7 +1030,7 @@ function runPiStreaming(
961
1030
  const stderr = stderrTail.text();
962
1031
  const finalOutput = getFinalOutput(messages) || rawStdoutTail.text().trim();
963
1032
  const spawnErrorMessage = spawnError instanceof Error ? spawnError.message : String(spawnError);
964
- resolve(omitUndefinedProperties({ stderr, exitCode: 1, messages, usage, toolCount, durationMs: Date.now() - startedAt, model, error: stopped ? (stopMessage ?? "Subagent stopped by user.") : timedOut ? (timeoutMessage ?? "Subagent timed out.") : turnBudgetExceeded ? turnBudgetMessage : error ?? assistantError ?? spawnErrorMessage, protocolError, finalOutput: (timedOut || stopped) && !finalOutput.trim() ? (stopped ? stopMessage ?? "Subagent stopped by user." : timeoutMessage ?? "Subagent timed out.") : finalOutput, outputState: finalOutput.trim() ? "present" : "absent", timedOut, stopped, turnBudget, turnBudgetExceeded, wrapUpRequested: turnBudget?.outcome === "wrap-up-requested" || turnBudget?.outcome === "termination-deferred" || turnBudgetExceeded || undefined, observedMutationAttempt, structuredOutputToolInvoked, structuredOutputMessageStartIndex, watchdog: childWatchdogState, processInstanceId }));
1033
+ resolve(omitUndefinedProperties({ stderr, exitCode: 1, messages, usage, toolCount, durationMs: Date.now() - startedAt, model, error: stopped ? (stopMessage ?? "Subagent stopped by user.") : timedOut ? (timeoutMessage ?? "Subagent timed out.") : turnBudgetExceeded ? turnBudgetMessage : error ?? assistantError ?? spawnErrorMessage, protocolError, finalOutput: (timedOut || stopped) && !finalOutput.trim() ? (stopped ? stopMessage ?? "Subagent stopped by user." : timeoutMessage ?? "Subagent timed out.") : finalOutput, outputState: finalOutput.trim() ? "present" : "absent", timedOut, stopped, turnBudget, turnBudgetExceeded, wrapUpRequested: turnBudget?.outcome === "wrap-up-requested" || turnBudget?.outcome === "termination-deferred" || turnBudgetExceeded || undefined, observedMutationAttempt, structuredOutputToolInvoked, structuredOutputMessageStartIndex, watchdog: childWatchdogState, processInstanceId, processTree: { state: "unknown", reason: "verification-failed", diagnostic: spawnErrorMessage } }));
965
1034
  });
966
1035
  });
967
1036
  }
@@ -1100,20 +1169,26 @@ interface SingleStepContext {
1100
1169
  stopSignal?: AbortSignal;
1101
1170
  timeoutMessage?: string;
1102
1171
  stopMessage?: string;
1172
+ /** Resolved configured hard per-tool-call timeout (ms); fast tools still have a default when undefined. */
1173
+ toolTimeoutMs?: number;
1174
+ /** Effective step deadline (Date.now() + effective timeout) when a run budget exists. */
1175
+ deadlineAt?: number;
1103
1176
  turnBudget?: ResolvedTurnBudget;
1104
1177
  childIntercomTarget?: string;
1105
1178
  orchestratorIntercomTarget?: string;
1106
1179
  nestedRoute?: NestedRouteInfo;
1107
1180
  capabilityCeiling?: ResolvedSubagentCapabilityCeiling;
1181
+ runFanoutBudget?: RunFanoutBudgetDescriptor;
1108
1182
  onAttemptStart?: (attempt: { model?: string; thinking?: string }) => void;
1109
1183
  onChildEvent?: (event: ChildEvent) => void;
1110
1184
  onWriterProcess?: (writer: { state: "none" | "spawning" } | { state: "running"; pid: number }) => void;
1111
1185
  onExternalProcess?: (process: ExternalProcessStatus) => void;
1112
1186
  skipAcceptance?: () => boolean;
1187
+ orcaProgressTab?: OrcaProgressTab;
1113
1188
  }
1114
1189
 
1115
1190
  /** Run a single pi agent step, returning output and metadata */
1116
- async function runSingleStep(
1191
+ async function runSingleStepInner(
1117
1192
  step: SubagentStep,
1118
1193
  ctx: SingleStepContext,
1119
1194
  ): Promise<StepResult & { completionGuardTriggered?: boolean }> {
@@ -1206,7 +1281,7 @@ async function runSingleStep(
1206
1281
  artifactPaths = getArtifactPaths(ctx.artifactsDir, ctx.id, step.agent, index);
1207
1282
  fs.mkdirSync(ctx.artifactsDir, { recursive: true });
1208
1283
  if (ctx.artifactConfig?.includeInput !== false) {
1209
- fs.writeFileSync(artifactPaths.inputPath, `# Task for ${step.agent}\n\n${task}`, "utf-8");
1284
+ fs.writeFileSync(artifactPaths.inputPath, `# Task for ${step.agent}\n\n${PROMPT_REDACTED}; live Prompt Audit only.\n`, "utf-8");
1210
1285
  }
1211
1286
  if (ctx.artifactConfig?.includeTranscript !== false) {
1212
1287
  transcriptWriter = createChildTranscriptWriter({
@@ -1219,7 +1294,7 @@ async function runSingleStep(
1219
1294
  });
1220
1295
  }
1221
1296
  }
1222
- transcriptWriter?.writeInitialUserMessage(task);
1297
+ transcriptWriter?.writeInitialUserMessage(`${PROMPT_REDACTED}; live Prompt Audit only.`);
1223
1298
 
1224
1299
  if (step.runner?.type === "external-cli") {
1225
1300
  const runner: ExternalCliRunnerStatus = {
@@ -1242,6 +1317,8 @@ async function runSingleStep(
1242
1317
  timeoutMessage: ctx.timeoutMessage,
1243
1318
  stopMessage: ctx.stopMessage,
1244
1319
  onProcess: ctx.onExternalProcess,
1320
+ onStdout: (chunk) => ctx.orcaProgressTab?.append(chunk.toString("utf-8")),
1321
+ onStderr: (chunk) => ctx.orcaProgressTab?.append(chunk.toString("utf-8")),
1245
1322
  }));
1246
1323
  try { fs.writeFileSync(ctx.outputFile, external.output, "utf-8"); } catch { /* Observability output is best-effort. */ }
1247
1324
  const resolvedOutput = step.outputPath && external.exitCode === 0
@@ -1262,7 +1339,7 @@ async function runSingleStep(
1262
1339
  artifactPaths,
1263
1340
  artifactConfig: ctx.artifactConfig,
1264
1341
  output: formatOutputArtifactContent(omitUndefinedProperties({ output: resolvedOutput.fullOutput, error: external.error, metadataPath: ctx.artifactConfig?.includeMetadata === false ? undefined : artifactPaths.metadataPath })),
1265
- metadata: { runId: ctx.id, agent: step.agent, task, runner, externalProcess: external.externalProcess, exitCode: external.exitCode, error: external.error, timestamp: Date.now() },
1342
+ metadata: { runId: ctx.id, agent: step.agent, task: PROMPT_REDACTED, runner, externalProcess: external.externalProcess, exitCode: external.exitCode, error: external.error, timestamp: Date.now() },
1266
1343
  })
1267
1344
  : {};
1268
1345
  return omitUndefinedProperties({
@@ -1292,7 +1369,7 @@ async function runSingleStep(
1292
1369
  let capabilityAudit: import("../shared/capability-ceiling.ts").SubagentCapabilityAudit | undefined;
1293
1370
  let launchResolvedExtensions = step.launchResolvedExtensions;
1294
1371
  const modelAttempts: ModelAttempt[] = [];
1295
- const writerProcesses: Array<{ processInstanceId: string; kind: "pi-writer"; attempt: number; closeObservedAt: number; exitCode: number | null; signal: string | null }> = [];
1372
+ const writerProcesses: PiWriterProcessInstanceExitV1[] = [];
1296
1373
  let writerAttemptCount = 0;
1297
1374
  const attemptNotes: string[] = [];
1298
1375
  const eventsPath = path.join(path.dirname(ctx.outputFile), "events.jsonl");
@@ -1306,6 +1383,9 @@ async function runSingleStep(
1306
1383
 
1307
1384
  let modelIndex = 0;
1308
1385
  let startupAttemptIndex = 0;
1386
+ // Escalated to "file" after an unexplained zero-activity startup failure so
1387
+ // retries keep the task text out of argv (endpoint pre-exec scans may deny it).
1388
+ let taskDeliveryOverride: SubagentTaskDelivery | undefined;
1309
1389
  modelAttemptsLoop: while (modelIndex < candidates.length) {
1310
1390
  if (ctx.timeoutSignal?.aborted || ctx.stopSignal?.aborted || ctx.skipAcceptance?.()) break;
1311
1391
  const candidate = candidates[modelIndex];
@@ -1331,6 +1411,7 @@ async function runSingleStep(
1331
1411
  parentSessionId: step.parentSessionId,
1332
1412
  baseArgs: ["--mode", "json", "-p"],
1333
1413
  task,
1414
+ taskDelivery: taskDeliveryOverride,
1334
1415
  sessionEnabled,
1335
1416
  sessionDir,
1336
1417
  sessionFile: step.sessionFile,
@@ -1356,6 +1437,10 @@ async function runSingleStep(
1356
1437
  parentControlInbox: ctx.nestedRoute?.controlInbox,
1357
1438
  parentRootRunId: ctx.nestedRoute?.rootRunId,
1358
1439
  parentCapabilityToken: ctx.nestedRoute?.capabilityToken,
1440
+ runFanoutBudget: ctx.runFanoutBudget ? {
1441
+ ...ctx.runFanoutBudget,
1442
+ ...(step.runFanoutPath ? { parentPath: `${ctx.runFanoutBudget.parentPath ? `${ctx.runFanoutBudget.parentPath}/` : ""}${step.runFanoutPath}` } : {}),
1443
+ } : undefined,
1359
1444
  steerInboxDir: ctx.steerInboxDir,
1360
1445
  steerCapabilityPath: ctx.steerCapabilityPath,
1361
1446
  steerAckDir: ctx.steerAckDir,
@@ -1420,6 +1505,9 @@ async function runSingleStep(
1420
1505
  ctx.stopMessage,
1421
1506
  ctx.registerTurnBudgetAbort,
1422
1507
  ctx.onWriterProcess,
1508
+ ctx.toolTimeoutMs,
1509
+ ctx.deadlineAt,
1510
+ ctx.orcaProgressTab,
1423
1511
  );
1424
1512
  if (run.processCloseObservedAt !== undefined) {
1425
1513
  writerProcesses.push({
@@ -1429,6 +1517,7 @@ async function runSingleStep(
1429
1517
  closeObservedAt: run.processCloseObservedAt,
1430
1518
  exitCode: run.exitCode,
1431
1519
  signal: run.processSignal ?? null,
1520
+ processTree: run.processTree,
1432
1521
  });
1433
1522
  }
1434
1523
  if (run.turnBudget) turnBudget = run.turnBudget;
@@ -1526,15 +1615,21 @@ async function runSingleStep(
1526
1615
  stopped: run.stopped,
1527
1616
  turnBudgetExceeded: run.turnBudgetExceeded,
1528
1617
  })) ? formatProcessSignalError(run.processSignal!) : undefined;
1529
- const error = toolAvailabilityError
1530
- ?? completionGuardError
1531
- ?? structuredError
1532
- ?? emptyOutputError
1533
- ?? (hiddenError?.hasError
1534
- ? hiddenError.details
1535
- ? `${hiddenError.errorType} failed (exit ${effectiveExitCode}): ${hiddenError.details}`
1536
- : `${hiddenError.errorType} failed with exit code ${effectiveExitCode}`
1537
- : run.error || signalError || (run.exitCode !== 0 && run.stderr.trim() ? run.stderr.trim() : undefined));
1618
+ const error = formatSubagentExtensionConflictError(
1619
+ toolAvailabilityError
1620
+ ?? completionGuardError
1621
+ ?? structuredError
1622
+ ?? emptyOutputError
1623
+ ?? (hiddenError?.hasError
1624
+ ? hiddenError.details
1625
+ ? `${hiddenError.errorType} failed (exit ${effectiveExitCode}): ${hiddenError.details}`
1626
+ : `${hiddenError.errorType} failed with exit code ${effectiveExitCode}`
1627
+ : run.error || signalError || (run.exitCode !== 0 && run.stderr.trim() ? run.stderr.trim() : undefined)),
1628
+ {
1629
+ agent: step.agent,
1630
+ ambientExtensionsEnabled: launchResolvedExtensions?.disableAmbientExtensions === false,
1631
+ },
1632
+ );
1538
1633
  const attempt: ModelAttempt = omitUndefinedProperties({
1539
1634
  model: candidate ?? run.model ?? step.model ?? "default",
1540
1635
  success: effectiveExitCode === 0 && !error,
@@ -1583,6 +1678,10 @@ async function runSingleStep(
1583
1678
  });
1584
1679
  const shouldRetry = await waitForSubagentStartupRetry(retryDelayMs, [ctx.timeoutSignal, ctx.stopSignal]);
1585
1680
  if (!shouldRetry || ctx.skipAcceptance?.()) break modelAttemptsLoop;
1681
+ if (!taskDeliveryOverride && run.processSignal === "SIGKILL") {
1682
+ taskDeliveryOverride = "file";
1683
+ attemptNotes.push("[startup-retry] retrying with file task delivery to keep the task text out of the child process argv.");
1684
+ }
1586
1685
  attempt.error = retryNote;
1587
1686
  attemptNotes.push(retryNote);
1588
1687
  startupAttemptIndex += 1;
@@ -1677,15 +1776,16 @@ async function runSingleStep(
1677
1776
  const acceptanceFailure = effectiveAcceptance ? acceptanceFailureMessage(effectiveAcceptance) : undefined;
1678
1777
  const acceptanceCanFailRun = acceptanceFailure && effectiveAcceptance?.explicit && (finalResult?.exitCode ?? 1) === 0 && !finalResult?.interrupted && !timedOutAfterAcceptance && !stoppedAfterAcceptance && !turnBudgetExceeded && !isAgentContractV1(step.agentContract);
1679
1778
  const effectiveFinalExitCode = timedOutAfterAcceptance || stoppedAfterAcceptance || turnBudgetExceeded ? 1 : acceptanceCanFailRun ? 1 : finalResult?.exitCode ?? 1;
1779
+ const intercomDetachReceipt = finalResult?.finalOutput === INTERCOM_DETACH_RECEIPT;
1680
1780
  const effectiveFinalError = stoppedAfterAcceptance
1681
1781
  ? ctx.stopMessage ?? "Subagent stopped by user."
1682
1782
  : timedOutAfterAcceptance
1683
- ? ctx.timeoutMessage ?? "Subagent timed out."
1783
+ ? finalResult?.error ?? ctx.timeoutMessage ?? "Subagent timed out."
1684
1784
  : turnBudgetExceeded
1685
1785
  ? finalResult?.error ?? (turnBudget ? turnBudgetExceededMessage(turnBudget, turnBudget.turnCount) : "Subagent exceeded turn budget.")
1686
1786
  : acceptanceCanFailRun
1687
1787
  ? (finalResult?.error ? `${finalResult.error}\n${acceptanceFailure}` : acceptanceFailure)
1688
- : finalResult?.error;
1788
+ : finalResult?.error ?? (intercomDetachReceipt ? INTERCOM_DETACH_RECEIPT : undefined);
1689
1789
 
1690
1790
  const artifactErrors = artifactPaths && ctx.artifactConfig?.enabled !== false
1691
1791
  ? persistStepArtifacts({
@@ -1700,7 +1800,7 @@ async function runSingleStep(
1700
1800
  metadata: {
1701
1801
  runId: ctx.id,
1702
1802
  agent: step.agent,
1703
- task,
1803
+ task: PROMPT_REDACTED,
1704
1804
  exitCode: effectiveFinalExitCode,
1705
1805
  model: finalResult?.model,
1706
1806
  attemptedModels: attemptedModels.length > 0 ? attemptedModels : undefined,
@@ -1765,6 +1865,27 @@ async function runSingleStep(
1765
1865
  return isAgentContractV1(step.agentContract) ? attachContractProjections(result as unknown as import("../../shared/types.ts").SingleResult) as unknown as typeof result : result;
1766
1866
  }
1767
1867
 
1868
+ async function runSingleStep(
1869
+ step: SubagentStep,
1870
+ ctx: SingleStepContext,
1871
+ ): Promise<StepResult & { completionGuardTriggered?: boolean }> {
1872
+ if (step.importAsyncRoot) return runSingleStepInner(step, ctx);
1873
+ const orcaProgressTab = createOrcaProgressTab({
1874
+ cwd: step.cwd ?? ctx.cwd,
1875
+ runId: ctx.id,
1876
+ agent: step.agent,
1877
+ index: ctx.flatIndex,
1878
+ });
1879
+ try {
1880
+ const result = await runSingleStepInner(step, { ...ctx, orcaProgressTab });
1881
+ orcaProgressTab?.finish(result.stopped ? "stopped" : result.exitCode === 0 && !result.error ? "completed" : "failed", result.sessionFile);
1882
+ return result;
1883
+ } catch (error) {
1884
+ orcaProgressTab?.finish("failed");
1885
+ throw error;
1886
+ }
1887
+ }
1888
+
1768
1889
  type RunnerStatusStep = NonNullable<AsyncStatus["steps"]>[number] & {
1769
1890
  exitCode?: number | null;
1770
1891
  description?: string;
@@ -1989,6 +2110,7 @@ async function runSingleStepWithTimeout(
1989
2110
  return await runSingleStep(step, {
1990
2111
  ...ctx,
1991
2112
  registerTimeout,
2113
+ deadlineAt: Date.now() + timeoutMs,
1992
2114
  timeoutSignal: combinedAbortSignal([ctx.timeoutSignal, timeoutController.signal]),
1993
2115
  timeoutMessage,
1994
2116
  });
@@ -2171,6 +2293,7 @@ async function runSubagent(
2171
2293
  ...(config.launchContractDigest ? { launchContractDigest: config.launchContractDigest } : {}),
2172
2294
  ...(config.launchResolvedExtensions ? { launchResolvedExtensions: config.launchResolvedExtensions } : {}),
2173
2295
  ...(config.capabilityCeiling ? { capabilityCeiling: config.capabilityCeiling } : {}),
2296
+ ...(config.runFanoutBudget ? { runFanoutBudget: getRunFanoutBudgetSnapshot(config.runFanoutBudget) } : {}),
2174
2297
  ...(config.parentWorkflowRunId ? { parentWorkflowRunId: config.parentWorkflowRunId } : {}),
2175
2298
  ...(config.workflowKey ? { workflowKey: config.workflowKey } : {}),
2176
2299
  ...(config.runnerProcessInstanceId ? { processTerminal: { version: 1 as const, state: "pending" as const, runId: id, runnerProcessInstanceId: config.runnerProcessInstanceId } } : {}),
@@ -2182,7 +2305,7 @@ async function runSubagent(
2182
2305
 
2183
2306
  fs.mkdirSync(asyncDir, { recursive: true });
2184
2307
  writeAtomicJson(statusPath, statusPayload);
2185
- updateActiveRunIndex(asyncDir, statusPayload.state);
2308
+ updateActiveRunIndex(asyncDir, statusPayload.state, statusPayload.toolCallId);
2186
2309
  let pendingParallelUsageCost: CostSummary = { inputTokens: 0, outputTokens: 0, costUsd: 0 };
2187
2310
  const currentUsageTotals = (): CostSummary => {
2188
2311
  const cost = results.reduce<CostSummary>((sum, result) => ({
@@ -2255,15 +2378,72 @@ async function runSubagent(
2255
2378
  for (const node of graph.nodes) updateNode(node);
2256
2379
  statusPayload.workflowGraph = graph;
2257
2380
  };
2381
+ let finalResultCommitted = false;
2258
2382
  let lastIndexedActiveState = isActiveAsyncState(statusPayload.state);
2383
+ const statusResultState = (): AsyncStatus["state"] | undefined => {
2384
+ if (statusPayload.state === "running" || statusPayload.state === "queued") return undefined;
2385
+ if (statusPayload.state === "paused" && statusPayload.checkpoint?.status === "pending") return undefined;
2386
+ return statusPayload.state;
2387
+ };
2388
+ const statusResultSummary = (state: AsyncStatus["state"]): string => {
2389
+ if (statusPayload.error) return statusPayload.error;
2390
+ if (state === "paused") return "Paused after interrupt. Waiting for explicit next action.";
2391
+ if (state === "stopped") return stopMessage;
2392
+ if (state === "rejected") return statusPayload.checkpoint?.name ? `Checkpoint '${statusPayload.checkpoint.name}' rejected.` : "Subagent rejected.";
2393
+ return state === "complete" ? "Subagent completed." : "Subagent failed.";
2394
+ };
2395
+ const statusResultSuccess = (state: AsyncStatus["state"], step: RunnerStatusStep): boolean | undefined => {
2396
+ if (step.status === "complete" || step.status === "completed") return true;
2397
+ if (step.status === "failed" || step.status === "stopped" || step.status === "rejected") return false;
2398
+ if (state === "complete") return true;
2399
+ if (state === "failed" || state === "stopped" || state === "rejected") return false;
2400
+ return undefined;
2401
+ };
2402
+ const writeRecoverableStatusResult = (): void => {
2403
+ const state = statusResultState();
2404
+ if (!state || finalResultCommitted || !config.sessionId) return;
2405
+ const now = statusPayload.endedAt ?? statusPayload.lastUpdate ?? Date.now();
2406
+ const summary = statusResultSummary(state);
2407
+ writePendingAsyncResultFile(resultPath, omitUndefinedProperties({
2408
+ lifecycleArtifactVersion: SUBAGENT_LIFECYCLE_ARTIFACT_VERSION,
2409
+ id,
2410
+ runId: id,
2411
+ agent: statusPayload.steps.length === 1 ? statusPayload.steps[0]!.agent : statusPayload.mode === "parallel" ? `parallel:${statusPayload.steps.map((step) => step.agent).join("+")}` : `chain:${statusPayload.steps.map((step) => step.agent).join("->")}`,
2412
+ mode: statusPayload.mode,
2413
+ success: state === "complete",
2414
+ state,
2415
+ summary,
2416
+ error: state === "failed" || state === "stopped" || state === "rejected" ? summary : undefined,
2417
+ stopped: state === "stopped" ? true : undefined,
2418
+ checkpoint: statusPayload.checkpoint,
2419
+ results: statusPayload.steps.map((step) => omitUndefinedProperties({
2420
+ agent: step.agent,
2421
+ output: step.status === "complete" || step.status === "completed" ? "" : step.error ?? summary,
2422
+ error: step.error,
2423
+ success: statusResultSuccess(state, step),
2424
+ sessionFile: step.sessionFile,
2425
+ model: step.model,
2426
+ attemptedModels: step.attemptedModels,
2427
+ modelAttempts: step.modelAttempts,
2428
+ })),
2429
+ exitCode: state === "complete" || state === "paused" ? 0 : 1,
2430
+ timestamp: now,
2431
+ durationMs: Math.max(0, now - overallStartTime),
2432
+ asyncDir,
2433
+ cwd,
2434
+ sessionId: config.sessionId,
2435
+ sessionFile: statusPayload.sessionFile ?? latestSessionFile,
2436
+ }));
2437
+ };
2259
2438
  const writeStatusPayloadNow = (): void => {
2260
2439
  refreshWorkflowGraph();
2440
+ writeRecoverableStatusResult();
2261
2441
  writeAtomicJson(statusPath, statusPayload);
2262
2442
  const activeState = isActiveAsyncState(statusPayload.state);
2263
2443
  if (activeState !== lastIndexedActiveState) {
2264
- updateActiveRunIndex(asyncDir, statusPayload.state);
2265
- lastIndexedActiveState = activeState;
2444
+ updateActiveRunIndex(asyncDir, statusPayload.state, statusPayload.toolCallId);
2266
2445
  }
2446
+ lastIndexedActiveState = activeState;
2267
2447
  emitNestedSelfEvent(statusPayload.state === "running" || statusPayload.state === "queued" ? "subagent.nested.updated" : "subagent.nested.completed");
2268
2448
  };
2269
2449
  const statusWriteCoalescer = createFileCoalescer(writeStatusPayloadNow, 100);
@@ -2506,6 +2686,72 @@ async function runSubagent(
2506
2686
  const activeLongRunningSteps = new Set<number>();
2507
2687
  const mutatingFailureStates = initialStatusSteps.map(() => createMutatingFailureState());
2508
2688
  const pendingToolResults: Array<{ tool: string; path?: string; mutates: boolean; startedAt?: number } | undefined> = initialStatusSteps.map(() => undefined);
2689
+ type ActiveToolCall = { key: string; tool: string; args: string; startedAt: number; path?: string; blocksSupervisor: boolean };
2690
+ const activeToolCalls = initialStatusSteps.map(() => new Map<string, ActiveToolCall>());
2691
+ const activeToolKeysByName = initialStatusSteps.map(() => new Map<string, string[]>());
2692
+ const activeToolSequences = initialStatusSteps.map(() => 0);
2693
+ const latestActiveToolCall = (flatIndex: number): ActiveToolCall | undefined => [...(activeToolCalls[flatIndex]?.values() ?? [])].sort((left, right) => right.startedAt - left.startedAt)[0];
2694
+ const refreshStepCurrentTool = (flatIndex: number): void => {
2695
+ const step = statusPayload.steps[flatIndex];
2696
+ if (!step) return;
2697
+ const active = latestActiveToolCall(flatIndex);
2698
+ if (!active) {
2699
+ delete step.currentTool;
2700
+ delete step.currentToolArgs;
2701
+ delete step.currentToolStartedAt;
2702
+ delete step.currentPath;
2703
+ return;
2704
+ }
2705
+ step.currentTool = active.tool;
2706
+ step.currentToolArgs = active.args;
2707
+ step.currentToolStartedAt = active.startedAt;
2708
+ setOptionalProperty(step, "currentPath", active.path);
2709
+ };
2710
+ const recordActiveToolCall = (flatIndex: number, event: { toolCallId?: unknown; toolName: string }, input: { argsPreview: string; currentPath?: string; blocksSupervisor: boolean; now: number }): ActiveToolCall => {
2711
+ const sequence = (activeToolSequences[flatIndex] ?? 0) + 1;
2712
+ activeToolSequences[flatIndex] = sequence;
2713
+ const key = toolTimeoutCallKey(event, sequence);
2714
+ const active: ActiveToolCall = {
2715
+ key,
2716
+ tool: event.toolName,
2717
+ args: input.argsPreview,
2718
+ startedAt: input.now,
2719
+ blocksSupervisor: input.blocksSupervisor,
2720
+ ...(input.currentPath !== undefined ? { path: input.currentPath } : {}),
2721
+ };
2722
+ activeToolCalls[flatIndex]?.set(key, active);
2723
+ const keysByName = activeToolKeysByName[flatIndex];
2724
+ const keys = keysByName?.get(active.tool) ?? [];
2725
+ keys.push(key);
2726
+ keysByName?.set(active.tool, keys);
2727
+ refreshStepCurrentTool(flatIndex);
2728
+ return active;
2729
+ };
2730
+ const removeActiveToolCallKey = (flatIndex: number, key: string): ActiveToolCall | undefined => {
2731
+ const calls = activeToolCalls[flatIndex];
2732
+ const active = calls?.get(key);
2733
+ if (!active) return undefined;
2734
+ calls?.delete(key);
2735
+ const keysByName = activeToolKeysByName[flatIndex];
2736
+ const keys = keysByName?.get(active.tool)?.filter((candidate) => candidate !== key) ?? [];
2737
+ if (keys.length > 0) keysByName?.set(active.tool, keys);
2738
+ else keysByName?.delete(active.tool);
2739
+ return active;
2740
+ };
2741
+ const removeActiveToolCall = (flatIndex: number, event: { toolCallId?: unknown; toolName?: unknown }): ActiveToolCall | undefined => {
2742
+ const calls = activeToolCalls[flatIndex];
2743
+ const key = typeof event.toolCallId === "string" && event.toolCallId.length > 0
2744
+ ? `id:${event.toolCallId}`
2745
+ : typeof event.toolName === "string"
2746
+ ? activeToolKeysByName[flatIndex]?.get(event.toolName)?.[0]
2747
+ : calls?.size === 1
2748
+ ? [...calls.keys()][0]
2749
+ : undefined;
2750
+ return key ? removeActiveToolCallKey(flatIndex, key) : undefined;
2751
+ };
2752
+ const openToolAttentionTarget = (flatIndex: number, now: number): ActiveToolCall | undefined => [...(activeToolCalls[flatIndex]?.values() ?? [])]
2753
+ .filter((active) => shouldEmitOpenToolAttention({ config: controlConfig, currentTool: active.tool, currentToolStartedAt: active.startedAt, now }))
2754
+ .sort((left, right) => left.startedAt - right.startedAt)[0];
2509
2755
  const supervisorAttentionSteps = new Map<number, ActivityState | undefined>();
2510
2756
  const mutatingFailureWindowMs = 5 * 60_000;
2511
2757
  const appendControlEvent = (event: ReturnType<typeof buildControlEvent>) => {
@@ -2546,6 +2792,34 @@ async function runSubagent(
2546
2792
  currentActivityState = nextRunState;
2547
2793
  setOptionalProperty(statusPayload, "activityState", nextRunState);
2548
2794
  };
2795
+ const maybeEmitOpenToolAttention = (flatIndex: number, now: number): boolean => {
2796
+ const step = statusPayload.steps[flatIndex];
2797
+ if (!step || step.status !== "running" || step.activityState === "needs_attention") return false;
2798
+ const target = openToolAttentionTarget(flatIndex, now);
2799
+ if (!target) return false;
2800
+ const previous = step.activityState;
2801
+ step.activityState = "needs_attention";
2802
+ statusPayload.activityState = "needs_attention";
2803
+ const toolDurationMs = Math.max(0, now - target.startedAt);
2804
+ appendControlEvent(buildControlEvent(omitUndefinedProperties({
2805
+ type: "needs_attention",
2806
+ from: previous,
2807
+ to: "needs_attention",
2808
+ runId: id,
2809
+ agent: step.agent,
2810
+ index: flatIndex,
2811
+ ts: now,
2812
+ message: `${step.agent} has had tool '${target.tool}' open for ${Math.floor(toolDurationMs / 1000)}s`,
2813
+ reason: "tool_open_threshold",
2814
+ turns: step.turnCount,
2815
+ tokens: step.tokens?.total,
2816
+ toolCount: step.toolCount,
2817
+ currentTool: target.tool,
2818
+ currentToolDurationMs: toolDurationMs,
2819
+ currentPath: target.path,
2820
+ })));
2821
+ return true;
2822
+ };
2549
2823
  const maybeEmitActiveLongRunning = (flatIndex: number, now: number): boolean => {
2550
2824
  if (!controlConfig.enabled || activeLongRunningSteps.has(flatIndex)) return false;
2551
2825
  const step = statusPayload.steps[flatIndex];
@@ -2806,20 +3080,19 @@ async function runSubagent(
2806
3080
  if (event.type === "tool_execution_start" && event.toolName) {
2807
3081
  const mutates = isMutatingTool(event.toolName, event.args);
2808
3082
  const currentPath = resolveCurrentPath(event.toolName, event.args);
3083
+ const argsPreview = extractToolArgsPreview(event.args ?? {});
3084
+ const blocksSupervisor = isBlockingSupervisorTool(event.toolName, event.args);
2809
3085
  step.toolCount = (step.toolCount ?? 0) + 1;
2810
3086
  const configuredToolBudget = flatSteps[flatIndex]?.toolBudget;
2811
3087
  if (configuredToolBudget) {
2812
3088
  step.toolBudget = toolBudgetState(configuredToolBudget, step.toolCount);
2813
3089
  statusPayload.toolBudget = step.toolBudget;
2814
3090
  }
2815
- step.currentTool = event.toolName;
2816
- step.currentToolArgs = extractToolArgsPreview(event.args ?? {});
2817
- step.currentToolStartedAt = now;
2818
- setOptionalProperty(step, "currentPath", currentPath);
3091
+ recordActiveToolCall(flatIndex, { toolCallId: (event as { toolCallId?: unknown }).toolCallId, toolName: event.toolName }, { argsPreview, currentPath, blocksSupervisor, now });
2819
3092
  pendingToolResults[flatIndex] = omitUndefinedProperties({ tool: event.toolName, path: currentPath, mutates, startedAt: now });
2820
3093
  statusPayload.toolCount = (statusPayload.toolCount ?? 0) + 1;
2821
3094
  syncTopLevelCurrentTool();
2822
- if (controlConfig.enabled && isBlockingSupervisorTool(event.toolName, event.args) && step.activityState !== "needs_attention") {
3095
+ if (controlConfig.enabled && blocksSupervisor && step.activityState !== "needs_attention") {
2823
3096
  const previous = step.activityState;
2824
3097
  step.activityState = "needs_attention";
2825
3098
  supervisorAttentionSteps.set(flatIndex, previous);
@@ -2844,16 +3117,15 @@ async function runSubagent(
2844
3117
  })));
2845
3118
  }
2846
3119
  } else if (event.type === "tool_execution_end") {
2847
- if (step.currentTool) {
3120
+ const endedTool = removeActiveToolCall(flatIndex, event);
3121
+ if (endedTool) {
2848
3122
  step.recentTools ??= [];
2849
- step.recentTools.push({ tool: step.currentTool, args: step.currentToolArgs || "", endMs: now });
3123
+ step.recentTools.push({ tool: endedTool.tool, args: endedTool.args, endMs: now });
2850
3124
  }
3125
+ refreshStepCurrentTool(flatIndex);
2851
3126
  const supervisorPreviousActivity = supervisorAttentionSteps.get(flatIndex);
2852
- const clearedSupervisorAttention = supervisorAttentionSteps.delete(flatIndex);
2853
- delete step.currentTool;
2854
- delete step.currentToolArgs;
2855
- delete step.currentToolStartedAt;
2856
- delete step.currentPath;
3127
+ const stillBlockingSupervisor = [...(activeToolCalls[flatIndex]?.values() ?? [])].some((active) => active.blocksSupervisor);
3128
+ const clearedSupervisorAttention = endedTool?.blocksSupervisor && !stillBlockingSupervisor ? supervisorAttentionSteps.delete(flatIndex) : false;
2857
3129
  if (clearedSupervisorAttention && step.activityState === "needs_attention") {
2858
3130
  setOptionalProperty(step, "activityState", supervisorPreviousActivity);
2859
3131
  syncAggregateActivityState();
@@ -2973,6 +3245,8 @@ async function runSubagent(
2973
3245
  })));
2974
3246
  changed = true;
2975
3247
  }
3248
+ } else if (maybeEmitOpenToolAttention(index, now)) {
3249
+ changed = true;
2976
3250
  } else if (maybeEmitActiveLongRunning(index, now)) {
2977
3251
  changed = true;
2978
3252
  }
@@ -3035,7 +3309,6 @@ async function runSubagent(
3035
3309
  if (stopped || timedOut || interrupted || statusPayload.state !== "running") return;
3036
3310
  stopped = true;
3037
3311
  const now = Date.now();
3038
- statusPayload.state = "stopped";
3039
3312
  statusPayload.stopped = true;
3040
3313
  statusPayload.error = stopMessage;
3041
3314
  currentActivityState = undefined;
@@ -3068,7 +3341,6 @@ async function runSubagent(
3068
3341
  timedOut = true;
3069
3342
  const now = Date.now();
3070
3343
  const message = timeoutMessage ?? "Subagent timed out.";
3071
- statusPayload.state = "failed";
3072
3344
  statusPayload.timedOut = true;
3073
3345
  statusPayload.error = message;
3074
3346
  currentActivityState = undefined;
@@ -3258,6 +3530,9 @@ async function runSubagent(
3258
3530
  throw new DynamicFanoutError(`Dynamic chain step ${stepIndex + 1} materialized ${materialized.parallel.length} items that resolve output to the same path: ${step.parallel.outputPath}. Remove the explicit output path or use an inherited relative agent output so each item can be isolated.`);
3259
3531
  }
3260
3532
  if (materialized.collectedOnEmpty) await validateDynamicCollection(step.collect.outputSchema, materialized.collectedOnEmpty);
3533
+ if (!config.runFanoutBudget) throw new Error("Async runner is missing its run fan-out budget identity.");
3534
+ const runFanoutBudget = claimRunFanoutBatch(config.runFanoutBudget, materialized.parallel.map((_, itemIndex) => `chain[${stepIndex}].expand[${itemIndex}]`));
3535
+ statusPayload.runFanoutBudget = runFanoutBudget;
3261
3536
  } catch (error) {
3262
3537
  const now = Date.now();
3263
3538
  const message = error instanceof DynamicFanoutError ? error.message : error instanceof Error ? error.message : String(error);
@@ -3364,6 +3639,7 @@ async function runSubagent(
3364
3639
  const materializedTask = step.parallel.namespaceOutputPath ? injectSingleOutputInstruction(taskText, outputPath, step.parallel) : taskText;
3365
3640
  return omitUndefinedProperties({
3366
3641
  ...step.parallel,
3642
+ runFanoutPath: `chain[${stepIndex}].expand[${itemIndex}]`,
3367
3643
  task: materializedTask,
3368
3644
  effectiveAcceptance: resolveEffectiveAcceptance(omitUndefinedProperties({
3369
3645
  explicit: step.parallel.acceptanceInput,
@@ -3523,6 +3799,7 @@ async function runSubagent(
3523
3799
  orchestratorIntercomTarget: config.controlIntercomTarget,
3524
3800
  nestedRoute: config.nestedRoute,
3525
3801
  capabilityCeiling: config.capabilityCeiling,
3802
+ runFanoutBudget: config.runFanoutBudget,
3526
3803
  registerInterrupt: (interrupt) => registerStepInterrupt(fi, interrupt),
3527
3804
  registerTimeout: (interrupt) => registerStepTimeout(fi, interrupt),
3528
3805
  registerStop: (stop) => registerStepStop(fi, stop),
@@ -3532,6 +3809,7 @@ async function runSubagent(
3532
3809
  timeoutMessage,
3533
3810
  stopMessage,
3534
3811
  turnBudget: config.turnBudget,
3812
+ toolTimeoutMs: task.toolTimeoutMs ?? config.toolTimeoutMs,
3535
3813
  onAttemptStart: (attempt) => updateStepModel(fi, attempt.model, attempt.thinking),
3536
3814
  onChildEvent: (event) => updateStepFromChildEvent(fi, event),
3537
3815
  onWriterProcess,
@@ -3905,6 +4183,7 @@ async function runSubagent(
3905
4183
  orchestratorIntercomTarget: config.controlIntercomTarget,
3906
4184
  nestedRoute: config.nestedRoute,
3907
4185
  capabilityCeiling: config.capabilityCeiling,
4186
+ runFanoutBudget: config.runFanoutBudget,
3908
4187
  registerInterrupt: (interrupt) => registerStepInterrupt(fi, interrupt),
3909
4188
  registerTimeout: (interrupt) => registerStepTimeout(fi, interrupt),
3910
4189
  registerStop: (stop) => registerStepStop(fi, stop),
@@ -3914,6 +4193,7 @@ async function runSubagent(
3914
4193
  timeoutMessage,
3915
4194
  stopMessage,
3916
4195
  turnBudget: config.turnBudget,
4196
+ toolTimeoutMs: taskForRun.toolTimeoutMs ?? config.toolTimeoutMs,
3917
4197
  onAttemptStart: (attempt) => updateStepModel(fi, attempt.model, attempt.thinking),
3918
4198
  onChildEvent: (event) => updateStepFromChildEvent(fi, event),
3919
4199
  onWriterProcess,
@@ -4194,6 +4474,7 @@ async function runSubagent(
4194
4474
  orchestratorIntercomTarget: config.controlIntercomTarget,
4195
4475
  nestedRoute: config.nestedRoute,
4196
4476
  capabilityCeiling: config.capabilityCeiling,
4477
+ runFanoutBudget: config.runFanoutBudget,
4197
4478
  registerInterrupt: (interrupt) => registerStepInterrupt(flatIndex, interrupt),
4198
4479
  registerTimeout: (interrupt) => registerStepTimeout(flatIndex, interrupt),
4199
4480
  registerStop: (stop) => registerStepStop(flatIndex, stop),
@@ -4203,6 +4484,7 @@ async function runSubagent(
4203
4484
  timeoutMessage,
4204
4485
  stopMessage,
4205
4486
  turnBudget: config.turnBudget,
4487
+ toolTimeoutMs: seqStep.toolTimeoutMs ?? config.toolTimeoutMs,
4206
4488
  onAttemptStart: (attempt) => updateStepModel(flatIndex, attempt.model, attempt.thinking),
4207
4489
  onChildEvent: (event) => updateStepFromChildEvent(flatIndex, event),
4208
4490
  onWriterProcess,
@@ -4438,6 +4720,9 @@ async function runSubagent(
4438
4720
  clearTimeout(timeoutTimer);
4439
4721
  timeoutTimer = undefined;
4440
4722
  }
4723
+ if (!timedOut && !stopped && !interrupted && config.timeoutMs !== undefined && results.some((result) => result.timedOut === true && result.error === timeoutMessage)) {
4724
+ timedOut = true;
4725
+ }
4441
4726
  const signalTerminated = !stopped && !timedOut && !turnBudgetExceeded && !interrupted && results.some((result) => result.exitCode !== 0 && isUnexplainedProcessSignal(omitUndefinedProperties({
4442
4727
  processSignal: result.processSignal,
4443
4728
  interrupted: result.interrupted,
@@ -4497,9 +4782,8 @@ async function runSubagent(
4497
4782
  statusPayload.error = `Step failed: ${failedStep.agent}`;
4498
4783
  }
4499
4784
  }
4500
- writeStatusPayload();
4501
4785
  try {
4502
- writeAtomicJson(resultPath, {
4786
+ writeAsyncResultFile(resultPath, {
4503
4787
  lifecycleArtifactVersion: SUBAGENT_LIFECYCLE_ARTIFACT_VERSION,
4504
4788
  id,
4505
4789
  agent: agentName,
@@ -4594,9 +4878,15 @@ async function runSubagent(
4594
4878
  ...(taskIndex !== undefined && { taskIndex }),
4595
4879
  ...(totalTasks !== undefined && { totalTasks }),
4596
4880
  });
4881
+ finalResultCommitted = true;
4597
4882
  } catch (err) {
4598
- console.error(`Failed to write result file ${resultPath}:`, err);
4883
+ const message = `Failed to write result file ${resultPath}: ${err instanceof Error ? err.message : String(err)}`;
4884
+ console.error(message, err);
4885
+ statusPayload.state = "failed";
4886
+ statusPayload.error = message;
4887
+ statusPayload.lastUpdate = Date.now();
4599
4888
  }
4889
+ writeStatusPayload();
4600
4890
  appendJsonl(
4601
4891
  eventsPath,
4602
4892
  JSON.stringify({
@@ -4630,7 +4920,7 @@ async function runSubagent(
4630
4920
  shareError,
4631
4921
  }));
4632
4922
  if (config.runnerProcessInstanceId) {
4633
- const writers: Record<string, Array<{ processInstanceId: string; kind: "pi-writer"; attempt: number; closeObservedAt: number; exitCode: number | null; signal: string | null }>> = {};
4923
+ const writers: Record<string, PiWriterProcessInstanceExitV1[]> = {};
4634
4924
  const expectedWriters: Record<string, number> = {};
4635
4925
  for (const [index, result] of results.entries()) {
4636
4926
  writers[String(index)] = result.writerProcesses ?? [];