@stigmer/runner 3.1.3 → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/.build-fingerprint +1 -1
  2. package/dist/activities/execute-cursor/capture-flow.d.ts +26 -15
  3. package/dist/activities/execute-cursor/capture-flow.js +56 -18
  4. package/dist/activities/execute-cursor/capture-flow.js.map +1 -1
  5. package/dist/activities/execute-cursor/command-provenance.d.ts +11 -25
  6. package/dist/activities/execute-cursor/command-provenance.js +25 -115
  7. package/dist/activities/execute-cursor/command-provenance.js.map +1 -1
  8. package/dist/activities/execute-cursor/index.js +296 -477
  9. package/dist/activities/execute-cursor/index.js.map +1 -1
  10. package/dist/activities/execute-cursor/todo-tracker.d.ts +6 -1
  11. package/dist/activities/execute-cursor/todo-tracker.js +15 -43
  12. package/dist/activities/execute-cursor/todo-tracker.js.map +1 -1
  13. package/dist/activities/execute-cursor/turn-stream.d.ts +141 -0
  14. package/dist/activities/execute-cursor/turn-stream.js +249 -0
  15. package/dist/activities/execute-cursor/turn-stream.js.map +1 -0
  16. package/dist/activities/execute-deep-agent/command-provenance.d.ts +61 -0
  17. package/dist/activities/execute-deep-agent/command-provenance.js +72 -0
  18. package/dist/activities/execute-deep-agent/command-provenance.js.map +1 -0
  19. package/dist/activities/execute-deep-agent/index.js +60 -18
  20. package/dist/activities/execute-deep-agent/index.js.map +1 -1
  21. package/dist/activities/execute-deep-agent/status-builder.js +8 -1
  22. package/dist/activities/execute-deep-agent/status-builder.js.map +1 -1
  23. package/dist/activities/execute-deep-agent/v3-status-builder.js +12 -1
  24. package/dist/activities/execute-deep-agent/v3-status-builder.js.map +1 -1
  25. package/dist/otel.js +10 -0
  26. package/dist/otel.js.map +1 -1
  27. package/dist/shared/filereview/cas-progress.d.ts +63 -0
  28. package/dist/shared/filereview/cas-progress.js +128 -0
  29. package/dist/shared/filereview/cas-progress.js.map +1 -0
  30. package/dist/shared/filereview/cas-substrate.d.ts +29 -0
  31. package/dist/shared/filereview/cas-substrate.js +46 -21
  32. package/dist/shared/filereview/cas-substrate.js.map +1 -1
  33. package/dist/shared/filereview/command-provenance.d.ts +93 -0
  34. package/dist/shared/filereview/command-provenance.js +132 -0
  35. package/dist/shared/filereview/command-provenance.js.map +1 -0
  36. package/dist/shared/filereview/git-substrate.d.ts +9 -3
  37. package/dist/shared/filereview/git-substrate.js.map +1 -1
  38. package/dist/shared/filereview/index.d.ts +4 -3
  39. package/dist/shared/filereview/index.js +3 -2
  40. package/dist/shared/filereview/index.js.map +1 -1
  41. package/dist/shared/filereview/progress.d.ts +105 -34
  42. package/dist/shared/filereview/progress.js +96 -34
  43. package/dist/shared/filereview/progress.js.map +1 -1
  44. package/dist/shared/plan-mode-prompt.d.ts +9 -0
  45. package/dist/shared/plan-mode-prompt.js +18 -0
  46. package/dist/shared/plan-mode-prompt.js.map +1 -1
  47. package/dist/shared/todos.d.ts +56 -0
  48. package/dist/shared/todos.js +98 -0
  49. package/dist/shared/todos.js.map +1 -0
  50. package/dist/shared/tool-row.d.ts +16 -0
  51. package/dist/shared/tool-row.js +31 -0
  52. package/dist/shared/tool-row.js.map +1 -1
  53. package/package.json +2 -2
  54. package/src/__tests__/otel-turn-span.test.ts +61 -0
  55. package/src/activities/execute-cursor/__tests__/progress-substrate.test.ts +169 -0
  56. package/src/activities/execute-cursor/__tests__/turn-stream.test.ts +349 -0
  57. package/src/activities/execute-cursor/capture-flow.ts +71 -25
  58. package/src/activities/execute-cursor/command-provenance.ts +25 -120
  59. package/src/activities/execute-cursor/index.ts +344 -504
  60. package/src/activities/execute-cursor/todo-tracker.ts +17 -59
  61. package/src/activities/execute-cursor/turn-stream.ts +418 -0
  62. package/src/activities/execute-deep-agent/__tests__/command-provenance.test.ts +252 -0
  63. package/src/activities/execute-deep-agent/__tests__/status-builder.test.ts +78 -0
  64. package/src/activities/execute-deep-agent/__tests__/v3-status-builder.test.ts +105 -1
  65. package/src/activities/execute-deep-agent/command-provenance.ts +102 -0
  66. package/src/activities/execute-deep-agent/index.ts +74 -18
  67. package/src/activities/execute-deep-agent/status-builder.ts +9 -0
  68. package/src/activities/execute-deep-agent/v3-status-builder.ts +13 -0
  69. package/src/otel.ts +8 -0
  70. package/src/shared/__tests__/todos.test.ts +216 -0
  71. package/src/shared/filereview/__tests__/cas-progress.test.ts +228 -0
  72. package/src/shared/filereview/__tests__/cas-substrate.test.ts +66 -0
  73. package/src/shared/filereview/__tests__/command-provenance.test.ts +252 -0
  74. package/src/shared/filereview/__tests__/progress.test.ts +112 -10
  75. package/src/shared/filereview/cas-progress.ts +170 -0
  76. package/src/shared/filereview/cas-substrate.ts +69 -24
  77. package/src/shared/filereview/command-provenance.ts +180 -0
  78. package/src/shared/filereview/git-substrate.ts +9 -3
  79. package/src/shared/filereview/index.ts +15 -1
  80. package/src/shared/filereview/progress.ts +171 -47
  81. package/src/shared/plan-mode-prompt.ts +18 -0
  82. package/src/shared/todos.ts +126 -0
  83. package/src/shared/tool-row.ts +34 -0
@@ -35,7 +35,7 @@ import type { SubAgentExecution } from "@stigmer/protos/ai/stigmer/agentic/agent
35
35
  import type { PendingApproval } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/approval_pb";
36
36
  import type { AgentExecution, AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
37
37
  import { ExecutionControlSignal, ExecutionPhase, FileChangeSetStatus, InteractionMode, MessageType, ApprovalAction } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
38
- import type { SDKMessage, Run, ConversationTurn } from "@cursor/sdk";
38
+ import type { Run, ConversationTurn } from "@cursor/sdk";
39
39
 
40
40
  import type { Config } from "../../config.js";
41
41
  import { StigmerClient } from "../../client/stigmer-client.js";
@@ -46,12 +46,11 @@ import { determineCursorMode, isCloudMode } from "./cursor-mode.js";
46
46
  import { MessageAccumulator, cancelInProgressSubAgentProtos, collapseRedundantToolCallTwins } from "./message-translator.js";
47
47
  import { utcTimestamp, persistStatus, reportSetupProgress, slimStatus } from "../../shared/status.js";
48
48
  import { withholdSecretContentFromMessages } from "../../shared/tool-row.js";
49
- import { startStallWatchdog, StallTimeoutError, formatStallFailure, type StallWatchdog } from "../../shared/stall-watchdog.js";
49
+ import { StallTimeoutError, formatStallFailure } from "../../shared/stall-watchdog.js";
50
50
  import { resolveUsableArtifactStorage, loadArtifactStorageConfig, type ArtifactStorage } from "../../shared/artifact-storage.js";
51
51
  import { publishPlanArtifact } from "../../shared/plan-artifact.js";
52
52
  import { DeltaEnricher } from "./delta-enricher.js";
53
53
  import { TodoTracker } from "./todo-tracker.js";
54
- import { shouldPersistStreamingStatus } from "./persist-decision.js";
55
54
  import { StreamingUpdateScheduler, loadStreamingConfig } from "../../shared/streaming-scheduler.js";
56
55
  import { createCursorEventRecorder } from "./cursor-event-recorder.js";
57
56
  import { resolveMcpServers, validateMcpServerEnv } from "./mcp-resolver.js";
@@ -74,19 +73,28 @@ import {
74
73
  type ReleaseWorkspaceLock,
75
74
  } from "../../shared/workspace/workspace-lock.js";
76
75
  import { LocalWorkspaceBackend } from "../../shared/workspace/local-backend.js";
77
- import { buildApprovalState, buildApprovalGrants, emitCursorGrantReceipts, readDenialLedger, reconstructAdjudicatedApprovals, watchDenialLedger } from "./approval-state.js";
76
+ import { buildApprovalState, buildApprovalGrants, emitCursorGrantReceipts, reconstructAdjudicatedApprovals, watchDenialLedger } from "./approval-state.js";
78
77
  import { applyApprovedWholeFileWrites, excludeAppliedFromGrants } from "./exact-apply.js";
79
78
  import { isGitWorkTree } from "../../shared/filereview/git-substrate.js";
80
79
  import {
81
80
  captureBaselineToLedger,
82
- captureProgressToStatus,
81
+ buildCursorProgressSubstrate,
83
82
  applyCaptureDecisions,
84
83
  deriveCaptureMode,
85
84
  } from "./capture-flow.js";
86
85
  import { runTurnBoundary, type TurnBoundaryResult } from "./turn-boundary.js";
87
86
  import {
87
+ consumeCursorTurnStream,
88
+ makeCursorTurnOnDelta,
89
+ newTurnStreamState,
90
+ type CursorTurnStreamDeps,
91
+ type TurnOnDeltaDeps,
92
+ } from "./turn-stream.js";
93
+ import {
94
+ captureFileChangeProgress,
88
95
  newProgressCaptureState,
89
96
  type ProgressCaptureState,
97
+ type ProgressSubstrate,
90
98
  } from "../../shared/filereview/progress.js";
91
99
  import { deriveExecutionFingerprintKey } from "../../shared/approval-fingerprint.js";
92
100
  import { getRunnerHitlMasterSecret } from "../../shared/fingerprint-secret.js";
@@ -200,40 +208,30 @@ async function executeCursorInner(
200
208
 
201
209
  let sessionId: string | undefined;
202
210
  let session: import("@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb").Session | undefined;
203
- let pauseDetected = false;
211
+ // Single owner for every flag the turn's stream produces (pause, stall,
212
+ // first-denial, platform-stop, event count, the stall watchdog, …). Created
213
+ // once here — before the fs denial-watcher, the SDK onDelta, the stall
214
+ // watchdog, and the stream loop are wired — so all four producers plus the
215
+ // epilogue and the outer catch/finally share ONE source of truth. The primary
216
+ // turn and both recovery retries drive the same stream code against this
217
+ // object (see turn-stream.ts for per-field ownership).
218
+ const turnState = newTurnStreamState();
219
+ // NOT part of turnState: derived post-loop from the periodic heartbeat +
220
+ // shutdown signal (a runner-manager shutdown, not a stream event), and read by
221
+ // the epilogue + outer catch. Kept as a plain let alongside the stream flags.
204
222
  let workerShutdownDetected = false;
205
- // Set by the stall watchdog when the SDK stream makes no progress for
206
- // config.cursorStreamStallTimeoutMs. stallError carries the recognizable
207
- // message surfaced to the user; both feed the Phase 11a stall branch.
208
- let stallDetected = false;
209
- let stallError: StallTimeoutError | undefined;
210
- // Set the moment the preToolUse hook records its first denial in the ledger.
211
- // We then stop consuming the stream and cancel the run so the model never
212
- // reacts to Cursor's tool-failure surface (narrate defeat, attempt a second
213
- // gated tool) — converging the Cursor harness toward the native harness, which
214
- // pauses BEFORE the model sees a denial. Phase 12 reconciles the denied tool
215
- // calls into WAITING_FOR_APPROVAL exactly as after a natural stream end.
216
- let firstDenialDetected = false;
217
- // Flipped by the denial-ledger fs watcher the instant the hook writes a
218
- // denial, so the NEXT stream event of ANY type triggers the ledger read —
219
- // instead of waiting for the next tool_call event, during which the model's
220
- // full post-denial reaction (thinking, narration, a workaround tool) would
221
- // stream and persist (observed in production: aex_01kwj07f7g23c3wp9sn8496z5g).
222
- // The tool_call-event read below remains the backstop where fs.watch is
223
- // unreliable.
224
- let denialLedgerDirty = false;
225
223
  let stopDenialWatcher: (() => void) | undefined;
226
- // The in-flight run.cancel() started by the first-denial stop. Awaited
227
- // (timeboxed) before Phase 12 so the agent process has actually stopped
228
- // before the final ledger read and the turn-boundary tree capture — closing
229
- // the race where a post-denial workaround's ledger entry lands after the
230
- // read (it would then never be collapsed) or a late tool mutates the tree
231
- // mid-capture.
232
- let denialCancelSettled: Promise<void> | undefined;
233
224
  let periodicHeartbeat: ReturnType<typeof startHeartbeat> | undefined;
234
- // Progress-based stall watchdog (see ../../shared/stall-watchdog.ts). Stopped
235
- // in the finally on every exit path; complements the liveness heartbeat.
236
- let stallWatchdog: StallWatchdog | undefined;
225
+ // Ends the OTel turn span + records turn metrics with the FINAL token snapshot.
226
+ // Hoisted and invoked from the finally so the span is closed exactly once on
227
+ // EVERY exit path — a happy completion, an approval pause, an early return, a
228
+ // throw, or a recovery retry (whose tokens accrue AFTER the primary stream
229
+ // ends). Ending it inline in the epilogue leaked the span on every non-happy
230
+ // path and excluded retry tokens/duration. Assigned when the span is created
231
+ // (once usageAccumulator exists); undefined — a no-op — before then (e.g. a
232
+ // pure-reconcile resume that returns before the agent runs) or when OTel is
233
+ // off. Idempotent: safe to call more than once.
234
+ let finishTurnTelemetry: (() => Promise<void>) | undefined;
237
235
  // Session HITL directory (runner-owned, outside the workspace) where the hook
238
236
  // script, approval-state file, and denial ledger live. Set once the gate is
239
237
  // installed; the WAITING_FOR_APPROVAL path reads the denial ledger from here.
@@ -689,7 +687,22 @@ async function executeCursorInner(
689
687
  // reset may flip the flag once before the run starts; the loop's read then
690
688
  // sees an empty ledger and clears it — harmless by construction.
691
689
  stopDenialWatcher = watchDenialLedger(hitlDir, () => {
692
- denialLedgerDirty = true;
690
+ turnState.denialLedgerDirty = true;
691
+ });
692
+
693
+ // Mid-run live capture (DD-32 / DD-33): choose the progress substrate for this
694
+ // turn's workspace shape ONCE (git / non-git CAS / hybrid). It owns its own
695
+ // short-circuit cache across the loop's persists; the floor lives in
696
+ // progressState. Undefined outside capture mode — writes are deny-gated and
697
+ // nothing is captured.
698
+ const progressSubstrate: ProgressSubstrate | undefined = buildCursorProgressSubstrate({
699
+ captureMode,
700
+ gitWorkspace,
701
+ workspaceRoot: primaryWorkspaceDir,
702
+ baselineTree,
703
+ executionId,
704
+ hitlDir,
705
+ storage: artifactStorage,
693
706
  });
694
707
 
695
708
  // Phase 5d: Ensure model pricing registry is populated before validation
@@ -841,13 +854,42 @@ async function executeCursorInner(
841
854
  await ensurePricingLoaded();
842
855
  const usageAccumulator = new UsageAccumulator(validatedModel);
843
856
 
844
- // Phase 10c: Start OTel turn span (coarse-grained — wraps entire agent.send + stream)
857
+ // Phase 10c: Start OTel turn span. Coarse-grained — spans the whole turn
858
+ // (agent.send + stream + any recovery retry + the turn boundary), ended once
859
+ // from the finally via finishTurnTelemetry with the final token snapshot.
845
860
  const { startCursorTurnSpan } = await import("../../otel.js");
846
861
  const turnSpan = await startCursorTurnSpan({
847
862
  model: validatedModel,
848
863
  mode: agentMode,
849
864
  sessionId: sessionId ?? "",
850
865
  });
866
+ // Bind the telemetry-finish closure now that the span + usage accumulator
867
+ // exist. Reads usageAccumulator at CALL time (in the finally), so it captures
868
+ // tokens from any recovery retry that ran after the primary stream. Guarded
869
+ // so a second call (finally after an inline path already finished it) is a
870
+ // no-op. Metrics failures are swallowed — OTel is optional.
871
+ let turnTelemetryFinished = false;
872
+ finishTurnTelemetry = async () => {
873
+ if (turnTelemetryFinished) return;
874
+ turnTelemetryFinished = true;
875
+ const usage = usageAccumulator.snapshot();
876
+ turnSpan.setTokens(Number(usage.inputTokens), Number(usage.outputTokens));
877
+ turnSpan.end();
878
+ try {
879
+ const { recordTurnMetrics } = await import("../../otel.js");
880
+ const durationMs =
881
+ Date.now() - (status.startedAt ? new Date(status.startedAt).getTime() : Date.now());
882
+ await recordTurnMetrics({
883
+ durationMs,
884
+ inputTokens: Number(usage.inputTokens),
885
+ outputTokens: Number(usage.outputTokens),
886
+ model: validatedModel,
887
+ mode: agentMode,
888
+ });
889
+ } catch {
890
+ // Metrics not initialized — silently skip.
891
+ }
892
+ };
851
893
 
852
894
  // Phase 11: Send message and stream events
853
895
  status.phase = ExecutionPhase.EXECUTION_IN_PROGRESS;
@@ -856,13 +898,21 @@ async function executeCursorInner(
856
898
  const todoTracker = new TodoTracker(status.todos);
857
899
  const eventRecorder = createCursorEventRecorder(executionId);
858
900
 
859
- let platformStopSignaled = false;
860
- let firstTurnAttributionLogged = false;
861
- let streamErrorMessage: string | undefined;
901
+ // The two recovery retries (poisoned-handle / transport-timeout) below run at
902
+ // most once per turn; this guard is the latch.
862
903
  let alreadyRetriedWithFreshAgent = false;
863
- // Most recent tool name observed on the stream, used only to enrich the
864
- // stall message ("last tool: …") so a wedged turn names the likely culprit.
865
- let lastToolName: string | undefined;
904
+
905
+ // The shared onDelta only needs the usage/enricher/heartbeat/state subset,
906
+ // and it is wired at SEND time — before the accumulator exists — so it takes
907
+ // the narrow deps. The primary send and both retry sends reuse this object.
908
+ const onDeltaDeps: TurnOnDeltaDeps = {
909
+ usageAccumulator,
910
+ deltaEnricher,
911
+ heartbeat,
912
+ promptEstimatedTokens,
913
+ executionId,
914
+ state: turnState,
915
+ };
866
916
 
867
917
  // Periodic heartbeat keeps Temporal informed during silent SDK operations
868
918
  // (e.g. long tool calls, MCP requests, model thinking). Without this,
@@ -890,61 +940,11 @@ async function executeCursorInner(
890
940
  // (e.g. older SDK), the warning is harmless — ignore.
891
941
  }
892
942
 
943
+ // The stall watchdog is armed inside consumeCursorTurnStream (it needs the
944
+ // run to cancel), stored on turnState.stallWatchdog so this shared onDelta can
945
+ // reset it and the activity's finally can stop it as a backstop.
893
946
  const run = await resolution.agent.send(effectivePrompt, {
894
- onDelta: ({ update }) => {
895
- // Reset the stall timer on the delta channel too: a long model
896
- // generation emits token deltas but few discrete stream events, so
897
- // resetting only in the stream loop would false-positive a stall.
898
- stallWatchdog?.recordActivity();
899
- if (update.type === "turn-ended" && update.usage) {
900
- usageAccumulator.addTurn(update.usage);
901
-
902
- if (!firstTurnAttributionLogged) {
903
- firstTurnAttributionLogged = true;
904
- const sdkInputTokens = update.usage.inputTokens ?? 0;
905
- const cursorOverhead = Math.max(0, sdkInputTokens - promptEstimatedTokens);
906
- console.log(
907
- `ExecuteCursor context attribution (first turn): execution=${executionId}, ` +
908
- `sdkInputTokens=${sdkInputTokens}, stigmerPreamble=${promptEstimatedTokens}, ` +
909
- `cursorOverhead=${cursorOverhead} (estimated)`,
910
- );
911
- }
912
- }
913
- deltaEnricher.processDelta(update);
914
- try {
915
- heartbeat();
916
- } catch (hbErr) {
917
- if (hbErr instanceof CancelledFailure) {
918
- pauseDetected = true;
919
- return;
920
- }
921
- throw hbErr;
922
- }
923
- },
924
- });
925
-
926
- // Arm the stall watchdog now that the run exists. The periodic heartbeat
927
- // above proves the process is alive, not that the agent is progressing: if
928
- // the stream wedges (a tool call or model connection that never returns),
929
- // no event/delta arrives, Phase 12 below is never reached, and the
930
- // execution hangs at EXECUTION_IN_PROGRESS forever. On stall we end the run
931
- // cleanly via the SDK's run.cancel() (guarded by supports("cancel")), which
932
- // unblocks the for-await; the stallDetected branch in Phase 11a then
933
- // reports EXECUTION_FAILED with a recognizable, actionable message.
934
- stallWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
935
- stallDetected = true;
936
- stallError = new StallTimeoutError(idleMs, lastToolName ? `last tool: ${lastToolName}` : undefined);
937
- console.warn(
938
- `ExecuteCursor stall detected: execution=${executionId}, idleMs=${idleMs}, lastTool=${lastToolName ?? "none"}`,
939
- );
940
- if (run.supports?.("cancel")) {
941
- void run.cancel().catch((cancelErr) => {
942
- console.warn(
943
- `ExecuteCursor run.cancel() after stall failed (non-fatal): execution=${executionId}, ` +
944
- `error=${cancelErr instanceof Error ? cancelErr.message : cancelErr}`,
945
- );
946
- });
947
- }
947
+ onDelta: makeCursorTurnOnDelta(onDeltaDeps),
948
948
  });
949
949
 
950
950
  // Everything at an index >= this was produced by THIS turn's stream — the
@@ -962,305 +962,185 @@ async function executeCursorInner(
962
962
  // flush; high-frequency token deltas ride this scheduler's time cadence
963
963
  // (env-tunable via STREAMING_* — see loadStreamingConfig).
964
964
  const scheduler = new StreamingUpdateScheduler(loadStreamingConfig());
965
- let eventCount = 0;
966
-
967
- try {
968
- for await (const event of run.stream()) {
969
- if (pauseDetected || Context.current().cancellationSignal.aborted) {
970
- pauseDetected = true;
971
- break;
972
- }
973
- if (stallDetected) break;
974
-
975
- // Progress: reset the stall timer on every stream event.
976
- stallWatchdog.recordActivity();
977
- if (event.type === "tool_call" && typeof event.name === "string") {
978
- lastToolName = event.name;
979
- }
980
-
981
- eventRecorder?.record(event, eventCount);
982
-
983
- accumulator.processEvent(event);
984
- todoTracker.processEvent(event);
985
-
986
- if (event.type === "tool_call" && event.name === "task") {
987
- accumulator.trackSubAgentExecution(
988
- event as Extract<SDKMessage, { type: "tool_call" }>,
989
- );
990
- }
991
-
992
- // First-denial stop (HITL clean pause). In CAPTURE mode this fires only for
993
- // an IRREVERSIBLE tool the hook still gates (shell, MCP, or a gitignored
994
- // write/delete) — file edits flow freely and are captured at the turn
995
- // boundary, so they never enter the ledger. In the deny-gate FALLBACK
996
- // (non-git workspace) it fires for every gated file edit too. Either way:
997
- // the preToolUse hook appends to the denial ledger the instant it gates a
998
- // tool — before Cursor surfaces the failure to the model — and the fs
999
- // watcher flips denialLedgerDirty the moment that write lands. Confirming
1000
- // the flag with a read on the very next event (of ANY type — thinking
1001
- // deltas arrive within milliseconds) ends the turn before the model's
1002
- // reaction can persist: waiting for the next tool_call event let the full
1003
- // post-denial reaction (thinking, narration, a workaround shell) stream
1004
- // and persist live (production case aex_01kwj07f7g23c3wp9sn8496z5g). The
1005
- // tool_call-event read stays as the backstop for platforms where fs.watch
1006
- // is unreliable; the current event was already accumulated above, so the
1007
- // anchor's own row is always present for the Phase 12 gate overlay. This
1008
- // mirrors the native harness's pause-before-react semantics; Phase 12
1009
- // reconciles the denied calls and its trim remains the last-resort
1010
- // backstop for anything that persisted before the stop.
1011
- if (!firstDenialDetected && hitlDir && (denialLedgerDirty || event.type === "tool_call")) {
1012
- denialLedgerDirty = false;
1013
- const denials = await readDenialLedger(hitlDir);
1014
- if (denials.length > 0) {
1015
- firstDenialDetected = true;
1016
- console.log(
1017
- `ExecuteCursor first denial detected (${denials.length} ledger ` +
1018
- `entr${denials.length === 1 ? "y" : "ies"}); stopping turn to pause ` +
1019
- `cleanly for approval: execution=${executionId}`,
1020
- );
1021
- if (run.supports?.("cancel")) {
1022
- // Kept (not fire-and-forget): awaited timeboxed before Phase 12 so
1023
- // the ledger read and tree capture see a stopped agent.
1024
- denialCancelSettled = run.cancel().then(
1025
- () => {},
1026
- (cancelErr: unknown) => {
1027
- console.warn(
1028
- `ExecuteCursor run.cancel() after first denial failed (non-fatal): ` +
1029
- `execution=${executionId}, ` +
1030
- `error=${cancelErr instanceof Error ? cancelErr.message : cancelErr}`,
1031
- );
1032
- },
1033
- );
1034
- }
1035
- break;
1036
- }
1037
- }
1038
-
1039
- deltaEnricher.applyEnrichments(status.messages);
1040
- eventCount++;
1041
-
1042
- if (event.type === "status") {
1043
- console.log(
1044
- `ExecuteCursor stream status: execution=${executionId}, status=${JSON.stringify(event)}`,
1045
- );
1046
- const statusEvent = event as { status?: string; message?: string };
1047
- if (statusEvent.status === "ERROR" && statusEvent.message) {
1048
- streamErrorMessage = statusEvent.message;
1049
- }
1050
- }
1051
965
 
1052
- const shouldPersist = shouldPersistStreamingStatus(
1053
- {
1054
- deltaEnricherDirty: deltaEnricher.isDirty,
1055
- todosDirty: todoTracker.isDirty,
1056
- contentDirty: accumulator.isDirty,
1057
- },
1058
- scheduler,
1059
- eventCount,
1060
- );
1061
- if (usageAccumulator.hasTurns) {
1062
- status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
1063
- }
1064
- if (shouldPersist) {
1065
- // Sync sub-agent executions into status before every persist so the
1066
- // live UI reflects delegation (including the IN_PROGRESS state) while
1067
- // the parent is still running — matching the native harness, which
1068
- // calls syncSubAgentExecutions() on each persist. Without this, the
1069
- // accumulator tracked sub-agents in memory but they only reached the
1070
- // status (and the subscriber stream) after the loop ended.
1071
- status.subAgentExecutions = accumulator.subAgentExecutions;
1072
- // Mid-run live capture (DD-32): attach the "N files changed so far"
1073
- // snapshot onto status.file_change_progress, throttled internally by the
1074
- // floor + tree-sha short-circuit. Git capture mode only (a pinned baseline
1075
- // exists); shell + sub-agent + tool edits are all captured for free by the
1076
- // workspace-wide diff. Never authoritative — the turn-boundary candidate
1077
- // remains the reviewed diff.
1078
- if (captureMode && gitWorkspace && baselineTree && primaryWorkspaceDir) {
1079
- await captureProgressToStatus({
1080
- status,
1081
- gitRoot: primaryWorkspaceDir,
1082
- executionId,
1083
- changeSetId,
1084
- baselineTree,
1085
- state: progressState,
1086
- });
1087
- }
1088
- const signal = await persist(status);
1089
- deltaEnricher.markPersisted();
1090
- todoTracker.markPersisted();
1091
- accumulator.markPersisted();
1092
- scheduler.markUpdateSent(eventCount);
1093
- heartbeat();
1094
- if (signal === ExecutionControlSignal.STOP) {
1095
- platformStopSignaled = true;
1096
- console.warn(
1097
- `ExecuteCursor platform stop signal received: execution=${executionId}`,
1098
- );
1099
- }
1100
- }
966
+ // Full deps for the shared stream loop — the collaborators + the injected
967
+ // heartbeat/cancellation (so the loop is testable, mirroring the deep-agent
968
+ // streamExecution seam), all keyed off the single turnState. Consumed by the
969
+ // primary stream here and by both recovery retries below.
970
+ const streamDeps: CursorTurnStreamDeps = {
971
+ ...onDeltaDeps,
972
+ status,
973
+ accumulator,
974
+ todoTracker,
975
+ eventRecorder,
976
+ scheduler,
977
+ progressSubstrate,
978
+ progressState,
979
+ changeSetId,
980
+ hitlDir,
981
+ stallTimeoutMs: config.cursorStreamStallTimeoutMs,
982
+ persist,
983
+ isCancelled: () => Context.current().cancellationSignal.aborted,
984
+ };
1101
985
 
1102
- if (platformStopSignaled) {
1103
- console.log(`ExecuteCursor stopping stream due to platform stop signal: execution=${executionId}`);
1104
- break;
1105
- }
1106
- }
1107
- } catch (streamErr) {
1108
- // run.cancel() — from the stall watchdog or the first-denial stop — can
1109
- // make the stream iterator reject as it tears down; that is the expected
1110
- // teardown for both, so swallow it and fall through (stallDetected ->
1111
- // Phase 11a; firstDenialDetected -> Phase 12). Anything else is a genuine
1112
- // stream failure — rethrow it to the outer error handler.
1113
- if (!stallDetected && !firstDenialDetected) throw streamErr;
1114
- console.warn(
1115
- `ExecuteCursor stream ended via cancel: execution=${executionId}, ` +
1116
- `stall=${stallDetected}, firstDenial=${firstDenialDetected}`,
1117
- );
1118
- }
986
+ // Primary stream. consumeCursorTurnStream owns the per-event loop (transcript,
987
+ // todos, sub-agent tracking, live persist, DD-32/DD-33 mid-run progress, the
988
+ // first-denial early stop, and the stall watchdog) and reports why it ended;
989
+ // resolvePreBoundaryTerminal below maps that to a terminal outcome. The two
990
+ // recovery retries drive the identical loop, so they inherit every one of
991
+ // these behaviors instead of the old bare loop that dropped them.
992
+ await consumeCursorTurnStream(run, streamDeps);
1119
993
 
1120
994
  periodicHeartbeat.stop();
1121
- stallWatchdog.stop();
1122
- // Check both the heartbeat flag AND the shutdown signal directly.
1123
- // Race condition: the heartbeat timer may detect Temporal's CancelledFailure
1124
- // (from worker.shutdown()) before the AbortSignal microtask propagates,
1125
- // causing it to set `cancelled` instead of `workerShutdown`. The direct
1126
- // signal check catches this case.
995
+ // Worker-shutdown vs. user-pause disambiguation. Primary-only: the periodic
996
+ // heartbeat is stopped here, before any recovery retry runs, so a retry
997
+ // classifies a shutdown from the shutdown signal directly (in
998
+ // resolvePreBoundaryTerminal). The heartbeat timer may set `cancelled` before
999
+ // the AbortSignal microtask propagates; the direct signal check catches that.
1127
1000
  const isShutdown = periodicHeartbeat.workerShutdown || (shutdownSignal?.aborted ?? false);
1128
1001
  if (isShutdown) {
1129
- pauseDetected = false;
1002
+ turnState.pauseDetected = false;
1130
1003
  } else if (periodicHeartbeat.cancelled) {
1131
- pauseDetected = true;
1004
+ turnState.pauseDetected = true;
1132
1005
  }
1133
-
1134
1006
  workerShutdownDetected = isShutdown;
1135
1007
 
1136
- accumulator.finalize();
1137
- deltaEnricher.finalize(status.messages);
1138
- // A pause / cancel / worker shutdown aborts the Cursor SDK run, so any
1139
- // sub-agent the parent had delegated is no longer executing. Mark it
1140
- // CANCELLED rather than leaving a permanent IN_PROGRESS "zombie" in the
1141
- // final snapshot (parity with the native harness's cancelSubAgents()).
1142
- if (pauseDetected || workerShutdownDetected || stallDetected || Context.current().cancellationSignal.aborted) {
1143
- accumulator.cancelInProgressSubAgents();
1144
- }
1145
- status.subAgentExecutions = accumulator.subAgentExecutions;
1146
- await eventRecorder?.flush();
1147
- if (usageAccumulator.hasTurns) {
1148
- status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
1149
- }
1150
- console.log(
1151
- `ExecuteCursor stream ended: execution=${executionId}, events=${eventCount}, messages=${status.messages.length}, subAgents=${status.subAgentExecutions.length}`,
1152
- );
1153
-
1154
- // Persist immediately after finalize so the UI sees correct tool
1155
- // call statuses before run.wait() / structured output extraction.
1156
- // This is unconditional (not throttled) because finalize is a
1157
- // once-per-execution correctness boundary.
1158
- await persist(status);
1159
- heartbeat();
1160
-
1161
- // End OTel turn span with accumulated token usage
1162
- const usageSnapshot = usageAccumulator.snapshot();
1163
- turnSpan.setTokens(Number(usageSnapshot.inputTokens), Number(usageSnapshot.outputTokens));
1164
- turnSpan.end();
1165
-
1166
- // Record cursor turn metrics (duration, tokens)
1167
- try {
1168
- const { recordTurnMetrics } = await import("../../otel.js");
1169
- const turnDurationMs = Date.now() - (status.startedAt ? new Date(status.startedAt).getTime() : Date.now());
1170
- await recordTurnMetrics({
1171
- durationMs: turnDurationMs,
1172
- inputTokens: Number(usageSnapshot.inputTokens),
1173
- outputTokens: Number(usageSnapshot.outputTokens),
1174
- model: validatedModel,
1175
- mode: agentMode,
1176
- });
1177
- } catch {
1178
- // Metrics not initialized — silently skip.
1179
- }
1008
+ // Post-stream finalize, shared by the primary turn and both recovery retries:
1009
+ // finalize the transcript + streaming flags, mark any in-flight sub-agent
1010
+ // CANCELLED on an aborted turn, snapshot usage, flush the recorder, and
1011
+ // persist so the UI sees the settled rows. The unified loop applies delta
1012
+ // enrichments per-iteration, so unlike the old bare retry path — no
1013
+ // compensating applyEnrichments() is needed here.
1014
+ const finalizeStreamPhase = async () => {
1015
+ accumulator.finalize();
1016
+ deltaEnricher.finalize(status.messages);
1017
+ // A pause / cancel / worker shutdown aborts the Cursor SDK run, so any
1018
+ // sub-agent the parent had delegated is no longer executing. Mark it
1019
+ // CANCELLED rather than leaving a permanent IN_PROGRESS "zombie" in the
1020
+ // final snapshot (parity with the native harness's cancelSubAgents()).
1021
+ if (
1022
+ turnState.pauseDetected ||
1023
+ workerShutdownDetected ||
1024
+ turnState.stallDetected ||
1025
+ Context.current().cancellationSignal.aborted
1026
+ ) {
1027
+ accumulator.cancelInProgressSubAgents();
1028
+ }
1029
+ status.subAgentExecutions = accumulator.subAgentExecutions;
1030
+ await eventRecorder?.flush();
1031
+ if (usageAccumulator.hasTurns) {
1032
+ status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
1033
+ }
1034
+ console.log(
1035
+ `ExecuteCursor stream ended: execution=${executionId}, events=${turnState.eventCount}, messages=${status.messages.length}, subAgents=${status.subAgentExecutions.length}`,
1036
+ );
1037
+ // Persist immediately after finalize so the UI sees correct tool-call
1038
+ // statuses before the boundary / run.wait() / structured-output extraction.
1039
+ await persist(status);
1040
+ heartbeat();
1041
+ };
1180
1042
 
1043
+ // Pre-boundary terminal handling, shared by the primary turn and both retries
1044
+ // so a retry that stalls, pauses, is cancelled, or is platform-stopped is
1045
+ // mapped IDENTICALLY to the primary — the fix for the mid-retry pause that
1046
+ // used to surface as EXECUTION_FAILED. "proceed" (a normal completion or a
1047
+ // first denial) goes on to the turn boundary; a stall / platform-stop asks
1048
+ // the caller to RETURN a terminal status; a worker-shutdown / pause /
1049
+ // infra-cancel asks the caller to THROW CancelledFailure. The OTel turn span
1050
+ // + metrics are ended once from the finally (finishTurnTelemetry), so they
1051
+ // include any recovery retry and never leak on these exits.
1052
+ type PreBoundaryTerminal =
1053
+ | { kind: "proceed" }
1054
+ | { kind: "return" }
1055
+ | { kind: "throw"; message: string };
1056
+ const resolvePreBoundaryTerminal = async (): Promise<PreBoundaryTerminal> => {
1057
+ // Stall: the watchdog cancelled a turn that made no progress. RETURN (not
1058
+ // throw): re-running the identical prompt via Temporal retry would very
1059
+ // likely wedge again.
1060
+ if (turnState.stallDetected) {
1061
+ const err = turnState.stallError ?? new StallTimeoutError(config.cursorStreamStallTimeoutMs);
1062
+ status.phase = ExecutionPhase.EXECUTION_FAILED;
1063
+ status.error = formatStallFailure(err);
1064
+ status.completedAt = utcTimestamp();
1065
+ status.messages.push(create(AgentMessageSchema, {
1066
+ type: MessageType.MESSAGE_SYSTEM,
1067
+ content: `Execution failed: the agent made no progress for too long and was stopped (${err.message}). You can retry or resume.`,
1068
+ timestamp: utcTimestamp(),
1069
+ }));
1070
+ await persist(status);
1071
+ console.warn(`ExecuteCursor stalled: execution=${executionId}, events=${turnState.eventCount}, error=${status.error}`);
1072
+ return { kind: "return" };
1073
+ }
1181
1074
 
1182
- // Phase 11a: Handle stall, worker shutdown, pause, or infrastructure cancellation.
1075
+ // Worker shutdown: the runner-manager aborted the shutdown signal. NOT a
1076
+ // user pause. Checked via the shutdown signal directly so a retry (whose
1077
+ // periodic heartbeat is already stopped) still classifies it correctly.
1078
+ if (workerShutdownDetected || (shutdownSignal?.aborted ?? false)) {
1079
+ status.phase = ExecutionPhase.EXECUTION_FAILED;
1080
+ status.error = "Execution interrupted: runner worker was shut down. Retry or resume.";
1081
+ status.completedAt = utcTimestamp();
1082
+ status.messages.push(create(AgentMessageSchema, {
1083
+ type: MessageType.MESSAGE_SYSTEM,
1084
+ content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
1085
+ timestamp: utcTimestamp(),
1086
+ }));
1087
+ await persist(status);
1088
+ console.log(`ExecuteCursor interrupted (worker shutdown): execution=${executionId}, events=${turnState.eventCount}`);
1089
+ return { kind: "throw", message: "Activity cancelled (worker shutdown, not user pause)" };
1090
+ }
1183
1091
 
1184
- // Stall: the watchdog cancelled a turn that made no progress for longer
1185
- // than config.cursorStreamStallTimeoutMs (a wedged tool call or a dead
1186
- // model connection). The keep-alive heartbeat proves liveness, so Temporal
1187
- // never reaps this on its own — this branch is the only clean exit. We
1188
- // RETURN (not throw): re-running the identical prompt via Temporal retry
1189
- // would very likely wedge again. accumulator.finalize() above already
1190
- // cleared isStreaming, so the UI spinners stop.
1191
- if (stallDetected) {
1192
- const err = stallError ?? new StallTimeoutError(config.cursorStreamStallTimeoutMs);
1193
- status.phase = ExecutionPhase.EXECUTION_FAILED;
1194
- status.error = formatStallFailure(err);
1195
- status.completedAt = utcTimestamp();
1196
- status.messages.push(create(AgentMessageSchema, {
1197
- type: MessageType.MESSAGE_SYSTEM,
1198
- content: `Execution failed: the agent made no progress for too long and was stopped (${err.message}). You can retry or resume.`,
1199
- timestamp: utcTimestamp(),
1200
- }));
1201
- await persist(status);
1202
- console.warn(`ExecuteCursor stalled: execution=${executionId}, events=${eventCount}, error=${status.error}`);
1203
- return slimStatus(status);
1204
- }
1092
+ // pauseDetected is only true if a heartbeat() call threw CancelledFailure,
1093
+ // confirming the orchestrator explicitly requested a pause.
1094
+ if (turnState.pauseDetected) {
1095
+ status.phase = ExecutionPhase.EXECUTION_PAUSED;
1096
+ status.messages.push(create(AgentMessageSchema, {
1097
+ type: MessageType.MESSAGE_SYSTEM,
1098
+ content: "Execution paused by user. Use resume to continue.",
1099
+ timestamp: utcTimestamp(),
1100
+ }));
1101
+ await persist(status);
1102
+ console.log(`ExecuteCursor paused: execution=${executionId}, events=${turnState.eventCount}`);
1103
+ return { kind: "throw", message: "Activity paused by orchestrator" };
1104
+ }
1205
1105
 
1206
- // Worker shutdown: the runner-manager aborted the shutdown signal before
1207
- // calling worker.shutdown(). This is NOT a user-initiated pause — it's
1208
- // an infrastructure event (e.g., premature removal from UI race).
1209
- if (workerShutdownDetected) {
1210
- status.phase = ExecutionPhase.EXECUTION_FAILED;
1211
- status.error = "Execution interrupted: runner worker was shut down. Retry or resume.";
1212
- status.completedAt = utcTimestamp();
1213
- status.messages.push(create(AgentMessageSchema, {
1214
- type: MessageType.MESSAGE_SYSTEM,
1215
- content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
1216
- timestamp: utcTimestamp(),
1217
- }));
1218
- await persist(status); console.log(`ExecuteCursor interrupted (worker shutdown): execution=${executionId}, events=${eventCount}`);
1219
- throw new CancelledFailure("Activity cancelled (worker shutdown, not user pause)");
1220
- }
1106
+ // Cancellation without pauseDetected (e.g. heartbeat timeout): report as
1107
+ // failed rather than misleadingly labeling it a user pause.
1108
+ if (Context.current().cancellationSignal.aborted) {
1109
+ status.phase = ExecutionPhase.EXECUTION_FAILED;
1110
+ status.error = "Execution interrupted: agent was unresponsive (heartbeat timeout). Retry or resume.";
1111
+ status.completedAt = utcTimestamp();
1112
+ status.messages.push(create(AgentMessageSchema, {
1113
+ type: MessageType.MESSAGE_SYSTEM,
1114
+ content: "Execution interrupted: the agent was unresponsive for too long. You can retry or resume.",
1115
+ timestamp: utcTimestamp(),
1116
+ }));
1117
+ await persist(status);
1118
+ console.log(`ExecuteCursor interrupted (infrastructure cancel): execution=${executionId}, events=${turnState.eventCount}`);
1119
+ return { kind: "throw", message: "Activity cancelled (heartbeat timeout, not user pause)" };
1120
+ }
1221
1121
 
1222
- // pauseDetected is only true if a heartbeat() call threw CancelledFailure,
1223
- // confirming the orchestrator explicitly requested a pause.
1224
- if (pauseDetected) {
1225
- status.phase = ExecutionPhase.EXECUTION_PAUSED;
1226
- status.messages.push(create(AgentMessageSchema, {
1227
- type: MessageType.MESSAGE_SYSTEM,
1228
- content: "Execution paused by user. Use resume to continue.",
1229
- timestamp: utcTimestamp(),
1230
- }));
1231
- await persist(status); console.log(`ExecuteCursor paused: execution=${executionId}, events=${eventCount}`);
1232
- throw new CancelledFailure("Activity paused by orchestrator");
1233
- }
1122
+ // Platform stop signal: a clean COMPLETED early exit.
1123
+ if (turnState.platformStopSignaled) {
1124
+ status.phase = ExecutionPhase.EXECUTION_COMPLETED;
1125
+ status.completedAt = utcTimestamp();
1126
+ status.messages.push(create(AgentMessageSchema, {
1127
+ type: MessageType.MESSAGE_SYSTEM,
1128
+ content: "Execution stopped by the platform.",
1129
+ timestamp: utcTimestamp(),
1130
+ }));
1131
+ await persist(status);
1132
+ try { resolution.agent.close(); } catch { /* best effort */ }
1133
+ console.log(`ExecuteCursor completed (platform stop): execution=${executionId}`);
1134
+ return { kind: "return" };
1135
+ }
1234
1136
 
1235
- // If cancellation arrived without pauseDetected (e.g. heartbeat timeout
1236
- // that slipped past the periodic heartbeat, or worker shutdown), report
1237
- // as failed rather than misleadingly labeling it as user-paused.
1238
- if (Context.current().cancellationSignal.aborted) {
1239
- status.phase = ExecutionPhase.EXECUTION_FAILED;
1240
- status.error = "Execution interrupted: agent was unresponsive (heartbeat timeout). Retry or resume.";
1241
- status.completedAt = utcTimestamp();
1242
- status.messages.push(create(AgentMessageSchema, {
1243
- type: MessageType.MESSAGE_SYSTEM,
1244
- content: "Execution interrupted: the agent was unresponsive for too long. You can retry or resume.",
1245
- timestamp: utcTimestamp(),
1246
- }));
1247
- await persist(status); console.log(`ExecuteCursor interrupted (infrastructure cancel): execution=${executionId}, events=${eventCount}`);
1248
- throw new CancelledFailure("Activity cancelled (heartbeat timeout, not user pause)");
1249
- }
1137
+ return { kind: "proceed" };
1138
+ };
1250
1139
 
1251
- // Phase 11b: Handle platform stop signal early exit
1252
- if (platformStopSignaled) {
1253
- status.phase = ExecutionPhase.EXECUTION_COMPLETED;
1254
- status.completedAt = utcTimestamp();
1255
- status.messages.push(create(AgentMessageSchema, {
1256
- type: MessageType.MESSAGE_SYSTEM,
1257
- content: "Execution stopped by the platform.",
1258
- timestamp: utcTimestamp(),
1259
- }));
1260
- await persist(status); try { resolution.agent.close(); } catch { /* best effort */ }
1261
- console.log(`ExecuteCursor completed (platform stop): execution=${executionId}`);
1262
- return slimStatus(status);
1263
- }
1140
+ await finalizeStreamPhase();
1141
+ const primaryTerminal = await resolvePreBoundaryTerminal();
1142
+ if (primaryTerminal.kind === "return") return slimStatus(status);
1143
+ if (primaryTerminal.kind === "throw") throw new CancelledFailure(primaryTerminal.message);
1264
1144
 
1265
1145
  // Phase 12: The turn boundary — author this turn's change set to the
1266
1146
  // file_review ledger (CANDIDATE_CAPTURED) and overlay the hook's denials as
@@ -1301,37 +1181,24 @@ async function executeCursorInner(
1301
1181
  return slimStatus(status);
1302
1182
  };
1303
1183
 
1304
- // Stream epilogue + boundary re-entry for a recovery retry (poisoned-handle /
1305
- // transport-timeout). The retry re-runs the agent AFTER the primary epilogue
1306
- // and boundary already ran, so its stream state must be settled the same way:
1307
- // - flush the enricher's buffered deltas onto the rows (the primary loop
1308
- // applies them every iteration; the bare retry loop does not — without the
1309
- // flush, a completed tool call never receives its completedAt evidence and
1310
- // finalize's reconciliation sweep leaves it RUNNING forever);
1311
- // - finalize streaming state and sync sub-agents;
1312
- // - refresh streaming usage (the retry's turns accumulated in memory only)
1313
- // and re-stamp completedAt (stamped below BEFORE the retry ran);
1314
- // - re-enter the turn boundary so edits made by the retry reach the
1315
- // file_review ledger / approval gates — without this, a retry's file
1316
- // edits silently escape review (production case
1317
- // aex_01kws27q1e2esvkqjpvectttxf).
1318
- // Returns undefined for a cancelled retry: parity with the primary path,
1319
- // where cancellation exits before the boundary — there is no review to open.
1184
+ // Re-enter the turn boundary for a recovery retry: author the retry's net
1185
+ // change set to the file_review ledger and overlay any denials as gates —
1186
+ // without this a retry's file edits silently escape review (production case
1187
+ // aex_01kws27q1e2esvkqjpvectttxf). The stream finalize now runs through the
1188
+ // shared finalizeStreamPhase (in runRecoveryStream), so this is only the
1189
+ // boundary + completedAt. Returns undefined for a cancelled retry — there is
1190
+ // no review to open. Passes denialCancelSettled so a first denial that stopped
1191
+ // the RETRY waits for run.cancel() before the ledger read, exactly like the
1192
+ // primary path.
1320
1193
  const settleRetryTurn = async (
1321
1194
  retryResultStatus: string,
1322
1195
  ): Promise<TurnBoundaryResult | undefined> => {
1323
- accumulator.finalize();
1324
- deltaEnricher.applyEnrichments(status.messages);
1325
- deltaEnricher.finalize(status.messages);
1326
- status.subAgentExecutions = accumulator.subAgentExecutions;
1327
- if (usageAccumulator.hasTurns) {
1328
- status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
1329
- }
1330
- // Re-flush the (dev-only) event recorder: flush rewrites the full JSONL,
1331
- // so the recorded trace now includes the retry's events too.
1332
- await eventRecorder?.flush();
1333
1196
  const retryBoundary =
1334
- retryResultStatus === "cancelled" ? undefined : await runBoundary();
1197
+ retryResultStatus === "cancelled"
1198
+ ? undefined
1199
+ : await runBoundary(
1200
+ turnState.firstDenialDetected ? turnState.denialCancelSettled : undefined,
1201
+ );
1335
1202
  // Phase 13 stamped completedAt BEFORE the retry ran. A terminal outcome
1336
1203
  // re-stamps it to the true end; a review pause CLEARS it — the primary
1337
1204
  // pause path never stamps it (a waiting turn is not complete).
@@ -1339,9 +1206,53 @@ async function executeCursorInner(
1339
1206
  return retryBoundary;
1340
1207
  };
1341
1208
 
1209
+ // The shared recovery spine. A fresh agent runs the IDENTICAL stream loop,
1210
+ // finalize, and pre-boundary terminal handling as the primary turn, then — on
1211
+ // a normal completion or a first denial — waits and re-enters the boundary.
1212
+ // The two recovery call sites below differ only in how they build the fresh
1213
+ // agent/prompt and how they classify a retry ERROR; everything the primary
1214
+ // does mid-stream (live persist, DD-32/DD-33 mid-run progress, sub-agent
1215
+ // tracking, the first-denial stop, and correct pause/stall/platform-stop
1216
+ // mapping) they now inherit for free instead of the old bare loop.
1217
+ type RecoveryOutcome =
1218
+ | { proceeded: false; terminal: Exclude<PreBoundaryTerminal, { kind: "proceed" }> }
1219
+ | {
1220
+ proceeded: true;
1221
+ retryRun: Run;
1222
+ retryResult: Awaited<ReturnType<Run["wait"]>>;
1223
+ retryBoundary: TurnBoundaryResult | undefined;
1224
+ };
1225
+ const runRecoveryStream = async (
1226
+ freshAgent: AgentResolution["agent"],
1227
+ retryPrompt: string,
1228
+ ): Promise<RecoveryOutcome> => {
1229
+ // The fresh agent is now the live handle: point resolution at it so the
1230
+ // terminal close() (platform stop, or Phase 14 success) frees THIS agent's
1231
+ // executor lease rather than the disposed one it replaced. (Without this the
1232
+ // poisoned-handle path leaked the fresh agent — it closed the stale one.)
1233
+ resolution = { ...resolution, agent: freshAgent, agentId: freshAgent.agentId, isNew: true };
1234
+ turnState.streamErrorMessage = undefined;
1235
+ const retryRun = await freshAgent.send(retryPrompt, {
1236
+ onDelta: makeCursorTurnOnDelta(onDeltaDeps),
1237
+ });
1238
+ await consumeCursorTurnStream(retryRun, streamDeps);
1239
+ await finalizeStreamPhase();
1240
+ const terminal = await resolvePreBoundaryTerminal();
1241
+ if (terminal.kind !== "proceed") return { proceeded: false, terminal };
1242
+ const retryResult = await retryRun.wait();
1243
+ console.log(
1244
+ `ExecuteCursor retry run.wait(): execution=${executionId}, ` +
1245
+ `retryResult=${JSON.stringify(retryResult)}`,
1246
+ );
1247
+ const retryBoundary = await settleRetryTurn(retryResult.status);
1248
+ return { proceeded: true, retryRun, retryResult, retryBoundary };
1249
+ };
1250
+
1342
1251
  // The denial-settle wait applies only when a first denial stopped THIS run;
1343
- // the recovery retries have no early stop and pass no promise.
1344
- const boundary = await runBoundary(firstDenialDetected ? denialCancelSettled : undefined);
1252
+ // a normal completion passes no promise.
1253
+ const boundary = await runBoundary(
1254
+ turnState.firstDenialDetected ? turnState.denialCancelSettled : undefined,
1255
+ );
1345
1256
  if (boundary.waiting) {
1346
1257
  return enterApprovalPause(boundary);
1347
1258
  }
@@ -1382,7 +1293,7 @@ async function executeCursorInner(
1382
1293
 
1383
1294
  const classified = synthesizeError({
1384
1295
  sdkResultFields: sdkErrorStr,
1385
- streamErrorMessage,
1296
+ streamErrorMessage: turnState.streamErrorMessage,
1386
1297
  capturedRejection,
1387
1298
  conversationErrorText,
1388
1299
  isResumedHandle: resolution.reason === "resumed_successfully",
@@ -1448,53 +1359,13 @@ async function executeCursorInner(
1448
1359
  console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
1449
1360
  }
1450
1361
 
1451
- let retryWatchdog: StallWatchdog | undefined;
1452
- const retryRun = await freshAgent.send(freshPrompt, {
1453
- onDelta: ({ update }) => {
1454
- retryWatchdog?.recordActivity();
1455
- if (update.type === "turn-ended" && update.usage) {
1456
- usageAccumulator.addTurn(update.usage);
1457
- }
1458
- deltaEnricher.processDelta(update);
1459
- try { heartbeat(); } catch { /* swallow during retry */ }
1460
- },
1461
- });
1462
- // Mirror the primary stream's stall protection: a wedged retry must
1463
- // not hang the activity. On stall, cancel the run so the loop ends and
1464
- // retryRun.wait() resolves down the existing non-finished failure path.
1465
- retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
1466
- console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
1467
- if (retryRun.supports?.("cancel")) void retryRun.cancel().catch(() => { /* best effort */ });
1468
- });
1469
-
1470
- streamErrorMessage = undefined;
1471
-
1472
- try {
1473
- for await (const retryEvent of retryRun.stream()) {
1474
- if (Context.current().cancellationSignal.aborted) break;
1475
- retryWatchdog.recordActivity();
1476
- eventRecorder?.record(retryEvent, eventCount);
1477
- accumulator.processEvent(retryEvent);
1478
- eventCount++;
1479
- if (retryEvent.type === "status") {
1480
- const retryStatusEvent = retryEvent as { status?: string; message?: string };
1481
- if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
1482
- streamErrorMessage = retryStatusEvent.message;
1483
- }
1484
- }
1485
- heartbeat();
1486
- }
1487
- } finally {
1488
- retryWatchdog.stop();
1362
+ const outcome = await runRecoveryStream(freshAgent, freshPrompt);
1363
+ if (!outcome.proceeded) {
1364
+ if (outcome.terminal.kind === "return") return slimStatus(status);
1365
+ throw new CancelledFailure(outcome.terminal.message);
1489
1366
  }
1490
1367
 
1491
- const retryResult = await retryRun.wait();
1492
- console.log(
1493
- `ExecuteCursor retry run.wait(): execution=${executionId}, ` +
1494
- `retryResult=${JSON.stringify(retryResult)}`,
1495
- );
1496
-
1497
- const retryBoundary = await settleRetryTurn(retryResult.status);
1368
+ const { retryRun, retryResult, retryBoundary } = outcome;
1498
1369
  if (retryBoundary?.waiting) {
1499
1370
  // The retry's edits/denials armed the gate — pause for review. On a
1500
1371
  // retry error this supersedes the failure, exactly as on the primary
@@ -1526,7 +1397,7 @@ async function executeCursorInner(
1526
1397
 
1527
1398
  const retryClassified = synthesizeError({
1528
1399
  sdkResultFields: retryResult.result ? String(retryResult.result) : undefined,
1529
- streamErrorMessage,
1400
+ streamErrorMessage: turnState.streamErrorMessage,
1530
1401
  capturedRejection: retryRejection,
1531
1402
  conversationErrorText: retryConversationErrorText,
1532
1403
  isResumedHandle: false,
@@ -1571,61 +1442,24 @@ async function executeCursorInner(
1571
1442
  console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
1572
1443
  }
1573
1444
 
1574
- let retryWatchdog: StallWatchdog | undefined;
1575
- const retryRun = await freshAgent.send(effectivePrompt, {
1576
- onDelta: ({ update }) => {
1577
- retryWatchdog?.recordActivity();
1578
- if (update.type === "turn-ended" && update.usage) {
1579
- usageAccumulator.addTurn(update.usage);
1580
- }
1581
- deltaEnricher.processDelta(update);
1582
- try { heartbeat(); } catch { /* swallow during retry */ }
1583
- },
1584
- });
1585
- // Mirror the primary stream's stall protection: a wedged retry must
1586
- // not hang the activity. On stall, cancel the run so the loop ends and
1587
- // retryRun.wait() resolves down the existing non-finished failure path.
1588
- retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
1589
- console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
1590
- if (retryRun.supports?.("cancel")) void retryRun.cancel().catch(() => { /* best effort */ });
1591
- });
1592
-
1593
- streamErrorMessage = undefined;
1594
-
1595
- try {
1596
- for await (const retryEvent of retryRun.stream()) {
1597
- if (Context.current().cancellationSignal.aborted) break;
1598
- retryWatchdog.recordActivity();
1599
- eventRecorder?.record(retryEvent, eventCount);
1600
- accumulator.processEvent(retryEvent);
1601
- eventCount++;
1602
- if (retryEvent.type === "status") {
1603
- const retryStatusEvent = retryEvent as { status?: string; message?: string };
1604
- if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
1605
- streamErrorMessage = retryStatusEvent.message;
1606
- }
1607
- }
1608
- heartbeat();
1609
- }
1610
- } finally {
1611
- retryWatchdog.stop();
1445
+ const outcome = await runRecoveryStream(freshAgent, effectivePrompt);
1446
+ if (!outcome.proceeded) {
1447
+ if (outcome.terminal.kind === "return") return slimStatus(status);
1448
+ throw new CancelledFailure(outcome.terminal.message);
1612
1449
  }
1613
1450
 
1614
- const retryResult = await retryRun.wait();
1615
- const retryBoundary = await settleRetryTurn(retryResult.status);
1451
+ const { retryResult, retryBoundary } = outcome;
1616
1452
  if (retryBoundary?.waiting) {
1617
1453
  // The retry's edits/denials armed the gate — pause for review (see
1618
1454
  // the poisoned-handle branch above for the precedence rationale).
1619
1455
  console.log(
1620
1456
  `ExecuteCursor transport-timeout recovery paused for review: execution=${executionId}`,
1621
1457
  );
1622
- resolution = { ...resolution, agent: freshAgent, agentId: freshAgent.agentId, isNew: true };
1623
1458
  return enterApprovalPause(retryBoundary);
1624
1459
  }
1625
1460
 
1626
1461
  if (retryResult.status === "finished") {
1627
1462
  status.phase = ExecutionPhase.EXECUTION_COMPLETED;
1628
- resolution = { ...resolution, agent: freshAgent, agentId: freshAgent.agentId, isNew: true };
1629
1463
  break;
1630
1464
  }
1631
1465
 
@@ -1769,7 +1603,7 @@ async function executeCursorInner(
1769
1603
  content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
1770
1604
  timestamp: utcTimestamp(),
1771
1605
  }));
1772
- } else if (pauseDetected) {
1606
+ } else if (turnState.pauseDetected) {
1773
1607
  console.log(`ExecuteCursor cancelled (pause) for execution ${executionId}`);
1774
1608
  status.phase = ExecutionPhase.EXECUTION_PAUSED;
1775
1609
  status.messages.push(create(AgentMessageSchema, {
@@ -1798,7 +1632,7 @@ async function executeCursorInner(
1798
1632
  // treat the execution as paused rather than failed. The error was likely
1799
1633
  // caused by the cancellation (e.g. SDK stream teardown) and should not
1800
1634
  // overwrite the PAUSED state that the Pause RPC already set in the DB.
1801
- if (pauseDetected) {
1635
+ if (turnState.pauseDetected) {
1802
1636
  const errDetail = err instanceof Error ? err.message : String(err);
1803
1637
  console.log(
1804
1638
  `ExecuteCursor error during pause (treating as pause): execution=${executionId}, error=${errDetail}`,
@@ -1901,10 +1735,16 @@ async function executeCursorInner(
1901
1735
 
1902
1736
  return slimStatus(status);
1903
1737
  } finally {
1904
- // Disarm the stall watchdog on EVERY exit path (idempotent). The happy
1905
- // path stops it after the stream loop; this covers throws before that
1906
- // point so no orphaned timer survives the activity.
1907
- stallWatchdog?.stop();
1738
+ // End the OTel turn span + record metrics with the final token snapshot on
1739
+ // EVERY exit path (idempotent). Placed here so the span covers any recovery
1740
+ // retry (whose tokens accrue after the primary stream) and never leaks on an
1741
+ // early return or throw. A no-op when OTel is off or the span never opened.
1742
+ await finishTurnTelemetry?.();
1743
+
1744
+ // Disarm the stall watchdog on EVERY exit path (idempotent). consumeCursorTurnStream
1745
+ // stops the one it armed; this covers throws before that point so no orphaned
1746
+ // timer survives the activity.
1747
+ turnState.stallWatchdog?.stop();
1908
1748
 
1909
1749
  // Close the denial-ledger watcher on EVERY exit path (idempotent) so no
1910
1750
  // orphaned fs.watch handle survives the activity.