@stigmer/runner 3.1.2 → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/dist/.build-fingerprint +1 -1
  2. package/dist/activities/execute-cursor/attachment-resolver.d.ts +1 -1
  3. package/dist/activities/execute-cursor/attachment-resolver.js +1 -1
  4. package/dist/activities/execute-cursor/capture-flow.d.ts +29 -0
  5. package/dist/activities/execute-cursor/capture-flow.js +59 -0
  6. package/dist/activities/execute-cursor/capture-flow.js.map +1 -1
  7. package/dist/activities/execute-cursor/command-provenance.d.ts +11 -25
  8. package/dist/activities/execute-cursor/command-provenance.js +25 -115
  9. package/dist/activities/execute-cursor/command-provenance.js.map +1 -1
  10. package/dist/activities/execute-cursor/index.js +358 -557
  11. package/dist/activities/execute-cursor/index.js.map +1 -1
  12. package/dist/activities/execute-cursor/todo-tracker.d.ts +6 -1
  13. package/dist/activities/execute-cursor/todo-tracker.js +15 -43
  14. package/dist/activities/execute-cursor/todo-tracker.js.map +1 -1
  15. package/dist/activities/execute-cursor/turn-boundary.d.ts +108 -0
  16. package/dist/activities/execute-cursor/turn-boundary.js +163 -0
  17. package/dist/activities/execute-cursor/turn-boundary.js.map +1 -0
  18. package/dist/activities/execute-cursor/turn-stream.d.ts +141 -0
  19. package/dist/activities/execute-cursor/turn-stream.js +249 -0
  20. package/dist/activities/execute-cursor/turn-stream.js.map +1 -0
  21. package/dist/activities/execute-deep-agent/command-provenance.d.ts +61 -0
  22. package/dist/activities/execute-deep-agent/command-provenance.js +72 -0
  23. package/dist/activities/execute-deep-agent/command-provenance.js.map +1 -0
  24. package/dist/activities/execute-deep-agent/index.js +73 -4
  25. package/dist/activities/execute-deep-agent/index.js.map +1 -1
  26. package/dist/activities/execute-deep-agent/status-builder.js +8 -1
  27. package/dist/activities/execute-deep-agent/status-builder.js.map +1 -1
  28. package/dist/activities/execute-deep-agent/streaming-v3.js +6 -1
  29. package/dist/activities/execute-deep-agent/streaming-v3.js.map +1 -1
  30. package/dist/activities/execute-deep-agent/streaming.d.ts +7 -0
  31. package/dist/activities/execute-deep-agent/streaming.js +5 -1
  32. package/dist/activities/execute-deep-agent/streaming.js.map +1 -1
  33. package/dist/activities/execute-deep-agent/v3-status-builder.js +12 -1
  34. package/dist/activities/execute-deep-agent/v3-status-builder.js.map +1 -1
  35. package/dist/otel.js +10 -0
  36. package/dist/otel.js.map +1 -1
  37. package/dist/shared/filereview/capture.d.ts +18 -1
  38. package/dist/shared/filereview/capture.js +105 -28
  39. package/dist/shared/filereview/capture.js.map +1 -1
  40. package/dist/shared/filereview/cas-progress.d.ts +63 -0
  41. package/dist/shared/filereview/cas-progress.js +128 -0
  42. package/dist/shared/filereview/cas-progress.js.map +1 -0
  43. package/dist/shared/filereview/cas-substrate.d.ts +38 -0
  44. package/dist/shared/filereview/cas-substrate.js +48 -15
  45. package/dist/shared/filereview/cas-substrate.js.map +1 -1
  46. package/dist/shared/filereview/command-provenance.d.ts +93 -0
  47. package/dist/shared/filereview/command-provenance.js +132 -0
  48. package/dist/shared/filereview/command-provenance.js.map +1 -0
  49. package/dist/shared/filereview/events.d.ts +16 -0
  50. package/dist/shared/filereview/events.js +37 -0
  51. package/dist/shared/filereview/events.js.map +1 -1
  52. package/dist/shared/filereview/git-substrate.d.ts +48 -0
  53. package/dist/shared/filereview/git-substrate.js +78 -0
  54. package/dist/shared/filereview/git-substrate.js.map +1 -1
  55. package/dist/shared/filereview/index.d.ts +5 -2
  56. package/dist/shared/filereview/index.js +5 -2
  57. package/dist/shared/filereview/index.js.map +1 -1
  58. package/dist/shared/filereview/line-counts.d.ts +48 -0
  59. package/dist/shared/filereview/line-counts.js +72 -0
  60. package/dist/shared/filereview/line-counts.js.map +1 -0
  61. package/dist/shared/filereview/progress.d.ts +167 -0
  62. package/dist/shared/filereview/progress.js +196 -0
  63. package/dist/shared/filereview/progress.js.map +1 -0
  64. package/dist/shared/implement-plan-prompt.d.ts +6 -4
  65. package/dist/shared/implement-plan-prompt.js +38 -6
  66. package/dist/shared/implement-plan-prompt.js.map +1 -1
  67. package/dist/shared/plan-artifact.d.ts +59 -11
  68. package/dist/shared/plan-artifact.js +144 -16
  69. package/dist/shared/plan-artifact.js.map +1 -1
  70. package/dist/shared/plan-mode-prompt.d.ts +15 -4
  71. package/dist/shared/plan-mode-prompt.js +28 -5
  72. package/dist/shared/plan-mode-prompt.js.map +1 -1
  73. package/dist/shared/todos.d.ts +56 -0
  74. package/dist/shared/todos.js +98 -0
  75. package/dist/shared/todos.js.map +1 -0
  76. package/dist/shared/tool-row.d.ts +16 -0
  77. package/dist/shared/tool-row.js +31 -0
  78. package/dist/shared/tool-row.js.map +1 -1
  79. package/package.json +3 -2
  80. package/src/__tests__/otel-turn-span.test.ts +61 -0
  81. package/src/activities/execute-cursor/__tests__/build-prompt.test.ts +7 -0
  82. package/src/activities/execute-cursor/__tests__/delta-enricher.test.ts +68 -0
  83. package/src/activities/execute-cursor/__tests__/progress-substrate.test.ts +169 -0
  84. package/src/activities/execute-cursor/__tests__/turn-boundary.test.ts +280 -0
  85. package/src/activities/execute-cursor/__tests__/turn-stream.test.ts +349 -0
  86. package/src/activities/execute-cursor/attachment-resolver.ts +1 -1
  87. package/src/activities/execute-cursor/capture-flow.ts +78 -0
  88. package/src/activities/execute-cursor/command-provenance.ts +25 -120
  89. package/src/activities/execute-cursor/index.ts +419 -602
  90. package/src/activities/execute-cursor/todo-tracker.ts +17 -59
  91. package/src/activities/execute-cursor/turn-boundary.ts +274 -0
  92. package/src/activities/execute-cursor/turn-stream.ts +418 -0
  93. package/src/activities/execute-deep-agent/__tests__/command-provenance.test.ts +252 -0
  94. package/src/activities/execute-deep-agent/__tests__/prompt-builder.test.ts +11 -0
  95. package/src/activities/execute-deep-agent/__tests__/status-builder.test.ts +78 -0
  96. package/src/activities/execute-deep-agent/__tests__/v3-status-builder.test.ts +105 -1
  97. package/src/activities/execute-deep-agent/command-provenance.ts +102 -0
  98. package/src/activities/execute-deep-agent/index.ts +91 -4
  99. package/src/activities/execute-deep-agent/status-builder.ts +9 -0
  100. package/src/activities/execute-deep-agent/streaming-v3.ts +6 -1
  101. package/src/activities/execute-deep-agent/streaming.ts +12 -1
  102. package/src/activities/execute-deep-agent/v3-status-builder.ts +13 -0
  103. package/src/otel.ts +8 -0
  104. package/src/shared/__tests__/implement-plan-prompt.test.ts +40 -2
  105. package/src/shared/__tests__/plan-artifact.test.ts +142 -9
  106. package/src/shared/__tests__/todos.test.ts +216 -0
  107. package/src/shared/filereview/__tests__/capture.test.ts +182 -1
  108. package/src/shared/filereview/__tests__/cas-progress.test.ts +228 -0
  109. package/src/shared/filereview/__tests__/cas-substrate.test.ts +100 -0
  110. package/src/shared/filereview/__tests__/command-provenance.test.ts +252 -0
  111. package/src/shared/filereview/__tests__/events.test.ts +105 -0
  112. package/src/shared/filereview/__tests__/line-counts.test.ts +73 -0
  113. package/src/shared/filereview/__tests__/progress.test.ts +338 -0
  114. package/src/shared/filereview/capture.ts +134 -30
  115. package/src/shared/filereview/cas-progress.ts +170 -0
  116. package/src/shared/filereview/cas-substrate.ts +79 -18
  117. package/src/shared/filereview/command-provenance.ts +180 -0
  118. package/src/shared/filereview/events.ts +50 -0
  119. package/src/shared/filereview/git-substrate.ts +114 -0
  120. package/src/shared/filereview/index.ts +36 -1
  121. package/src/shared/filereview/line-counts.ts +83 -0
  122. package/src/shared/filereview/progress.ts +310 -0
  123. package/src/shared/implement-plan-prompt.ts +39 -8
  124. package/src/shared/plan-artifact.ts +152 -16
  125. package/src/shared/plan-mode-prompt.ts +28 -5
  126. package/src/shared/todos.ts +126 -0
  127. package/src/shared/tool-row.ts +34 -0
@@ -30,20 +30,19 @@ import { create, clone } from "@bufbuild/protobuf";
30
30
  import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
31
31
  import { AgentMessageSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
32
32
  import { SubAgentExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/subagent_pb";
33
- import { ExecutionControlSignal, ExecutionPhase, FileChangeSetStatus, InteractionMode, MessageType, ApprovalAction } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
33
+ import { ExecutionPhase, FileChangeSetStatus, InteractionMode, MessageType, ApprovalAction } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
34
34
  import { StigmerClient } from "../../client/stigmer-client.js";
35
35
  import { resolveAgent } from "./session-lifecycle.js";
36
36
  import { CursorMode } from "@stigmer/protos/ai/stigmer/agentic/session/v1/enum_pb";
37
37
  import { determineCursorMode, isCloudMode } from "./cursor-mode.js";
38
- import { MessageAccumulator, reconcileDeniedToolCalls, clearProvisionalPostDenialNarration, cancelInProgressSubAgentProtos, collapseRedundantToolCallTwins } from "./message-translator.js";
38
+ import { MessageAccumulator, cancelInProgressSubAgentProtos, collapseRedundantToolCallTwins } from "./message-translator.js";
39
39
  import { utcTimestamp, persistStatus, reportSetupProgress, slimStatus } from "../../shared/status.js";
40
- import { collectSubAgentToolCallIds, withholdSecretContentFromMessages } from "../../shared/tool-row.js";
41
- import { startStallWatchdog, StallTimeoutError, formatStallFailure } from "../../shared/stall-watchdog.js";
40
+ import { withholdSecretContentFromMessages } from "../../shared/tool-row.js";
41
+ import { StallTimeoutError, formatStallFailure } from "../../shared/stall-watchdog.js";
42
42
  import { resolveUsableArtifactStorage, loadArtifactStorageConfig } from "../../shared/artifact-storage.js";
43
43
  import { publishPlanArtifact } from "../../shared/plan-artifact.js";
44
44
  import { DeltaEnricher } from "./delta-enricher.js";
45
45
  import { TodoTracker } from "./todo-tracker.js";
46
- import { shouldPersistStreamingStatus } from "./persist-decision.js";
47
46
  import { StreamingUpdateScheduler, loadStreamingConfig } from "../../shared/streaming-scheduler.js";
48
47
  import { createCursorEventRecorder } from "./cursor-event-recorder.js";
49
48
  import { resolveMcpServers, validateMcpServerEnv } from "./mcp-resolver.js";
@@ -61,11 +60,13 @@ import { installHitlGate, removeHitlGate } from "./workspace-setup.js";
61
60
  import { ensureHitlDir } from "../../shared/workspace/platform-dir.js";
62
61
  import { acquireWorkspaceLock, WorkspaceLockCancelledError, WorkspaceLockTimeoutError, } from "../../shared/workspace/workspace-lock.js";
63
62
  import { LocalWorkspaceBackend } from "../../shared/workspace/local-backend.js";
64
- import { buildApprovalState, buildApprovalGrants, emitCursorGrantReceipts, primaryToken, readDenialLedger, reconstructAdjudicatedApprovals, watchDenialLedger } from "./approval-state.js";
65
- import { deriveTurnCommandProvenance } from "./command-provenance.js";
63
+ import { buildApprovalState, buildApprovalGrants, emitCursorGrantReceipts, reconstructAdjudicatedApprovals, watchDenialLedger } from "./approval-state.js";
66
64
  import { applyApprovedWholeFileWrites, excludeAppliedFromGrants } from "./exact-apply.js";
67
65
  import { isGitWorkTree } from "../../shared/filereview/git-substrate.js";
68
- import { captureBaselineToLedger, captureTurnToLedger, applyCaptureDecisions, deriveCaptureMode, } from "./capture-flow.js";
66
+ import { captureBaselineToLedger, buildCursorProgressSubstrate, applyCaptureDecisions, deriveCaptureMode, } from "./capture-flow.js";
67
+ import { runTurnBoundary } from "./turn-boundary.js";
68
+ import { consumeCursorTurnStream, makeCursorTurnOnDelta, newTurnStreamState, } from "./turn-stream.js";
69
+ import { newProgressCaptureState, } from "../../shared/filereview/progress.js";
69
70
  import { deriveExecutionFingerprintKey } from "../../shared/approval-fingerprint.js";
70
71
  import { getRunnerHitlMasterSecret } from "../../shared/fingerprint-secret.js";
71
72
  import { provisionCursorWorkspace } from "./workspace-provision.js";
@@ -82,11 +83,6 @@ import { createAgent, createCloudAgent } from "./session-lifecycle.js";
82
83
  import { setMaxListeners } from "node:events";
83
84
  import { startHeartbeat } from "../../shared/heartbeat.js";
84
85
  import { getShutdownSignalForQueue } from "../../runner-manager.js";
85
- // How long Phase 12 waits for the first-denial-stop's run.cancel() to settle
86
- // before reading the final denial ledger and capturing the turn's tree. Long
87
- // enough for the SDK's normal teardown, short enough that a wedged cancel
88
- // cannot noticeably delay the approval pause the user is already waiting on.
89
- const FIRST_DENIAL_CANCEL_TIMEOUT_MS = 5_000;
90
86
  /**
91
87
  * Creates the activity functions bound to the runner config.
92
88
  * Returned object is passed to Temporal Worker.create({ activities }).
@@ -159,40 +155,30 @@ turnSeq) {
159
155
  };
160
156
  let sessionId;
161
157
  let session;
162
- let pauseDetected = false;
158
+ // Single owner for every flag the turn's stream produces (pause, stall,
159
+ // first-denial, platform-stop, event count, the stall watchdog, …). Created
160
+ // once here — before the fs denial-watcher, the SDK onDelta, the stall
161
+ // watchdog, and the stream loop are wired — so all four producers plus the
162
+ // epilogue and the outer catch/finally share ONE source of truth. The primary
163
+ // turn and both recovery retries drive the same stream code against this
164
+ // object (see turn-stream.ts for per-field ownership).
165
+ const turnState = newTurnStreamState();
166
+ // NOT part of turnState: derived post-loop from the periodic heartbeat +
167
+ // shutdown signal (a runner-manager shutdown, not a stream event), and read by
168
+ // the epilogue + outer catch. Kept as a plain let alongside the stream flags.
163
169
  let workerShutdownDetected = false;
164
- // Set by the stall watchdog when the SDK stream makes no progress for
165
- // config.cursorStreamStallTimeoutMs. stallError carries the recognizable
166
- // message surfaced to the user; both feed the Phase 11a stall branch.
167
- let stallDetected = false;
168
- let stallError;
169
- // Set the moment the preToolUse hook records its first denial in the ledger.
170
- // We then stop consuming the stream and cancel the run so the model never
171
- // reacts to Cursor's tool-failure surface (narrate defeat, attempt a second
172
- // gated tool) — converging the Cursor harness toward the native harness, which
173
- // pauses BEFORE the model sees a denial. Phase 12 reconciles the denied tool
174
- // calls into WAITING_FOR_APPROVAL exactly as after a natural stream end.
175
- let firstDenialDetected = false;
176
- // Flipped by the denial-ledger fs watcher the instant the hook writes a
177
- // denial, so the NEXT stream event of ANY type triggers the ledger read —
178
- // instead of waiting for the next tool_call event, during which the model's
179
- // full post-denial reaction (thinking, narration, a workaround tool) would
180
- // stream and persist (observed in production: aex_01kwj07f7g23c3wp9sn8496z5g).
181
- // The tool_call-event read below remains the backstop where fs.watch is
182
- // unreliable.
183
- let denialLedgerDirty = false;
184
170
  let stopDenialWatcher;
185
- // The in-flight run.cancel() started by the first-denial stop. Awaited
186
- // (timeboxed) before Phase 12 so the agent process has actually stopped
187
- // before the final ledger read and the turn-boundary tree capture — closing
188
- // the race where a post-denial workaround's ledger entry lands after the
189
- // read (it would then never be collapsed) or a late tool mutates the tree
190
- // mid-capture.
191
- let denialCancelSettled;
192
171
  let periodicHeartbeat;
193
- // Progress-based stall watchdog (see ../../shared/stall-watchdog.ts). Stopped
194
- // in the finally on every exit path; complements the liveness heartbeat.
195
- let stallWatchdog;
172
+ // Ends the OTel turn span + records turn metrics with the FINAL token snapshot.
173
+ // Hoisted and invoked from the finally so the span is closed exactly once on
174
+ // EVERY exit path — a happy completion, an approval pause, an early return, a
175
+ // throw, or a recovery retry (whose tokens accrue AFTER the primary stream
176
+ // ends). Ending it inline in the epilogue leaked the span on every non-happy
177
+ // path and excluded retry tokens/duration. Assigned when the span is created
178
+ // (once usageAccumulator exists); undefined — a no-op — before then (e.g. a
179
+ // pure-reconcile resume that returns before the agent runs) or when OTel is
180
+ // off. Idempotent: safe to call more than once.
181
+ let finishTurnTelemetry;
196
182
  // Session HITL directory (runner-owned, outside the workspace) where the hook
197
183
  // script, approval-state file, and denial ledger live. Set once the gate is
198
184
  // installed; the WAITING_FOR_APPROVAL path reads the denial ledger from here.
@@ -254,6 +240,9 @@ turnSeq) {
254
240
  // Pre-turn baseline tree, pinned before the agent runs (capture mode only)
255
241
  // so the turn-end capture diffs against it and the tree restores exactly.
256
242
  let baselineTree;
243
+ // Per-turn state for mid-run live capture (DD-32): the last progress tree sha
244
+ // (short-circuit) + last capture time (floor), threaded across persists.
245
+ const progressState = newProgressCaptureState();
257
246
  // Deterministic id of the change set this turn may produce:
258
247
  // `${executionId}:${turnSeq}`. Minted from the workflow-threaded turn index
259
248
  // so it is stable across a Temporal retry (idempotent ledger authoring) and
@@ -586,7 +575,21 @@ turnSeq) {
586
575
  // reset may flip the flag once before the run starts; the loop's read then
587
576
  // sees an empty ledger and clears it — harmless by construction.
588
577
  stopDenialWatcher = watchDenialLedger(hitlDir, () => {
589
- denialLedgerDirty = true;
578
+ turnState.denialLedgerDirty = true;
579
+ });
580
+ // Mid-run live capture (DD-32 / DD-33): choose the progress substrate for this
581
+ // turn's workspace shape ONCE (git / non-git CAS / hybrid). It owns its own
582
+ // short-circuit cache across the loop's persists; the floor lives in
583
+ // progressState. Undefined outside capture mode — writes are deny-gated and
584
+ // nothing is captured.
585
+ const progressSubstrate = buildCursorProgressSubstrate({
586
+ captureMode,
587
+ gitWorkspace,
588
+ workspaceRoot: primaryWorkspaceDir,
589
+ baselineTree,
590
+ executionId,
591
+ hitlDir,
592
+ storage: artifactStorage,
590
593
  });
591
594
  // Phase 5d: Ensure model pricing registry is populated before validation
592
595
  await ensurePricingLoaded();
@@ -702,25 +705,62 @@ turnSeq) {
702
705
  // Phase 10b: Initialize usage accumulator for runner-side token tracking
703
706
  await ensurePricingLoaded();
704
707
  const usageAccumulator = new UsageAccumulator(validatedModel);
705
- // Phase 10c: Start OTel turn span (coarse-grained — wraps entire agent.send + stream)
708
+ // Phase 10c: Start OTel turn span. Coarse-grained — spans the whole turn
709
+ // (agent.send + stream + any recovery retry + the turn boundary), ended once
710
+ // from the finally via finishTurnTelemetry with the final token snapshot.
706
711
  const { startCursorTurnSpan } = await import("../../otel.js");
707
712
  const turnSpan = await startCursorTurnSpan({
708
713
  model: validatedModel,
709
714
  mode: agentMode,
710
715
  sessionId: sessionId ?? "",
711
716
  });
717
+ // Bind the telemetry-finish closure now that the span + usage accumulator
718
+ // exist. Reads usageAccumulator at CALL time (in the finally), so it captures
719
+ // tokens from any recovery retry that ran after the primary stream. Guarded
720
+ // so a second call (finally after an inline path already finished it) is a
721
+ // no-op. Metrics failures are swallowed — OTel is optional.
722
+ let turnTelemetryFinished = false;
723
+ finishTurnTelemetry = async () => {
724
+ if (turnTelemetryFinished)
725
+ return;
726
+ turnTelemetryFinished = true;
727
+ const usage = usageAccumulator.snapshot();
728
+ turnSpan.setTokens(Number(usage.inputTokens), Number(usage.outputTokens));
729
+ turnSpan.end();
730
+ try {
731
+ const { recordTurnMetrics } = await import("../../otel.js");
732
+ const durationMs = Date.now() - (status.startedAt ? new Date(status.startedAt).getTime() : Date.now());
733
+ await recordTurnMetrics({
734
+ durationMs,
735
+ inputTokens: Number(usage.inputTokens),
736
+ outputTokens: Number(usage.outputTokens),
737
+ model: validatedModel,
738
+ mode: agentMode,
739
+ });
740
+ }
741
+ catch {
742
+ // Metrics not initialized — silently skip.
743
+ }
744
+ };
712
745
  // Phase 11: Send message and stream events
713
746
  status.phase = ExecutionPhase.EXECUTION_IN_PROGRESS;
714
747
  const deltaEnricher = new DeltaEnricher();
715
748
  const todoTracker = new TodoTracker(status.todos);
716
749
  const eventRecorder = createCursorEventRecorder(executionId);
717
- let platformStopSignaled = false;
718
- let firstTurnAttributionLogged = false;
719
- let streamErrorMessage;
750
+ // The two recovery retries (poisoned-handle / transport-timeout) below run at
751
+ // most once per turn; this guard is the latch.
720
752
  let alreadyRetriedWithFreshAgent = false;
721
- // Most recent tool name observed on the stream, used only to enrich the
722
- // stall message ("last tool: …") so a wedged turn names the likely culprit.
723
- let lastToolName;
753
+ // The shared onDelta only needs the usage/enricher/heartbeat/state subset,
754
+ // and it is wired at SEND time before the accumulator exists — so it takes
755
+ // the narrow deps. The primary send and both retry sends reuse this object.
756
+ const onDeltaDeps = {
757
+ usageAccumulator,
758
+ deltaEnricher,
759
+ heartbeat,
760
+ promptEstimatedTokens,
761
+ executionId,
762
+ state: turnState,
763
+ };
724
764
  // Periodic heartbeat keeps Temporal informed during silent SDK operations
725
765
  // (e.g. long tool calls, MCP requests, model thinking). Without this,
726
766
  // the 2-minute heartbeat timeout can cancel the activity and mislabel
@@ -746,54 +786,11 @@ turnSeq) {
746
786
  // Fallback: if the Temporal signal doesn't support setMaxListeners
747
787
  // (e.g. older SDK), the warning is harmless — ignore.
748
788
  }
789
+ // The stall watchdog is armed inside consumeCursorTurnStream (it needs the
790
+ // run to cancel), stored on turnState.stallWatchdog so this shared onDelta can
791
+ // reset it and the activity's finally can stop it as a backstop.
749
792
  const run = await resolution.agent.send(effectivePrompt, {
750
- onDelta: ({ update }) => {
751
- // Reset the stall timer on the delta channel too: a long model
752
- // generation emits token deltas but few discrete stream events, so
753
- // resetting only in the stream loop would false-positive a stall.
754
- stallWatchdog?.recordActivity();
755
- if (update.type === "turn-ended" && update.usage) {
756
- usageAccumulator.addTurn(update.usage);
757
- if (!firstTurnAttributionLogged) {
758
- firstTurnAttributionLogged = true;
759
- const sdkInputTokens = update.usage.inputTokens ?? 0;
760
- const cursorOverhead = Math.max(0, sdkInputTokens - promptEstimatedTokens);
761
- console.log(`ExecuteCursor context attribution (first turn): execution=${executionId}, ` +
762
- `sdkInputTokens=${sdkInputTokens}, stigmerPreamble=${promptEstimatedTokens}, ` +
763
- `cursorOverhead=${cursorOverhead} (estimated)`);
764
- }
765
- }
766
- deltaEnricher.processDelta(update);
767
- try {
768
- heartbeat();
769
- }
770
- catch (hbErr) {
771
- if (hbErr instanceof CancelledFailure) {
772
- pauseDetected = true;
773
- return;
774
- }
775
- throw hbErr;
776
- }
777
- },
778
- });
779
- // Arm the stall watchdog now that the run exists. The periodic heartbeat
780
- // above proves the process is alive, not that the agent is progressing: if
781
- // the stream wedges (a tool call or model connection that never returns),
782
- // no event/delta arrives, Phase 12 below is never reached, and the
783
- // execution hangs at EXECUTION_IN_PROGRESS forever. On stall we end the run
784
- // cleanly via the SDK's run.cancel() (guarded by supports("cancel")), which
785
- // unblocks the for-await; the stallDetected branch in Phase 11a then
786
- // reports EXECUTION_FAILED with a recognizable, actionable message.
787
- stallWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
788
- stallDetected = true;
789
- stallError = new StallTimeoutError(idleMs, lastToolName ? `last tool: ${lastToolName}` : undefined);
790
- console.warn(`ExecuteCursor stall detected: execution=${executionId}, idleMs=${idleMs}, lastTool=${lastToolName ?? "none"}`);
791
- if (run.supports?.("cancel")) {
792
- void run.cancel().catch((cancelErr) => {
793
- console.warn(`ExecuteCursor run.cancel() after stall failed (non-fatal): execution=${executionId}, ` +
794
- `error=${cancelErr instanceof Error ? cancelErr.message : cancelErr}`);
795
- });
796
- }
793
+ onDelta: makeCursorTurnOnDelta(onDeltaDeps),
797
794
  });
798
795
  // Everything at an index >= this was produced by THIS turn's stream — the
799
796
  // positional turn boundary the approved-command provenance (DD-28) scopes
@@ -809,385 +806,243 @@ turnSeq) {
809
806
  // flush; high-frequency token deltas ride this scheduler's time cadence
810
807
  // (env-tunable via STREAMING_* — see loadStreamingConfig).
811
808
  const scheduler = new StreamingUpdateScheduler(loadStreamingConfig());
812
- let eventCount = 0;
813
- try {
814
- for await (const event of run.stream()) {
815
- if (pauseDetected || Context.current().cancellationSignal.aborted) {
816
- pauseDetected = true;
817
- break;
818
- }
819
- if (stallDetected)
820
- break;
821
- // Progress: reset the stall timer on every stream event.
822
- stallWatchdog.recordActivity();
823
- if (event.type === "tool_call" && typeof event.name === "string") {
824
- lastToolName = event.name;
825
- }
826
- eventRecorder?.record(event, eventCount);
827
- accumulator.processEvent(event);
828
- todoTracker.processEvent(event);
829
- if (event.type === "tool_call" && event.name === "task") {
830
- accumulator.trackSubAgentExecution(event);
831
- }
832
- // First-denial stop (HITL clean pause). In CAPTURE mode this fires only for
833
- // an IRREVERSIBLE tool the hook still gates (shell, MCP, or a gitignored
834
- // write/delete) file edits flow freely and are captured at the turn
835
- // boundary, so they never enter the ledger. In the deny-gate FALLBACK
836
- // (non-git workspace) it fires for every gated file edit too. Either way:
837
- // the preToolUse hook appends to the denial ledger the instant it gates a
838
- // tool — before Cursor surfaces the failure to the model — and the fs
839
- // watcher flips denialLedgerDirty the moment that write lands. Confirming
840
- // the flag with a read on the very next event (of ANY type — thinking
841
- // deltas arrive within milliseconds) ends the turn before the model's
842
- // reaction can persist: waiting for the next tool_call event let the full
843
- // post-denial reaction (thinking, narration, a workaround shell) stream
844
- // and persist live (production case aex_01kwj07f7g23c3wp9sn8496z5g). The
845
- // tool_call-event read stays as the backstop for platforms where fs.watch
846
- // is unreliable; the current event was already accumulated above, so the
847
- // anchor's own row is always present for the Phase 12 gate overlay. This
848
- // mirrors the native harness's pause-before-react semantics; Phase 12
849
- // reconciles the denied calls and its trim remains the last-resort
850
- // backstop for anything that persisted before the stop.
851
- if (!firstDenialDetected && hitlDir && (denialLedgerDirty || event.type === "tool_call")) {
852
- denialLedgerDirty = false;
853
- const denials = await readDenialLedger(hitlDir);
854
- if (denials.length > 0) {
855
- firstDenialDetected = true;
856
- console.log(`ExecuteCursor first denial detected (${denials.length} ledger ` +
857
- `entr${denials.length === 1 ? "y" : "ies"}); stopping turn to pause ` +
858
- `cleanly for approval: execution=${executionId}`);
859
- if (run.supports?.("cancel")) {
860
- // Kept (not fire-and-forget): awaited timeboxed before Phase 12 so
861
- // the ledger read and tree capture see a stopped agent.
862
- denialCancelSettled = run.cancel().then(() => { }, (cancelErr) => {
863
- console.warn(`ExecuteCursor run.cancel() after first denial failed (non-fatal): ` +
864
- `execution=${executionId}, ` +
865
- `error=${cancelErr instanceof Error ? cancelErr.message : cancelErr}`);
866
- });
867
- }
868
- break;
869
- }
870
- }
871
- deltaEnricher.applyEnrichments(status.messages);
872
- eventCount++;
873
- if (event.type === "status") {
874
- console.log(`ExecuteCursor stream status: execution=${executionId}, status=${JSON.stringify(event)}`);
875
- const statusEvent = event;
876
- if (statusEvent.status === "ERROR" && statusEvent.message) {
877
- streamErrorMessage = statusEvent.message;
878
- }
879
- }
880
- const shouldPersist = shouldPersistStreamingStatus({
881
- deltaEnricherDirty: deltaEnricher.isDirty,
882
- todosDirty: todoTracker.isDirty,
883
- contentDirty: accumulator.isDirty,
884
- }, scheduler, eventCount);
885
- if (usageAccumulator.hasTurns) {
886
- status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
887
- }
888
- if (shouldPersist) {
889
- // Sync sub-agent executions into status before every persist so the
890
- // live UI reflects delegation (including the IN_PROGRESS state) while
891
- // the parent is still running — matching the native harness, which
892
- // calls syncSubAgentExecutions() on each persist. Without this, the
893
- // accumulator tracked sub-agents in memory but they only reached the
894
- // status (and the subscriber stream) after the loop ended.
895
- status.subAgentExecutions = accumulator.subAgentExecutions;
896
- const signal = await persist(status);
897
- deltaEnricher.markPersisted();
898
- todoTracker.markPersisted();
899
- accumulator.markPersisted();
900
- scheduler.markUpdateSent(eventCount);
901
- heartbeat();
902
- if (signal === ExecutionControlSignal.STOP) {
903
- platformStopSignaled = true;
904
- console.warn(`ExecuteCursor platform stop signal received: execution=${executionId}`);
905
- }
906
- }
907
- if (platformStopSignaled) {
908
- console.log(`ExecuteCursor stopping stream due to platform stop signal: execution=${executionId}`);
909
- break;
910
- }
911
- }
912
- }
913
- catch (streamErr) {
914
- // run.cancel() — from the stall watchdog or the first-denial stop — can
915
- // make the stream iterator reject as it tears down; that is the expected
916
- // teardown for both, so swallow it and fall through (stallDetected ->
917
- // Phase 11a; firstDenialDetected -> Phase 12). Anything else is a genuine
918
- // stream failure — rethrow it to the outer error handler.
919
- if (!stallDetected && !firstDenialDetected)
920
- throw streamErr;
921
- console.warn(`ExecuteCursor stream ended via cancel: execution=${executionId}, ` +
922
- `stall=${stallDetected}, firstDenial=${firstDenialDetected}`);
923
- }
809
+ // Full deps for the shared stream loop — the collaborators + the injected
810
+ // heartbeat/cancellation (so the loop is testable, mirroring the deep-agent
811
+ // streamExecution seam), all keyed off the single turnState. Consumed by the
812
+ // primary stream here and by both recovery retries below.
813
+ const streamDeps = {
814
+ ...onDeltaDeps,
815
+ status,
816
+ accumulator,
817
+ todoTracker,
818
+ eventRecorder,
819
+ scheduler,
820
+ progressSubstrate,
821
+ progressState,
822
+ changeSetId,
823
+ hitlDir,
824
+ stallTimeoutMs: config.cursorStreamStallTimeoutMs,
825
+ persist,
826
+ isCancelled: () => Context.current().cancellationSignal.aborted,
827
+ };
828
+ // Primary stream. consumeCursorTurnStream owns the per-event loop (transcript,
829
+ // todos, sub-agent tracking, live persist, DD-32/DD-33 mid-run progress, the
830
+ // first-denial early stop, and the stall watchdog) and reports why it ended;
831
+ // resolvePreBoundaryTerminal below maps that to a terminal outcome. The two
832
+ // recovery retries drive the identical loop, so they inherit every one of
833
+ // these behaviors instead of the old bare loop that dropped them.
834
+ await consumeCursorTurnStream(run, streamDeps);
924
835
  periodicHeartbeat.stop();
925
- stallWatchdog.stop();
926
- // Check both the heartbeat flag AND the shutdown signal directly.
927
- // Race condition: the heartbeat timer may detect Temporal's CancelledFailure
928
- // (from worker.shutdown()) before the AbortSignal microtask propagates,
929
- // causing it to set `cancelled` instead of `workerShutdown`. The direct
930
- // signal check catches this case.
836
+ // Worker-shutdown vs. user-pause disambiguation. Primary-only: the periodic
837
+ // heartbeat is stopped here, before any recovery retry runs, so a retry
838
+ // classifies a shutdown from the shutdown signal directly (in
839
+ // resolvePreBoundaryTerminal). The heartbeat timer may set `cancelled` before
840
+ // the AbortSignal microtask propagates; the direct signal check catches that.
931
841
  const isShutdown = periodicHeartbeat.workerShutdown || (shutdownSignal?.aborted ?? false);
932
842
  if (isShutdown) {
933
- pauseDetected = false;
843
+ turnState.pauseDetected = false;
934
844
  }
935
845
  else if (periodicHeartbeat.cancelled) {
936
- pauseDetected = true;
846
+ turnState.pauseDetected = true;
937
847
  }
938
848
  workerShutdownDetected = isShutdown;
939
- accumulator.finalize();
940
- deltaEnricher.finalize(status.messages);
941
- // A pause / cancel / worker shutdown aborts the Cursor SDK run, so any
942
- // sub-agent the parent had delegated is no longer executing. Mark it
943
- // CANCELLED rather than leaving a permanent IN_PROGRESS "zombie" in the
944
- // final snapshot (parity with the native harness's cancelSubAgents()).
945
- if (pauseDetected || workerShutdownDetected || stallDetected || Context.current().cancellationSignal.aborted) {
946
- accumulator.cancelInProgressSubAgents();
947
- }
948
- status.subAgentExecutions = accumulator.subAgentExecutions;
949
- await eventRecorder?.flush();
950
- if (usageAccumulator.hasTurns) {
951
- status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
952
- }
953
- console.log(`ExecuteCursor stream ended: execution=${executionId}, events=${eventCount}, messages=${status.messages.length}, subAgents=${status.subAgentExecutions.length}`);
954
- // Persist immediately after finalize so the UI sees correct tool
955
- // call statuses before run.wait() / structured output extraction.
956
- // This is unconditional (not throttled) because finalize is a
957
- // once-per-execution correctness boundary.
958
- await persist(status);
959
- heartbeat();
960
- // End OTel turn span with accumulated token usage
961
- const usageSnapshot = usageAccumulator.snapshot();
962
- turnSpan.setTokens(Number(usageSnapshot.inputTokens), Number(usageSnapshot.outputTokens));
963
- turnSpan.end();
964
- // Record cursor turn metrics (duration, tokens)
965
- try {
966
- const { recordTurnMetrics } = await import("../../otel.js");
967
- const turnDurationMs = Date.now() - (status.startedAt ? new Date(status.startedAt).getTime() : Date.now());
968
- await recordTurnMetrics({
969
- durationMs: turnDurationMs,
970
- inputTokens: Number(usageSnapshot.inputTokens),
971
- outputTokens: Number(usageSnapshot.outputTokens),
972
- model: validatedModel,
973
- mode: agentMode,
974
- });
975
- }
976
- catch {
977
- // Metrics not initialized — silently skip.
978
- }
979
- // Phase 11a: Handle stall, worker shutdown, pause, or infrastructure cancellation.
980
- // Stall: the watchdog cancelled a turn that made no progress for longer
981
- // than config.cursorStreamStallTimeoutMs (a wedged tool call or a dead
982
- // model connection). The keep-alive heartbeat proves liveness, so Temporal
983
- // never reaps this on its own — this branch is the only clean exit. We
984
- // RETURN (not throw): re-running the identical prompt via Temporal retry
985
- // would very likely wedge again. accumulator.finalize() above already
986
- // cleared isStreaming, so the UI spinners stop.
987
- if (stallDetected) {
988
- const err = stallError ?? new StallTimeoutError(config.cursorStreamStallTimeoutMs);
989
- status.phase = ExecutionPhase.EXECUTION_FAILED;
990
- status.error = formatStallFailure(err);
991
- status.completedAt = utcTimestamp();
992
- status.messages.push(create(AgentMessageSchema, {
993
- type: MessageType.MESSAGE_SYSTEM,
994
- content: `Execution failed: the agent made no progress for too long and was stopped (${err.message}). You can retry or resume.`,
995
- timestamp: utcTimestamp(),
996
- }));
997
- await persist(status);
998
- console.warn(`ExecuteCursor stalled: execution=${executionId}, events=${eventCount}, error=${status.error}`);
999
- return slimStatus(status);
1000
- }
1001
- // Worker shutdown: the runner-manager aborted the shutdown signal before
1002
- // calling worker.shutdown(). This is NOT a user-initiated pause — it's
1003
- // an infrastructure event (e.g., premature removal from UI race).
1004
- if (workerShutdownDetected) {
1005
- status.phase = ExecutionPhase.EXECUTION_FAILED;
1006
- status.error = "Execution interrupted: runner worker was shut down. Retry or resume.";
1007
- status.completedAt = utcTimestamp();
1008
- status.messages.push(create(AgentMessageSchema, {
1009
- type: MessageType.MESSAGE_SYSTEM,
1010
- content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
1011
- timestamp: utcTimestamp(),
1012
- }));
1013
- await persist(status);
1014
- console.log(`ExecuteCursor interrupted (worker shutdown): execution=${executionId}, events=${eventCount}`);
1015
- throw new CancelledFailure("Activity cancelled (worker shutdown, not user pause)");
1016
- }
1017
- // pauseDetected is only true if a heartbeat() call threw CancelledFailure,
1018
- // confirming the orchestrator explicitly requested a pause.
1019
- if (pauseDetected) {
1020
- status.phase = ExecutionPhase.EXECUTION_PAUSED;
1021
- status.messages.push(create(AgentMessageSchema, {
1022
- type: MessageType.MESSAGE_SYSTEM,
1023
- content: "Execution paused by user. Use resume to continue.",
1024
- timestamp: utcTimestamp(),
1025
- }));
1026
- await persist(status);
1027
- console.log(`ExecuteCursor paused: execution=${executionId}, events=${eventCount}`);
1028
- throw new CancelledFailure("Activity paused by orchestrator");
1029
- }
1030
- // If cancellation arrived without pauseDetected (e.g. heartbeat timeout
1031
- // that slipped past the periodic heartbeat, or worker shutdown), report
1032
- // as failed rather than misleadingly labeling it as user-paused.
1033
- if (Context.current().cancellationSignal.aborted) {
1034
- status.phase = ExecutionPhase.EXECUTION_FAILED;
1035
- status.error = "Execution interrupted: agent was unresponsive (heartbeat timeout). Retry or resume.";
1036
- status.completedAt = utcTimestamp();
1037
- status.messages.push(create(AgentMessageSchema, {
1038
- type: MessageType.MESSAGE_SYSTEM,
1039
- content: "Execution interrupted: the agent was unresponsive for too long. You can retry or resume.",
1040
- timestamp: utcTimestamp(),
1041
- }));
1042
- await persist(status);
1043
- console.log(`ExecuteCursor interrupted (infrastructure cancel): execution=${executionId}, events=${eventCount}`);
1044
- throw new CancelledFailure("Activity cancelled (heartbeat timeout, not user pause)");
1045
- }
1046
- // Phase 11b: Handle platform stop signal early exit
1047
- if (platformStopSignaled) {
1048
- status.phase = ExecutionPhase.EXECUTION_COMPLETED;
1049
- status.completedAt = utcTimestamp();
1050
- status.messages.push(create(AgentMessageSchema, {
1051
- type: MessageType.MESSAGE_SYSTEM,
1052
- content: "Execution stopped by the platform.",
1053
- timestamp: utcTimestamp(),
1054
- }));
849
+ // Post-stream finalize, shared by the primary turn and both recovery retries:
850
+ // finalize the transcript + streaming flags, mark any in-flight sub-agent
851
+ // CANCELLED on an aborted turn, snapshot usage, flush the recorder, and
852
+ // persist so the UI sees the settled rows. The unified loop applies delta
853
+ // enrichments per-iteration, so unlike the old bare retry path — no
854
+ // compensating applyEnrichments() is needed here.
855
+ const finalizeStreamPhase = async () => {
856
+ accumulator.finalize();
857
+ deltaEnricher.finalize(status.messages);
858
+ // A pause / cancel / worker shutdown aborts the Cursor SDK run, so any
859
+ // sub-agent the parent had delegated is no longer executing. Mark it
860
+ // CANCELLED rather than leaving a permanent IN_PROGRESS "zombie" in the
861
+ // final snapshot (parity with the native harness's cancelSubAgents()).
862
+ if (turnState.pauseDetected ||
863
+ workerShutdownDetected ||
864
+ turnState.stallDetected ||
865
+ Context.current().cancellationSignal.aborted) {
866
+ accumulator.cancelInProgressSubAgents();
867
+ }
868
+ status.subAgentExecutions = accumulator.subAgentExecutions;
869
+ await eventRecorder?.flush();
870
+ if (usageAccumulator.hasTurns) {
871
+ status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
872
+ }
873
+ console.log(`ExecuteCursor stream ended: execution=${executionId}, events=${turnState.eventCount}, messages=${status.messages.length}, subAgents=${status.subAgentExecutions.length}`);
874
+ // Persist immediately after finalize so the UI sees correct tool-call
875
+ // statuses before the boundary / run.wait() / structured-output extraction.
1055
876
  await persist(status);
1056
- try {
1057
- resolution.agent.close();
877
+ heartbeat();
878
+ };
879
+ const resolvePreBoundaryTerminal = async () => {
880
+ // Stall: the watchdog cancelled a turn that made no progress. RETURN (not
881
+ // throw): re-running the identical prompt via Temporal retry would very
882
+ // likely wedge again.
883
+ if (turnState.stallDetected) {
884
+ const err = turnState.stallError ?? new StallTimeoutError(config.cursorStreamStallTimeoutMs);
885
+ status.phase = ExecutionPhase.EXECUTION_FAILED;
886
+ status.error = formatStallFailure(err);
887
+ status.completedAt = utcTimestamp();
888
+ status.messages.push(create(AgentMessageSchema, {
889
+ type: MessageType.MESSAGE_SYSTEM,
890
+ content: `Execution failed: the agent made no progress for too long and was stopped (${err.message}). You can retry or resume.`,
891
+ timestamp: utcTimestamp(),
892
+ }));
893
+ await persist(status);
894
+ console.warn(`ExecuteCursor stalled: execution=${executionId}, events=${turnState.eventCount}, error=${status.error}`);
895
+ return { kind: "return" };
1058
896
  }
1059
- catch { /* best effort */ }
1060
- console.log(`ExecuteCursor completed (platform stop): execution=${executionId}`);
1061
- return slimStatus(status);
1062
- }
1063
- // Phase 12: Surface tools the preToolUse hook gated (HITL).
1064
- //
1065
- // The hook records each denial to the ledger; we mark the corresponding tool
1066
- // calls WAITING_APPROVAL. The backend projects pending_approvals from that
1067
- // tool-call status (PendingApprovalComputer), so — exactly like the native
1068
- // harness the approval surface is driven entirely by tool-call status. We
1069
- // deliberately do NOT set status.pendingApprovals here: any value would be
1070
- // discarded by the backend's recompute on the next updateStatus.
1071
- //
1072
- // Before reading the ledger, wait (timeboxed) for the first-denial-stop's
1073
- // run.cancel() to settle. run.cancel() races the SDK's auto-execution: until
1074
- // it lands, the agent process may still attempt a post-denial workaround
1075
- // whose hook denial would land AFTER a premature ledger read — the row then
1076
- // never collapses and renders as RUNNING forever (production case
1077
- // aex_01kwj07f7g23c3wp9sn8496z5g) — or a late tool could mutate the tree
1078
- // mid-capture. The timebox keeps a wedged cancel from hanging the pause;
1079
- // the Phase 12 trims below remain the backstop for that degraded case.
1080
- if (firstDenialDetected && denialCancelSettled) {
1081
- await Promise.race([
1082
- denialCancelSettled,
1083
- new Promise((resolve) => {
1084
- const timer = setTimeout(resolve, FIRST_DENIAL_CANCEL_TIMEOUT_MS);
1085
- timer.unref();
1086
- }),
1087
- ]);
1088
- }
1089
- const deniedLedger = await readDenialLedger(hitlDir ?? "");
1090
- // Capture mode: author the net change set to the file_review ledger as the
1091
- // CANDIDATE_CAPTURED event (projected server-side to a file_change_set
1092
- // AWAITING_REVIEW — the single review surface). The runner-owned gate files
1093
- // are excluded from the capture. The agent's edits are LEFT applied on the
1094
- // working tree (Cursor parity — the user reviews the real change; nothing is
1095
- // committed and the next turn is blocked until approval, and a reject snaps
1096
- // each file back on resume). Runs BEFORE the denial reconcile so a denied
1097
- // (gitignored) write stays on the deny-gate path while every flowed edit is
1098
- // captured to the ledger.
1099
- let capturedChangeCount = 0;
1100
- // `baselineTree !== undefined` means a baseline was authored this turn — the
1101
- // git tree sha for a git workspace, or "" (empty, but authored) for a non-git
1102
- // one. A plain truthiness check would wrongly skip the non-git capture.
1103
- if (captureMode && baselineTree !== undefined && primaryWorkspaceDir) {
1104
- const deniedTokens = new Set(deniedLedger.map((e) => e.token));
1105
- // Approved-command turn facts (DD-28): when every mutation-capable call
1106
- // this turn was a consented shell command, attach the provenance so the
1107
- // backend can verify the cited consent rows and auto-keep the set instead
1108
- // of arming a second gate. Fail-closed: any non-qualifying turn attaches
1109
- // nothing and reviews manually exactly as before.
1110
- const commandProvenance = deriveTurnCommandProvenance({
1111
- messages: status.messages,
1112
- turnStartIndex: turnStartMessageIndex,
1113
- deniedTokens,
1114
- grantTokenToConsentId: new Map((approvalGrants ?? []).map((g) => [
1115
- primaryToken(g.key, g.salient, g.contentDigest),
1116
- g.sourceToolCallId,
1117
- ])),
1118
- globalBypass,
1119
- });
1120
- if (commandProvenance) {
1121
- console.log(`ExecuteCursor capture: turn qualifies for approved-command auto-keep ` +
1122
- `(consent rows: ${commandProvenance.consentToolCallIds.join(",") || "(auto_approve_all)"}); ` +
1123
- `attaching provenance to candidate (execution=${executionId})`);
897
+ // Worker shutdown: the runner-manager aborted the shutdown signal. NOT a
898
+ // user pause. Checked via the shutdown signal directly so a retry (whose
899
+ // periodic heartbeat is already stopped) still classifies it correctly.
900
+ if (workerShutdownDetected || (shutdownSignal?.aborted ?? false)) {
901
+ status.phase = ExecutionPhase.EXECUTION_FAILED;
902
+ status.error = "Execution interrupted: runner worker was shut down. Retry or resume.";
903
+ status.completedAt = utcTimestamp();
904
+ status.messages.push(create(AgentMessageSchema, {
905
+ type: MessageType.MESSAGE_SYSTEM,
906
+ content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
907
+ timestamp: utcTimestamp(),
908
+ }));
909
+ await persist(status);
910
+ console.log(`ExecuteCursor interrupted (worker shutdown): execution=${executionId}, events=${turnState.eventCount}`);
911
+ return { kind: "throw", message: "Activity cancelled (worker shutdown, not user pause)" };
1124
912
  }
1125
- const captured = await captureTurnToLedger({
1126
- status,
1127
- gitRoot: primaryWorkspaceDir,
1128
- executionId,
1129
- changeSetId,
1130
- baselineTree,
1131
- messages: status.messages,
1132
- deniedTokens,
1133
- commandProvenance,
1134
- // Scope sub-agent row stamping to this turn: the seeded prior sub-agents
1135
- // (cloned in on resume) are the "before this turn" rows to skip.
1136
- priorSubAgentToolCallIds: collectSubAgentToolCallIds(seededSubAgents),
1137
- // The CAS half: read the sidecar the hook staged this turn and compose it
1138
- // into the change set. hitlDir + storage are present when captureIgnored
1139
- // was on (a git tree's gitignored writes, or ALL writes in a non-git
1140
- // workspace). In a git tree this composes with the git diff (HYBRID); in a
1141
- // non-git workspace it IS the whole change set (CAS-only).
1142
- hitlDir,
1143
- storage: artifactStorage,
1144
- gitWorkspace,
1145
- });
1146
- capturedChangeCount = captured.length;
1147
- if (capturedChangeCount > 0) {
1148
- console.log(`ExecuteCursor capture: ${capturedChangeCount} file change(s) authored to the ` +
1149
- `file_review ledger (change_set=${changeSetId}), working tree left applied ` +
1150
- `for review (execution=${executionId})`);
913
+ // pauseDetected is only true if a heartbeat() call threw CancelledFailure,
914
+ // confirming the orchestrator explicitly requested a pause.
915
+ if (turnState.pauseDetected) {
916
+ status.phase = ExecutionPhase.EXECUTION_PAUSED;
917
+ status.messages.push(create(AgentMessageSchema, {
918
+ type: MessageType.MESSAGE_SYSTEM,
919
+ content: "Execution paused by user. Use resume to continue.",
920
+ timestamp: utcTimestamp(),
921
+ }));
922
+ await persist(status);
923
+ console.log(`ExecuteCursor paused: execution=${executionId}, events=${turnState.eventCount}`);
924
+ return { kind: "throw", message: "Activity paused by orchestrator" };
1151
925
  }
1152
- }
1153
- // The gate reads each denied file's pre-edit `before` from the workspace the
1154
- // runner is co-located with (local FS for OSS; the sandbox in cloud), so a
1155
- // whole-file rewrite gate renders a true before/after diff. The tool was
1156
- // DENIED, so disk still holds the old content. User files are never platform
1157
- // paths, so no platformDir routing is needed here.
1158
- const gateWorkspaceBackend = new LocalWorkspaceBackend(primaryWorkspaceDir);
1159
- const deniedToolCalls = await reconcileDeniedToolCalls(status.messages, deniedLedger, mergedPolicies, gateWorkspaceBackend);
1160
- // Observability: a synthesized placeholder (id `approval:*`) means a denial
1161
- // correlated to NO streamed tool call in either the exact or the normalized
1162
- // pass. After the normalized-path fallback this should be ~0; a non-zero rate
1163
- // is the early-warning signal of a NEW identity drift (the gate would then
1164
- // show "No preview available" with no diff). Logged, not thrown — the
1165
- // synthesized gate still safely surfaces the approval.
1166
- const synthesizedGateCount = deniedToolCalls.filter((tc) => tc.id.startsWith("approval:")).length;
1167
- if (synthesizedGateCount > 0) {
1168
- console.warn(`ExecuteCursor reconcile synthesized ${synthesizedGateCount} placeholder gate(s) ` +
1169
- `with no correlated stream call (execution=${executionId}); ` +
1170
- `possible hook/stream identity drift — gate(s) will lack a diff`);
1171
- }
1172
- if (deniedToolCalls.length > 0 || capturedChangeCount > 0) {
1173
- if (deniedToolCalls.length > 0) {
1174
- // Deterministic clean-pause: a turn that pauses for approval must read as
1175
- // the same shape the native harness produces — pre-tool text + the gated
1176
- // tool calls — never the model's provisional reaction to Cursor's deny
1177
- // (e.g. "blocked by a hook; enable it in your Cursor settings"). We blank
1178
- // that reaction in place (keeping the message count, so the finalize stays
1179
- // append-only) rather than removing it. See
1180
- // clearProvisionalPostDenialNarration for the full rationale.
1181
- const redactedNarration = clearProvisionalPostDenialNarration(status.messages, deniedToolCalls);
1182
- if (redactedNarration.length > 0) {
1183
- console.log(`ExecuteCursor redacted ${redactedNarration.length} provisional post-denial narration message(s) before pausing for approval`);
926
+ // Cancellation without pauseDetected (e.g. heartbeat timeout): report as
927
+ // failed rather than misleadingly labeling it a user pause.
928
+ if (Context.current().cancellationSignal.aborted) {
929
+ status.phase = ExecutionPhase.EXECUTION_FAILED;
930
+ status.error = "Execution interrupted: agent was unresponsive (heartbeat timeout). Retry or resume.";
931
+ status.completedAt = utcTimestamp();
932
+ status.messages.push(create(AgentMessageSchema, {
933
+ type: MessageType.MESSAGE_SYSTEM,
934
+ content: "Execution interrupted: the agent was unresponsive for too long. You can retry or resume.",
935
+ timestamp: utcTimestamp(),
936
+ }));
937
+ await persist(status);
938
+ console.log(`ExecuteCursor interrupted (infrastructure cancel): execution=${executionId}, events=${turnState.eventCount}`);
939
+ return { kind: "throw", message: "Activity cancelled (heartbeat timeout, not user pause)" };
940
+ }
941
+ // Platform stop signal: a clean COMPLETED early exit.
942
+ if (turnState.platformStopSignaled) {
943
+ status.phase = ExecutionPhase.EXECUTION_COMPLETED;
944
+ status.completedAt = utcTimestamp();
945
+ status.messages.push(create(AgentMessageSchema, {
946
+ type: MessageType.MESSAGE_SYSTEM,
947
+ content: "Execution stopped by the platform.",
948
+ timestamp: utcTimestamp(),
949
+ }));
950
+ await persist(status);
951
+ try {
952
+ resolution.agent.close();
1184
953
  }
954
+ catch { /* best effort */ }
955
+ console.log(`ExecuteCursor completed (platform stop): execution=${executionId}`);
956
+ return { kind: "return" };
1185
957
  }
958
+ return { kind: "proceed" };
959
+ };
960
+ await finalizeStreamPhase();
961
+ const primaryTerminal = await resolvePreBoundaryTerminal();
962
+ if (primaryTerminal.kind === "return")
963
+ return slimStatus(status);
964
+ if (primaryTerminal.kind === "throw")
965
+ throw new CancelledFailure(primaryTerminal.message);
966
+ // Phase 12: The turn boundary — author this turn's change set to the
967
+ // file_review ledger (CANDIDATE_CAPTURED) and overlay the hook's denials as
968
+ // WAITING_APPROVAL gate rows. The full pipeline and its ordering rationale
969
+ // live in turn-boundary.ts; this closure binds the turn's state so the
970
+ // recovery retries below (which re-run the agent AFTER this primary call)
971
+ // can re-enter the IDENTICAL pipeline — a retry's edits must reach the
972
+ // ledger or they silently escape review. `baselineTree` is read at call
973
+ // time, so both entries see the baseline authored at turn start.
974
+ const runBoundary = (denialSettled) => runTurnBoundary({
975
+ status,
976
+ executionId,
977
+ changeSetId,
978
+ hitlDir,
979
+ captureMode,
980
+ baselineTree,
981
+ primaryWorkspaceDir,
982
+ gitWorkspace,
983
+ turnStartMessageIndex,
984
+ approvalGrants,
985
+ globalBypass,
986
+ seededSubAgents,
987
+ artifactStorage,
988
+ mergedPolicies,
989
+ denialCancelSettled: denialSettled,
990
+ });
991
+ // Pauses for review exactly like the native harness: the boundary mutated
992
+ // the transcript in place; we flip the phase, persist, and RETURN to the
993
+ // workflow, which waits for the approval/file-review signal and reinvokes.
994
+ const enterApprovalPause = async (boundary) => {
1186
995
  status.phase = ExecutionPhase.EXECUTION_WAITING_FOR_APPROVAL;
1187
996
  await persist(status);
1188
- console.log(`ExecuteCursor returning WAITING_FOR_APPROVAL: ${deniedToolCalls.length} gated tool(s), ` +
1189
- `${capturedChangeCount} file card(s) pending`);
997
+ console.log(`ExecuteCursor returning WAITING_FOR_APPROVAL: ${boundary.deniedToolCallCount} gated tool(s), ` +
998
+ `${boundary.capturedChangeCount} file card(s) pending`);
1190
999
  return slimStatus(status);
1000
+ };
1001
+ // Re-enter the turn boundary for a recovery retry: author the retry's net
1002
+ // change set to the file_review ledger and overlay any denials as gates —
1003
+ // without this a retry's file edits silently escape review (production case
1004
+ // aex_01kws27q1e2esvkqjpvectttxf). The stream finalize now runs through the
1005
+ // shared finalizeStreamPhase (in runRecoveryStream), so this is only the
1006
+ // boundary + completedAt. Returns undefined for a cancelled retry — there is
1007
+ // no review to open. Passes denialCancelSettled so a first denial that stopped
1008
+ // the RETRY waits for run.cancel() before the ledger read, exactly like the
1009
+ // primary path.
1010
+ const settleRetryTurn = async (retryResultStatus) => {
1011
+ const retryBoundary = retryResultStatus === "cancelled"
1012
+ ? undefined
1013
+ : await runBoundary(turnState.firstDenialDetected ? turnState.denialCancelSettled : undefined);
1014
+ // Phase 13 stamped completedAt BEFORE the retry ran. A terminal outcome
1015
+ // re-stamps it to the true end; a review pause CLEARS it — the primary
1016
+ // pause path never stamps it (a waiting turn is not complete).
1017
+ status.completedAt = retryBoundary?.waiting ? "" : utcTimestamp();
1018
+ return retryBoundary;
1019
+ };
1020
+ const runRecoveryStream = async (freshAgent, retryPrompt) => {
1021
+ // The fresh agent is now the live handle: point resolution at it so the
1022
+ // terminal close() (platform stop, or Phase 14 success) frees THIS agent's
1023
+ // executor lease rather than the disposed one it replaced. (Without this the
1024
+ // poisoned-handle path leaked the fresh agent — it closed the stale one.)
1025
+ resolution = { ...resolution, agent: freshAgent, agentId: freshAgent.agentId, isNew: true };
1026
+ turnState.streamErrorMessage = undefined;
1027
+ const retryRun = await freshAgent.send(retryPrompt, {
1028
+ onDelta: makeCursorTurnOnDelta(onDeltaDeps),
1029
+ });
1030
+ await consumeCursorTurnStream(retryRun, streamDeps);
1031
+ await finalizeStreamPhase();
1032
+ const terminal = await resolvePreBoundaryTerminal();
1033
+ if (terminal.kind !== "proceed")
1034
+ return { proceeded: false, terminal };
1035
+ const retryResult = await retryRun.wait();
1036
+ console.log(`ExecuteCursor retry run.wait(): execution=${executionId}, ` +
1037
+ `retryResult=${JSON.stringify(retryResult)}`);
1038
+ const retryBoundary = await settleRetryTurn(retryResult.status);
1039
+ return { proceeded: true, retryRun, retryResult, retryBoundary };
1040
+ };
1041
+ // The denial-settle wait applies only when a first denial stopped THIS run;
1042
+ // a normal completion passes no promise.
1043
+ const boundary = await runBoundary(turnState.firstDenialDetected ? turnState.denialCancelSettled : undefined);
1044
+ if (boundary.waiting) {
1045
+ return enterApprovalPause(boundary);
1191
1046
  }
1192
1047
  // Phase 13: Map final result
1193
1048
  const result = await run.wait();
@@ -1218,7 +1073,7 @@ turnSeq) {
1218
1073
  clearCapturedRejection(executionId);
1219
1074
  const classified = synthesizeError({
1220
1075
  sdkResultFields: sdkErrorStr,
1221
- streamErrorMessage,
1076
+ streamErrorMessage: turnState.streamErrorMessage,
1222
1077
  capturedRejection,
1223
1078
  conversationErrorText,
1224
1079
  isResumedHandle: resolution.reason === "resumed_successfully",
@@ -1273,50 +1128,21 @@ turnSeq) {
1273
1128
  catch (updateErr) {
1274
1129
  console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
1275
1130
  }
1276
- let retryWatchdog;
1277
- const retryRun = await freshAgent.send(freshPrompt, {
1278
- onDelta: ({ update }) => {
1279
- retryWatchdog?.recordActivity();
1280
- if (update.type === "turn-ended" && update.usage) {
1281
- usageAccumulator.addTurn(update.usage);
1282
- }
1283
- deltaEnricher.processDelta(update);
1284
- try {
1285
- heartbeat();
1286
- }
1287
- catch { /* swallow during retry */ }
1288
- },
1289
- });
1290
- // Mirror the primary stream's stall protection: a wedged retry must
1291
- // not hang the activity. On stall, cancel the run so the loop ends and
1292
- // retryRun.wait() resolves down the existing non-finished failure path.
1293
- retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
1294
- console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
1295
- if (retryRun.supports?.("cancel"))
1296
- void retryRun.cancel().catch(() => { });
1297
- });
1298
- streamErrorMessage = undefined;
1299
- try {
1300
- for await (const retryEvent of retryRun.stream()) {
1301
- if (Context.current().cancellationSignal.aborted)
1302
- break;
1303
- retryWatchdog.recordActivity();
1304
- accumulator.processEvent(retryEvent);
1305
- if (retryEvent.type === "status") {
1306
- const retryStatusEvent = retryEvent;
1307
- if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
1308
- streamErrorMessage = retryStatusEvent.message;
1309
- }
1310
- }
1311
- heartbeat();
1312
- }
1131
+ const outcome = await runRecoveryStream(freshAgent, freshPrompt);
1132
+ if (!outcome.proceeded) {
1133
+ if (outcome.terminal.kind === "return")
1134
+ return slimStatus(status);
1135
+ throw new CancelledFailure(outcome.terminal.message);
1313
1136
  }
1314
- finally {
1315
- retryWatchdog.stop();
1137
+ const { retryRun, retryResult, retryBoundary } = outcome;
1138
+ if (retryBoundary?.waiting) {
1139
+ // The retry's edits/denials armed the gate — pause for review. On a
1140
+ // retry error this supersedes the failure, exactly as on the primary
1141
+ // path (a captured change pauses the turn before run.wait() is
1142
+ // consulted).
1143
+ console.log(`ExecuteCursor poisoned-handle recovery paused for review: execution=${executionId}`);
1144
+ return enterApprovalPause(retryBoundary);
1316
1145
  }
1317
- const retryResult = await retryRun.wait();
1318
- console.log(`ExecuteCursor retry run.wait(): execution=${executionId}, ` +
1319
- `retryResult=${JSON.stringify(retryResult)}`);
1320
1146
  if (retryResult.status === "finished") {
1321
1147
  status.phase = ExecutionPhase.EXECUTION_COMPLETED;
1322
1148
  console.log(`ExecuteCursor poisoned-handle recovery SUCCEEDED: execution=${executionId}`);
@@ -1332,7 +1158,7 @@ turnSeq) {
1332
1158
  const retryConversationErrorText = await introspectConversation(retryRun, executionId);
1333
1159
  const retryClassified = synthesizeError({
1334
1160
  sdkResultFields: retryResult.result ? String(retryResult.result) : undefined,
1335
- streamErrorMessage,
1161
+ streamErrorMessage: turnState.streamErrorMessage,
1336
1162
  capturedRejection: retryRejection,
1337
1163
  conversationErrorText: retryConversationErrorText,
1338
1164
  isResumedHandle: false,
@@ -1370,51 +1196,21 @@ turnSeq) {
1370
1196
  catch (updateErr) {
1371
1197
  console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
1372
1198
  }
1373
- let retryWatchdog;
1374
- const retryRun = await freshAgent.send(effectivePrompt, {
1375
- onDelta: ({ update }) => {
1376
- retryWatchdog?.recordActivity();
1377
- if (update.type === "turn-ended" && update.usage) {
1378
- usageAccumulator.addTurn(update.usage);
1379
- }
1380
- deltaEnricher.processDelta(update);
1381
- try {
1382
- heartbeat();
1383
- }
1384
- catch { /* swallow during retry */ }
1385
- },
1386
- });
1387
- // Mirror the primary stream's stall protection: a wedged retry must
1388
- // not hang the activity. On stall, cancel the run so the loop ends and
1389
- // retryRun.wait() resolves down the existing non-finished failure path.
1390
- retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
1391
- console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
1392
- if (retryRun.supports?.("cancel"))
1393
- void retryRun.cancel().catch(() => { });
1394
- });
1395
- streamErrorMessage = undefined;
1396
- try {
1397
- for await (const retryEvent of retryRun.stream()) {
1398
- if (Context.current().cancellationSignal.aborted)
1399
- break;
1400
- retryWatchdog.recordActivity();
1401
- accumulator.processEvent(retryEvent);
1402
- if (retryEvent.type === "status") {
1403
- const retryStatusEvent = retryEvent;
1404
- if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
1405
- streamErrorMessage = retryStatusEvent.message;
1406
- }
1407
- }
1408
- heartbeat();
1409
- }
1199
+ const outcome = await runRecoveryStream(freshAgent, effectivePrompt);
1200
+ if (!outcome.proceeded) {
1201
+ if (outcome.terminal.kind === "return")
1202
+ return slimStatus(status);
1203
+ throw new CancelledFailure(outcome.terminal.message);
1410
1204
  }
1411
- finally {
1412
- retryWatchdog.stop();
1205
+ const { retryResult, retryBoundary } = outcome;
1206
+ if (retryBoundary?.waiting) {
1207
+ // The retry's edits/denials armed the gate — pause for review (see
1208
+ // the poisoned-handle branch above for the precedence rationale).
1209
+ console.log(`ExecuteCursor transport-timeout recovery paused for review: execution=${executionId}`);
1210
+ return enterApprovalPause(retryBoundary);
1413
1211
  }
1414
- const retryResult = await retryRun.wait();
1415
1212
  if (retryResult.status === "finished") {
1416
1213
  status.phase = ExecutionPhase.EXECUTION_COMPLETED;
1417
- resolution = { ...resolution, agent: freshAgent, agentId: freshAgent.agentId, isNew: true };
1418
1214
  break;
1419
1215
  }
1420
1216
  status.phase = ExecutionPhase.EXECUTION_FAILED;
@@ -1471,10 +1267,10 @@ turnSeq) {
1471
1267
  if (structuredOutput !== undefined) {
1472
1268
  status.structuredOutput = structuredOutput;
1473
1269
  }
1474
- // Plan mode: publish the final plan message as a plan.md artifact. The
1475
- // Cursor harness has no auto-publish pipeline, so this is the only
1476
- // artifact path; build storage from the same config-driven factory the
1477
- // native harness uses.
1270
+ // Plan mode: publish the final plan message as a plan artifact (named
1271
+ // from the plan's title). The Cursor harness has no auto-publish
1272
+ // pipeline, so this is the only artifact path; build storage from the
1273
+ // same config-driven factory the native harness uses.
1478
1274
  if (interactionMode === InteractionMode.PLAN && finalText && artifactStorage) {
1479
1275
  try {
1480
1276
  await publishPlanArtifact({ status, executionId, planText: finalText, artifactStorage });
@@ -1533,7 +1329,7 @@ turnSeq) {
1533
1329
  timestamp: utcTimestamp(),
1534
1330
  }));
1535
1331
  }
1536
- else if (pauseDetected) {
1332
+ else if (turnState.pauseDetected) {
1537
1333
  console.log(`ExecuteCursor cancelled (pause) for execution ${executionId}`);
1538
1334
  status.phase = ExecutionPhase.EXECUTION_PAUSED;
1539
1335
  status.messages.push(create(AgentMessageSchema, {
@@ -1563,7 +1359,7 @@ turnSeq) {
1563
1359
  // treat the execution as paused rather than failed. The error was likely
1564
1360
  // caused by the cancellation (e.g. SDK stream teardown) and should not
1565
1361
  // overwrite the PAUSED state that the Pause RPC already set in the DB.
1566
- if (pauseDetected) {
1362
+ if (turnState.pauseDetected) {
1567
1363
  const errDetail = err instanceof Error ? err.message : String(err);
1568
1364
  console.log(`ExecuteCursor error during pause (treating as pause): execution=${executionId}, error=${errDetail}`);
1569
1365
  status.phase = ExecutionPhase.EXECUTION_PAUSED;
@@ -1654,10 +1450,15 @@ turnSeq) {
1654
1450
  return slimStatus(status);
1655
1451
  }
1656
1452
  finally {
1657
- // Disarm the stall watchdog on EVERY exit path (idempotent). The happy
1658
- // path stops it after the stream loop; this covers throws before that
1659
- // point so no orphaned timer survives the activity.
1660
- stallWatchdog?.stop();
1453
+ // End the OTel turn span + record metrics with the final token snapshot on
1454
+ // EVERY exit path (idempotent). Placed here so the span covers any recovery
1455
+ // retry (whose tokens accrue after the primary stream) and never leaks on an
1456
+ // early return or throw. A no-op when OTel is off or the span never opened.
1457
+ await finishTurnTelemetry?.();
1458
+ // Disarm the stall watchdog on EVERY exit path (idempotent). consumeCursorTurnStream
1459
+ // stops the one it armed; this covers throws before that point so no orphaned
1460
+ // timer survives the activity.
1461
+ turnState.stallWatchdog?.stop();
1661
1462
  // Close the denial-ledger watcher on EVERY exit path (idempotent) so no
1662
1463
  // orphaned fs.watch handle survives the activity.
1663
1464
  stopDenialWatcher?.();