@stigmer/runner 3.1.3 → 3.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/activities/execute-cursor/capture-flow.d.ts +26 -15
- package/dist/activities/execute-cursor/capture-flow.js +56 -18
- package/dist/activities/execute-cursor/capture-flow.js.map +1 -1
- package/dist/activities/execute-cursor/command-provenance.d.ts +11 -25
- package/dist/activities/execute-cursor/command-provenance.js +25 -115
- package/dist/activities/execute-cursor/command-provenance.js.map +1 -1
- package/dist/activities/execute-cursor/index.js +296 -477
- package/dist/activities/execute-cursor/index.js.map +1 -1
- package/dist/activities/execute-cursor/todo-tracker.d.ts +6 -1
- package/dist/activities/execute-cursor/todo-tracker.js +15 -43
- package/dist/activities/execute-cursor/todo-tracker.js.map +1 -1
- package/dist/activities/execute-cursor/turn-stream.d.ts +141 -0
- package/dist/activities/execute-cursor/turn-stream.js +249 -0
- package/dist/activities/execute-cursor/turn-stream.js.map +1 -0
- package/dist/activities/execute-deep-agent/command-provenance.d.ts +61 -0
- package/dist/activities/execute-deep-agent/command-provenance.js +72 -0
- package/dist/activities/execute-deep-agent/command-provenance.js.map +1 -0
- package/dist/activities/execute-deep-agent/index.js +60 -18
- package/dist/activities/execute-deep-agent/index.js.map +1 -1
- package/dist/activities/execute-deep-agent/status-builder.js +8 -1
- package/dist/activities/execute-deep-agent/status-builder.js.map +1 -1
- package/dist/activities/execute-deep-agent/v3-status-builder.js +12 -1
- package/dist/activities/execute-deep-agent/v3-status-builder.js.map +1 -1
- package/dist/otel.js +10 -0
- package/dist/otel.js.map +1 -1
- package/dist/shared/filereview/cas-progress.d.ts +63 -0
- package/dist/shared/filereview/cas-progress.js +128 -0
- package/dist/shared/filereview/cas-progress.js.map +1 -0
- package/dist/shared/filereview/cas-substrate.d.ts +29 -0
- package/dist/shared/filereview/cas-substrate.js +46 -21
- package/dist/shared/filereview/cas-substrate.js.map +1 -1
- package/dist/shared/filereview/command-provenance.d.ts +93 -0
- package/dist/shared/filereview/command-provenance.js +132 -0
- package/dist/shared/filereview/command-provenance.js.map +1 -0
- package/dist/shared/filereview/git-substrate.d.ts +9 -3
- package/dist/shared/filereview/git-substrate.js.map +1 -1
- package/dist/shared/filereview/index.d.ts +4 -3
- package/dist/shared/filereview/index.js +3 -2
- package/dist/shared/filereview/index.js.map +1 -1
- package/dist/shared/filereview/progress.d.ts +105 -34
- package/dist/shared/filereview/progress.js +96 -34
- package/dist/shared/filereview/progress.js.map +1 -1
- package/dist/shared/plan-mode-prompt.d.ts +9 -0
- package/dist/shared/plan-mode-prompt.js +18 -0
- package/dist/shared/plan-mode-prompt.js.map +1 -1
- package/dist/shared/todos.d.ts +56 -0
- package/dist/shared/todos.js +98 -0
- package/dist/shared/todos.js.map +1 -0
- package/dist/shared/tool-row.d.ts +16 -0
- package/dist/shared/tool-row.js +31 -0
- package/dist/shared/tool-row.js.map +1 -1
- package/package.json +2 -2
- package/src/__tests__/otel-turn-span.test.ts +61 -0
- package/src/activities/execute-cursor/__tests__/progress-substrate.test.ts +169 -0
- package/src/activities/execute-cursor/__tests__/turn-stream.test.ts +349 -0
- package/src/activities/execute-cursor/capture-flow.ts +71 -25
- package/src/activities/execute-cursor/command-provenance.ts +25 -120
- package/src/activities/execute-cursor/index.ts +344 -504
- package/src/activities/execute-cursor/todo-tracker.ts +17 -59
- package/src/activities/execute-cursor/turn-stream.ts +418 -0
- package/src/activities/execute-deep-agent/__tests__/command-provenance.test.ts +252 -0
- package/src/activities/execute-deep-agent/__tests__/status-builder.test.ts +78 -0
- package/src/activities/execute-deep-agent/__tests__/v3-status-builder.test.ts +105 -1
- package/src/activities/execute-deep-agent/command-provenance.ts +102 -0
- package/src/activities/execute-deep-agent/index.ts +74 -18
- package/src/activities/execute-deep-agent/status-builder.ts +9 -0
- package/src/activities/execute-deep-agent/v3-status-builder.ts +13 -0
- package/src/otel.ts +8 -0
- package/src/shared/__tests__/todos.test.ts +216 -0
- package/src/shared/filereview/__tests__/cas-progress.test.ts +228 -0
- package/src/shared/filereview/__tests__/cas-substrate.test.ts +66 -0
- package/src/shared/filereview/__tests__/command-provenance.test.ts +252 -0
- package/src/shared/filereview/__tests__/progress.test.ts +112 -10
- package/src/shared/filereview/cas-progress.ts +170 -0
- package/src/shared/filereview/cas-substrate.ts +69 -24
- package/src/shared/filereview/command-provenance.ts +180 -0
- package/src/shared/filereview/git-substrate.ts +9 -3
- package/src/shared/filereview/index.ts +15 -1
- package/src/shared/filereview/progress.ts +171 -47
- package/src/shared/plan-mode-prompt.ts +18 -0
- package/src/shared/todos.ts +126 -0
- package/src/shared/tool-row.ts +34 -0
|
@@ -30,7 +30,7 @@ import { create, clone } from "@bufbuild/protobuf";
|
|
|
30
30
|
import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
|
|
31
31
|
import { AgentMessageSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
|
|
32
32
|
import { SubAgentExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/subagent_pb";
|
|
33
|
-
import {
|
|
33
|
+
import { ExecutionPhase, FileChangeSetStatus, InteractionMode, MessageType, ApprovalAction } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
|
|
34
34
|
import { StigmerClient } from "../../client/stigmer-client.js";
|
|
35
35
|
import { resolveAgent } from "./session-lifecycle.js";
|
|
36
36
|
import { CursorMode } from "@stigmer/protos/ai/stigmer/agentic/session/v1/enum_pb";
|
|
@@ -38,12 +38,11 @@ import { determineCursorMode, isCloudMode } from "./cursor-mode.js";
|
|
|
38
38
|
import { MessageAccumulator, cancelInProgressSubAgentProtos, collapseRedundantToolCallTwins } from "./message-translator.js";
|
|
39
39
|
import { utcTimestamp, persistStatus, reportSetupProgress, slimStatus } from "../../shared/status.js";
|
|
40
40
|
import { withholdSecretContentFromMessages } from "../../shared/tool-row.js";
|
|
41
|
-
import {
|
|
41
|
+
import { StallTimeoutError, formatStallFailure } from "../../shared/stall-watchdog.js";
|
|
42
42
|
import { resolveUsableArtifactStorage, loadArtifactStorageConfig } from "../../shared/artifact-storage.js";
|
|
43
43
|
import { publishPlanArtifact } from "../../shared/plan-artifact.js";
|
|
44
44
|
import { DeltaEnricher } from "./delta-enricher.js";
|
|
45
45
|
import { TodoTracker } from "./todo-tracker.js";
|
|
46
|
-
import { shouldPersistStreamingStatus } from "./persist-decision.js";
|
|
47
46
|
import { StreamingUpdateScheduler, loadStreamingConfig } from "../../shared/streaming-scheduler.js";
|
|
48
47
|
import { createCursorEventRecorder } from "./cursor-event-recorder.js";
|
|
49
48
|
import { resolveMcpServers, validateMcpServerEnv } from "./mcp-resolver.js";
|
|
@@ -61,11 +60,12 @@ import { installHitlGate, removeHitlGate } from "./workspace-setup.js";
|
|
|
61
60
|
import { ensureHitlDir } from "../../shared/workspace/platform-dir.js";
|
|
62
61
|
import { acquireWorkspaceLock, WorkspaceLockCancelledError, WorkspaceLockTimeoutError, } from "../../shared/workspace/workspace-lock.js";
|
|
63
62
|
import { LocalWorkspaceBackend } from "../../shared/workspace/local-backend.js";
|
|
64
|
-
import { buildApprovalState, buildApprovalGrants, emitCursorGrantReceipts,
|
|
63
|
+
import { buildApprovalState, buildApprovalGrants, emitCursorGrantReceipts, reconstructAdjudicatedApprovals, watchDenialLedger } from "./approval-state.js";
|
|
65
64
|
import { applyApprovedWholeFileWrites, excludeAppliedFromGrants } from "./exact-apply.js";
|
|
66
65
|
import { isGitWorkTree } from "../../shared/filereview/git-substrate.js";
|
|
67
|
-
import { captureBaselineToLedger,
|
|
66
|
+
import { captureBaselineToLedger, buildCursorProgressSubstrate, applyCaptureDecisions, deriveCaptureMode, } from "./capture-flow.js";
|
|
68
67
|
import { runTurnBoundary } from "./turn-boundary.js";
|
|
68
|
+
import { consumeCursorTurnStream, makeCursorTurnOnDelta, newTurnStreamState, } from "./turn-stream.js";
|
|
69
69
|
import { newProgressCaptureState, } from "../../shared/filereview/progress.js";
|
|
70
70
|
import { deriveExecutionFingerprintKey } from "../../shared/approval-fingerprint.js";
|
|
71
71
|
import { getRunnerHitlMasterSecret } from "../../shared/fingerprint-secret.js";
|
|
@@ -155,40 +155,30 @@ turnSeq) {
|
|
|
155
155
|
};
|
|
156
156
|
let sessionId;
|
|
157
157
|
let session;
|
|
158
|
-
|
|
158
|
+
// Single owner for every flag the turn's stream produces (pause, stall,
|
|
159
|
+
// first-denial, platform-stop, event count, the stall watchdog, …). Created
|
|
160
|
+
// once here — before the fs denial-watcher, the SDK onDelta, the stall
|
|
161
|
+
// watchdog, and the stream loop are wired — so all four producers plus the
|
|
162
|
+
// epilogue and the outer catch/finally share ONE source of truth. The primary
|
|
163
|
+
// turn and both recovery retries drive the same stream code against this
|
|
164
|
+
// object (see turn-stream.ts for per-field ownership).
|
|
165
|
+
const turnState = newTurnStreamState();
|
|
166
|
+
// NOT part of turnState: derived post-loop from the periodic heartbeat +
|
|
167
|
+
// shutdown signal (a runner-manager shutdown, not a stream event), and read by
|
|
168
|
+
// the epilogue + outer catch. Kept as a plain let alongside the stream flags.
|
|
159
169
|
let workerShutdownDetected = false;
|
|
160
|
-
// Set by the stall watchdog when the SDK stream makes no progress for
|
|
161
|
-
// config.cursorStreamStallTimeoutMs. stallError carries the recognizable
|
|
162
|
-
// message surfaced to the user; both feed the Phase 11a stall branch.
|
|
163
|
-
let stallDetected = false;
|
|
164
|
-
let stallError;
|
|
165
|
-
// Set the moment the preToolUse hook records its first denial in the ledger.
|
|
166
|
-
// We then stop consuming the stream and cancel the run so the model never
|
|
167
|
-
// reacts to Cursor's tool-failure surface (narrate defeat, attempt a second
|
|
168
|
-
// gated tool) — converging the Cursor harness toward the native harness, which
|
|
169
|
-
// pauses BEFORE the model sees a denial. Phase 12 reconciles the denied tool
|
|
170
|
-
// calls into WAITING_FOR_APPROVAL exactly as after a natural stream end.
|
|
171
|
-
let firstDenialDetected = false;
|
|
172
|
-
// Flipped by the denial-ledger fs watcher the instant the hook writes a
|
|
173
|
-
// denial, so the NEXT stream event of ANY type triggers the ledger read —
|
|
174
|
-
// instead of waiting for the next tool_call event, during which the model's
|
|
175
|
-
// full post-denial reaction (thinking, narration, a workaround tool) would
|
|
176
|
-
// stream and persist (observed in production: aex_01kwj07f7g23c3wp9sn8496z5g).
|
|
177
|
-
// The tool_call-event read below remains the backstop where fs.watch is
|
|
178
|
-
// unreliable.
|
|
179
|
-
let denialLedgerDirty = false;
|
|
180
170
|
let stopDenialWatcher;
|
|
181
|
-
// The in-flight run.cancel() started by the first-denial stop. Awaited
|
|
182
|
-
// (timeboxed) before Phase 12 so the agent process has actually stopped
|
|
183
|
-
// before the final ledger read and the turn-boundary tree capture — closing
|
|
184
|
-
// the race where a post-denial workaround's ledger entry lands after the
|
|
185
|
-
// read (it would then never be collapsed) or a late tool mutates the tree
|
|
186
|
-
// mid-capture.
|
|
187
|
-
let denialCancelSettled;
|
|
188
171
|
let periodicHeartbeat;
|
|
189
|
-
//
|
|
190
|
-
//
|
|
191
|
-
|
|
172
|
+
// Ends the OTel turn span + records turn metrics with the FINAL token snapshot.
|
|
173
|
+
// Hoisted and invoked from the finally so the span is closed exactly once on
|
|
174
|
+
// EVERY exit path — a happy completion, an approval pause, an early return, a
|
|
175
|
+
// throw, or a recovery retry (whose tokens accrue AFTER the primary stream
|
|
176
|
+
// ends). Ending it inline in the epilogue leaked the span on every non-happy
|
|
177
|
+
// path and excluded retry tokens/duration. Assigned when the span is created
|
|
178
|
+
// (once usageAccumulator exists); undefined — a no-op — before then (e.g. a
|
|
179
|
+
// pure-reconcile resume that returns before the agent runs) or when OTel is
|
|
180
|
+
// off. Idempotent: safe to call more than once.
|
|
181
|
+
let finishTurnTelemetry;
|
|
192
182
|
// Session HITL directory (runner-owned, outside the workspace) where the hook
|
|
193
183
|
// script, approval-state file, and denial ledger live. Set once the gate is
|
|
194
184
|
// installed; the WAITING_FOR_APPROVAL path reads the denial ledger from here.
|
|
@@ -585,7 +575,21 @@ turnSeq) {
|
|
|
585
575
|
// reset may flip the flag once before the run starts; the loop's read then
|
|
586
576
|
// sees an empty ledger and clears it — harmless by construction.
|
|
587
577
|
stopDenialWatcher = watchDenialLedger(hitlDir, () => {
|
|
588
|
-
denialLedgerDirty = true;
|
|
578
|
+
turnState.denialLedgerDirty = true;
|
|
579
|
+
});
|
|
580
|
+
// Mid-run live capture (DD-32 / DD-33): choose the progress substrate for this
|
|
581
|
+
// turn's workspace shape ONCE (git / non-git CAS / hybrid). It owns its own
|
|
582
|
+
// short-circuit cache across the loop's persists; the floor lives in
|
|
583
|
+
// progressState. Undefined outside capture mode — writes are deny-gated and
|
|
584
|
+
// nothing is captured.
|
|
585
|
+
const progressSubstrate = buildCursorProgressSubstrate({
|
|
586
|
+
captureMode,
|
|
587
|
+
gitWorkspace,
|
|
588
|
+
workspaceRoot: primaryWorkspaceDir,
|
|
589
|
+
baselineTree,
|
|
590
|
+
executionId,
|
|
591
|
+
hitlDir,
|
|
592
|
+
storage: artifactStorage,
|
|
589
593
|
});
|
|
590
594
|
// Phase 5d: Ensure model pricing registry is populated before validation
|
|
591
595
|
await ensurePricingLoaded();
|
|
@@ -701,25 +705,62 @@ turnSeq) {
|
|
|
701
705
|
// Phase 10b: Initialize usage accumulator for runner-side token tracking
|
|
702
706
|
await ensurePricingLoaded();
|
|
703
707
|
const usageAccumulator = new UsageAccumulator(validatedModel);
|
|
704
|
-
// Phase 10c: Start OTel turn span
|
|
708
|
+
// Phase 10c: Start OTel turn span. Coarse-grained — spans the whole turn
|
|
709
|
+
// (agent.send + stream + any recovery retry + the turn boundary), ended once
|
|
710
|
+
// from the finally via finishTurnTelemetry with the final token snapshot.
|
|
705
711
|
const { startCursorTurnSpan } = await import("../../otel.js");
|
|
706
712
|
const turnSpan = await startCursorTurnSpan({
|
|
707
713
|
model: validatedModel,
|
|
708
714
|
mode: agentMode,
|
|
709
715
|
sessionId: sessionId ?? "",
|
|
710
716
|
});
|
|
717
|
+
// Bind the telemetry-finish closure now that the span + usage accumulator
|
|
718
|
+
// exist. Reads usageAccumulator at CALL time (in the finally), so it captures
|
|
719
|
+
// tokens from any recovery retry that ran after the primary stream. Guarded
|
|
720
|
+
// so a second call (finally after an inline path already finished it) is a
|
|
721
|
+
// no-op. Metrics failures are swallowed — OTel is optional.
|
|
722
|
+
let turnTelemetryFinished = false;
|
|
723
|
+
finishTurnTelemetry = async () => {
|
|
724
|
+
if (turnTelemetryFinished)
|
|
725
|
+
return;
|
|
726
|
+
turnTelemetryFinished = true;
|
|
727
|
+
const usage = usageAccumulator.snapshot();
|
|
728
|
+
turnSpan.setTokens(Number(usage.inputTokens), Number(usage.outputTokens));
|
|
729
|
+
turnSpan.end();
|
|
730
|
+
try {
|
|
731
|
+
const { recordTurnMetrics } = await import("../../otel.js");
|
|
732
|
+
const durationMs = Date.now() - (status.startedAt ? new Date(status.startedAt).getTime() : Date.now());
|
|
733
|
+
await recordTurnMetrics({
|
|
734
|
+
durationMs,
|
|
735
|
+
inputTokens: Number(usage.inputTokens),
|
|
736
|
+
outputTokens: Number(usage.outputTokens),
|
|
737
|
+
model: validatedModel,
|
|
738
|
+
mode: agentMode,
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
catch {
|
|
742
|
+
// Metrics not initialized — silently skip.
|
|
743
|
+
}
|
|
744
|
+
};
|
|
711
745
|
// Phase 11: Send message and stream events
|
|
712
746
|
status.phase = ExecutionPhase.EXECUTION_IN_PROGRESS;
|
|
713
747
|
const deltaEnricher = new DeltaEnricher();
|
|
714
748
|
const todoTracker = new TodoTracker(status.todos);
|
|
715
749
|
const eventRecorder = createCursorEventRecorder(executionId);
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
let streamErrorMessage;
|
|
750
|
+
// The two recovery retries (poisoned-handle / transport-timeout) below run at
|
|
751
|
+
// most once per turn; this guard is the latch.
|
|
719
752
|
let alreadyRetriedWithFreshAgent = false;
|
|
720
|
-
//
|
|
721
|
-
//
|
|
722
|
-
|
|
753
|
+
// The shared onDelta only needs the usage/enricher/heartbeat/state subset,
|
|
754
|
+
// and it is wired at SEND time — before the accumulator exists — so it takes
|
|
755
|
+
// the narrow deps. The primary send and both retry sends reuse this object.
|
|
756
|
+
const onDeltaDeps = {
|
|
757
|
+
usageAccumulator,
|
|
758
|
+
deltaEnricher,
|
|
759
|
+
heartbeat,
|
|
760
|
+
promptEstimatedTokens,
|
|
761
|
+
executionId,
|
|
762
|
+
state: turnState,
|
|
763
|
+
};
|
|
723
764
|
// Periodic heartbeat keeps Temporal informed during silent SDK operations
|
|
724
765
|
// (e.g. long tool calls, MCP requests, model thinking). Without this,
|
|
725
766
|
// the 2-minute heartbeat timeout can cancel the activity and mislabel
|
|
@@ -745,54 +786,11 @@ turnSeq) {
|
|
|
745
786
|
// Fallback: if the Temporal signal doesn't support setMaxListeners
|
|
746
787
|
// (e.g. older SDK), the warning is harmless — ignore.
|
|
747
788
|
}
|
|
789
|
+
// The stall watchdog is armed inside consumeCursorTurnStream (it needs the
|
|
790
|
+
// run to cancel), stored on turnState.stallWatchdog so this shared onDelta can
|
|
791
|
+
// reset it and the activity's finally can stop it as a backstop.
|
|
748
792
|
const run = await resolution.agent.send(effectivePrompt, {
|
|
749
|
-
onDelta: (
|
|
750
|
-
// Reset the stall timer on the delta channel too: a long model
|
|
751
|
-
// generation emits token deltas but few discrete stream events, so
|
|
752
|
-
// resetting only in the stream loop would false-positive a stall.
|
|
753
|
-
stallWatchdog?.recordActivity();
|
|
754
|
-
if (update.type === "turn-ended" && update.usage) {
|
|
755
|
-
usageAccumulator.addTurn(update.usage);
|
|
756
|
-
if (!firstTurnAttributionLogged) {
|
|
757
|
-
firstTurnAttributionLogged = true;
|
|
758
|
-
const sdkInputTokens = update.usage.inputTokens ?? 0;
|
|
759
|
-
const cursorOverhead = Math.max(0, sdkInputTokens - promptEstimatedTokens);
|
|
760
|
-
console.log(`ExecuteCursor context attribution (first turn): execution=${executionId}, ` +
|
|
761
|
-
`sdkInputTokens=${sdkInputTokens}, stigmerPreamble=${promptEstimatedTokens}, ` +
|
|
762
|
-
`cursorOverhead=${cursorOverhead} (estimated)`);
|
|
763
|
-
}
|
|
764
|
-
}
|
|
765
|
-
deltaEnricher.processDelta(update);
|
|
766
|
-
try {
|
|
767
|
-
heartbeat();
|
|
768
|
-
}
|
|
769
|
-
catch (hbErr) {
|
|
770
|
-
if (hbErr instanceof CancelledFailure) {
|
|
771
|
-
pauseDetected = true;
|
|
772
|
-
return;
|
|
773
|
-
}
|
|
774
|
-
throw hbErr;
|
|
775
|
-
}
|
|
776
|
-
},
|
|
777
|
-
});
|
|
778
|
-
// Arm the stall watchdog now that the run exists. The periodic heartbeat
|
|
779
|
-
// above proves the process is alive, not that the agent is progressing: if
|
|
780
|
-
// the stream wedges (a tool call or model connection that never returns),
|
|
781
|
-
// no event/delta arrives, Phase 12 below is never reached, and the
|
|
782
|
-
// execution hangs at EXECUTION_IN_PROGRESS forever. On stall we end the run
|
|
783
|
-
// cleanly via the SDK's run.cancel() (guarded by supports("cancel")), which
|
|
784
|
-
// unblocks the for-await; the stallDetected branch in Phase 11a then
|
|
785
|
-
// reports EXECUTION_FAILED with a recognizable, actionable message.
|
|
786
|
-
stallWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
787
|
-
stallDetected = true;
|
|
788
|
-
stallError = new StallTimeoutError(idleMs, lastToolName ? `last tool: ${lastToolName}` : undefined);
|
|
789
|
-
console.warn(`ExecuteCursor stall detected: execution=${executionId}, idleMs=${idleMs}, lastTool=${lastToolName ?? "none"}`);
|
|
790
|
-
if (run.supports?.("cancel")) {
|
|
791
|
-
void run.cancel().catch((cancelErr) => {
|
|
792
|
-
console.warn(`ExecuteCursor run.cancel() after stall failed (non-fatal): execution=${executionId}, ` +
|
|
793
|
-
`error=${cancelErr instanceof Error ? cancelErr.message : cancelErr}`);
|
|
794
|
-
});
|
|
795
|
-
}
|
|
793
|
+
onDelta: makeCursorTurnOnDelta(onDeltaDeps),
|
|
796
794
|
});
|
|
797
795
|
// Everything at an index >= this was produced by THIS turn's stream — the
|
|
798
796
|
// positional turn boundary the approved-command provenance (DD-28) scopes
|
|
@@ -808,273 +806,163 @@ turnSeq) {
|
|
|
808
806
|
// flush; high-frequency token deltas ride this scheduler's time cadence
|
|
809
807
|
// (env-tunable via STREAMING_* — see loadStreamingConfig).
|
|
810
808
|
const scheduler = new StreamingUpdateScheduler(loadStreamingConfig());
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
// tool — before Cursor surfaces the failure to the model — and the fs
|
|
838
|
-
// watcher flips denialLedgerDirty the moment that write lands. Confirming
|
|
839
|
-
// the flag with a read on the very next event (of ANY type — thinking
|
|
840
|
-
// deltas arrive within milliseconds) ends the turn before the model's
|
|
841
|
-
// reaction can persist: waiting for the next tool_call event let the full
|
|
842
|
-
// post-denial reaction (thinking, narration, a workaround shell) stream
|
|
843
|
-
// and persist live (production case aex_01kwj07f7g23c3wp9sn8496z5g). The
|
|
844
|
-
// tool_call-event read stays as the backstop for platforms where fs.watch
|
|
845
|
-
// is unreliable; the current event was already accumulated above, so the
|
|
846
|
-
// anchor's own row is always present for the Phase 12 gate overlay. This
|
|
847
|
-
// mirrors the native harness's pause-before-react semantics; Phase 12
|
|
848
|
-
// reconciles the denied calls and its trim remains the last-resort
|
|
849
|
-
// backstop for anything that persisted before the stop.
|
|
850
|
-
if (!firstDenialDetected && hitlDir && (denialLedgerDirty || event.type === "tool_call")) {
|
|
851
|
-
denialLedgerDirty = false;
|
|
852
|
-
const denials = await readDenialLedger(hitlDir);
|
|
853
|
-
if (denials.length > 0) {
|
|
854
|
-
firstDenialDetected = true;
|
|
855
|
-
console.log(`ExecuteCursor first denial detected (${denials.length} ledger ` +
|
|
856
|
-
`entr${denials.length === 1 ? "y" : "ies"}); stopping turn to pause ` +
|
|
857
|
-
`cleanly for approval: execution=${executionId}`);
|
|
858
|
-
if (run.supports?.("cancel")) {
|
|
859
|
-
// Kept (not fire-and-forget): awaited timeboxed before Phase 12 so
|
|
860
|
-
// the ledger read and tree capture see a stopped agent.
|
|
861
|
-
denialCancelSettled = run.cancel().then(() => { }, (cancelErr) => {
|
|
862
|
-
console.warn(`ExecuteCursor run.cancel() after first denial failed (non-fatal): ` +
|
|
863
|
-
`execution=${executionId}, ` +
|
|
864
|
-
`error=${cancelErr instanceof Error ? cancelErr.message : cancelErr}`);
|
|
865
|
-
});
|
|
866
|
-
}
|
|
867
|
-
break;
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
deltaEnricher.applyEnrichments(status.messages);
|
|
871
|
-
eventCount++;
|
|
872
|
-
if (event.type === "status") {
|
|
873
|
-
console.log(`ExecuteCursor stream status: execution=${executionId}, status=${JSON.stringify(event)}`);
|
|
874
|
-
const statusEvent = event;
|
|
875
|
-
if (statusEvent.status === "ERROR" && statusEvent.message) {
|
|
876
|
-
streamErrorMessage = statusEvent.message;
|
|
877
|
-
}
|
|
878
|
-
}
|
|
879
|
-
const shouldPersist = shouldPersistStreamingStatus({
|
|
880
|
-
deltaEnricherDirty: deltaEnricher.isDirty,
|
|
881
|
-
todosDirty: todoTracker.isDirty,
|
|
882
|
-
contentDirty: accumulator.isDirty,
|
|
883
|
-
}, scheduler, eventCount);
|
|
884
|
-
if (usageAccumulator.hasTurns) {
|
|
885
|
-
status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
|
|
886
|
-
}
|
|
887
|
-
if (shouldPersist) {
|
|
888
|
-
// Sync sub-agent executions into status before every persist so the
|
|
889
|
-
// live UI reflects delegation (including the IN_PROGRESS state) while
|
|
890
|
-
// the parent is still running — matching the native harness, which
|
|
891
|
-
// calls syncSubAgentExecutions() on each persist. Without this, the
|
|
892
|
-
// accumulator tracked sub-agents in memory but they only reached the
|
|
893
|
-
// status (and the subscriber stream) after the loop ended.
|
|
894
|
-
status.subAgentExecutions = accumulator.subAgentExecutions;
|
|
895
|
-
// Mid-run live capture (DD-32): attach the "N files changed so far"
|
|
896
|
-
// snapshot onto status.file_change_progress, throttled internally by the
|
|
897
|
-
// floor + tree-sha short-circuit. Git capture mode only (a pinned baseline
|
|
898
|
-
// exists); shell + sub-agent + tool edits are all captured for free by the
|
|
899
|
-
// workspace-wide diff. Never authoritative — the turn-boundary candidate
|
|
900
|
-
// remains the reviewed diff.
|
|
901
|
-
if (captureMode && gitWorkspace && baselineTree && primaryWorkspaceDir) {
|
|
902
|
-
await captureProgressToStatus({
|
|
903
|
-
status,
|
|
904
|
-
gitRoot: primaryWorkspaceDir,
|
|
905
|
-
executionId,
|
|
906
|
-
changeSetId,
|
|
907
|
-
baselineTree,
|
|
908
|
-
state: progressState,
|
|
909
|
-
});
|
|
910
|
-
}
|
|
911
|
-
const signal = await persist(status);
|
|
912
|
-
deltaEnricher.markPersisted();
|
|
913
|
-
todoTracker.markPersisted();
|
|
914
|
-
accumulator.markPersisted();
|
|
915
|
-
scheduler.markUpdateSent(eventCount);
|
|
916
|
-
heartbeat();
|
|
917
|
-
if (signal === ExecutionControlSignal.STOP) {
|
|
918
|
-
platformStopSignaled = true;
|
|
919
|
-
console.warn(`ExecuteCursor platform stop signal received: execution=${executionId}`);
|
|
920
|
-
}
|
|
921
|
-
}
|
|
922
|
-
if (platformStopSignaled) {
|
|
923
|
-
console.log(`ExecuteCursor stopping stream due to platform stop signal: execution=${executionId}`);
|
|
924
|
-
break;
|
|
925
|
-
}
|
|
926
|
-
}
|
|
927
|
-
}
|
|
928
|
-
catch (streamErr) {
|
|
929
|
-
// run.cancel() — from the stall watchdog or the first-denial stop — can
|
|
930
|
-
// make the stream iterator reject as it tears down; that is the expected
|
|
931
|
-
// teardown for both, so swallow it and fall through (stallDetected ->
|
|
932
|
-
// Phase 11a; firstDenialDetected -> Phase 12). Anything else is a genuine
|
|
933
|
-
// stream failure — rethrow it to the outer error handler.
|
|
934
|
-
if (!stallDetected && !firstDenialDetected)
|
|
935
|
-
throw streamErr;
|
|
936
|
-
console.warn(`ExecuteCursor stream ended via cancel: execution=${executionId}, ` +
|
|
937
|
-
`stall=${stallDetected}, firstDenial=${firstDenialDetected}`);
|
|
938
|
-
}
|
|
809
|
+
// Full deps for the shared stream loop — the collaborators + the injected
|
|
810
|
+
// heartbeat/cancellation (so the loop is testable, mirroring the deep-agent
|
|
811
|
+
// streamExecution seam), all keyed off the single turnState. Consumed by the
|
|
812
|
+
// primary stream here and by both recovery retries below.
|
|
813
|
+
const streamDeps = {
|
|
814
|
+
...onDeltaDeps,
|
|
815
|
+
status,
|
|
816
|
+
accumulator,
|
|
817
|
+
todoTracker,
|
|
818
|
+
eventRecorder,
|
|
819
|
+
scheduler,
|
|
820
|
+
progressSubstrate,
|
|
821
|
+
progressState,
|
|
822
|
+
changeSetId,
|
|
823
|
+
hitlDir,
|
|
824
|
+
stallTimeoutMs: config.cursorStreamStallTimeoutMs,
|
|
825
|
+
persist,
|
|
826
|
+
isCancelled: () => Context.current().cancellationSignal.aborted,
|
|
827
|
+
};
|
|
828
|
+
// Primary stream. consumeCursorTurnStream owns the per-event loop (transcript,
|
|
829
|
+
// todos, sub-agent tracking, live persist, DD-32/DD-33 mid-run progress, the
|
|
830
|
+
// first-denial early stop, and the stall watchdog) and reports why it ended;
|
|
831
|
+
// resolvePreBoundaryTerminal below maps that to a terminal outcome. The two
|
|
832
|
+
// recovery retries drive the identical loop, so they inherit every one of
|
|
833
|
+
// these behaviors instead of the old bare loop that dropped them.
|
|
834
|
+
await consumeCursorTurnStream(run, streamDeps);
|
|
939
835
|
periodicHeartbeat.stop();
|
|
940
|
-
|
|
941
|
-
//
|
|
942
|
-
//
|
|
943
|
-
//
|
|
944
|
-
//
|
|
945
|
-
// signal check catches this case.
|
|
836
|
+
// Worker-shutdown vs. user-pause disambiguation. Primary-only: the periodic
|
|
837
|
+
// heartbeat is stopped here, before any recovery retry runs, so a retry
|
|
838
|
+
// classifies a shutdown from the shutdown signal directly (in
|
|
839
|
+
// resolvePreBoundaryTerminal). The heartbeat timer may set `cancelled` before
|
|
840
|
+
// the AbortSignal microtask propagates; the direct signal check catches that.
|
|
946
841
|
const isShutdown = periodicHeartbeat.workerShutdown || (shutdownSignal?.aborted ?? false);
|
|
947
842
|
if (isShutdown) {
|
|
948
|
-
pauseDetected = false;
|
|
843
|
+
turnState.pauseDetected = false;
|
|
949
844
|
}
|
|
950
845
|
else if (periodicHeartbeat.cancelled) {
|
|
951
|
-
pauseDetected = true;
|
|
846
|
+
turnState.pauseDetected = true;
|
|
952
847
|
}
|
|
953
848
|
workerShutdownDetected = isShutdown;
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
//
|
|
957
|
-
//
|
|
958
|
-
//
|
|
959
|
-
//
|
|
960
|
-
|
|
961
|
-
accumulator.
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
const { recordTurnMetrics } = await import("../../otel.js");
|
|
982
|
-
const turnDurationMs = Date.now() - (status.startedAt ? new Date(status.startedAt).getTime() : Date.now());
|
|
983
|
-
await recordTurnMetrics({
|
|
984
|
-
durationMs: turnDurationMs,
|
|
985
|
-
inputTokens: Number(usageSnapshot.inputTokens),
|
|
986
|
-
outputTokens: Number(usageSnapshot.outputTokens),
|
|
987
|
-
model: validatedModel,
|
|
988
|
-
mode: agentMode,
|
|
989
|
-
});
|
|
990
|
-
}
|
|
991
|
-
catch {
|
|
992
|
-
// Metrics not initialized — silently skip.
|
|
993
|
-
}
|
|
994
|
-
// Phase 11a: Handle stall, worker shutdown, pause, or infrastructure cancellation.
|
|
995
|
-
// Stall: the watchdog cancelled a turn that made no progress for longer
|
|
996
|
-
// than config.cursorStreamStallTimeoutMs (a wedged tool call or a dead
|
|
997
|
-
// model connection). The keep-alive heartbeat proves liveness, so Temporal
|
|
998
|
-
// never reaps this on its own — this branch is the only clean exit. We
|
|
999
|
-
// RETURN (not throw): re-running the identical prompt via Temporal retry
|
|
1000
|
-
// would very likely wedge again. accumulator.finalize() above already
|
|
1001
|
-
// cleared isStreaming, so the UI spinners stop.
|
|
1002
|
-
if (stallDetected) {
|
|
1003
|
-
const err = stallError ?? new StallTimeoutError(config.cursorStreamStallTimeoutMs);
|
|
1004
|
-
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
1005
|
-
status.error = formatStallFailure(err);
|
|
1006
|
-
status.completedAt = utcTimestamp();
|
|
1007
|
-
status.messages.push(create(AgentMessageSchema, {
|
|
1008
|
-
type: MessageType.MESSAGE_SYSTEM,
|
|
1009
|
-
content: `Execution failed: the agent made no progress for too long and was stopped (${err.message}). You can retry or resume.`,
|
|
1010
|
-
timestamp: utcTimestamp(),
|
|
1011
|
-
}));
|
|
1012
|
-
await persist(status);
|
|
1013
|
-
console.warn(`ExecuteCursor stalled: execution=${executionId}, events=${eventCount}, error=${status.error}`);
|
|
1014
|
-
return slimStatus(status);
|
|
1015
|
-
}
|
|
1016
|
-
// Worker shutdown: the runner-manager aborted the shutdown signal before
|
|
1017
|
-
// calling worker.shutdown(). This is NOT a user-initiated pause — it's
|
|
1018
|
-
// an infrastructure event (e.g., premature removal from UI race).
|
|
1019
|
-
if (workerShutdownDetected) {
|
|
1020
|
-
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
1021
|
-
status.error = "Execution interrupted: runner worker was shut down. Retry or resume.";
|
|
1022
|
-
status.completedAt = utcTimestamp();
|
|
1023
|
-
status.messages.push(create(AgentMessageSchema, {
|
|
1024
|
-
type: MessageType.MESSAGE_SYSTEM,
|
|
1025
|
-
content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
|
|
1026
|
-
timestamp: utcTimestamp(),
|
|
1027
|
-
}));
|
|
1028
|
-
await persist(status);
|
|
1029
|
-
console.log(`ExecuteCursor interrupted (worker shutdown): execution=${executionId}, events=${eventCount}`);
|
|
1030
|
-
throw new CancelledFailure("Activity cancelled (worker shutdown, not user pause)");
|
|
1031
|
-
}
|
|
1032
|
-
// pauseDetected is only true if a heartbeat() call threw CancelledFailure,
|
|
1033
|
-
// confirming the orchestrator explicitly requested a pause.
|
|
1034
|
-
if (pauseDetected) {
|
|
1035
|
-
status.phase = ExecutionPhase.EXECUTION_PAUSED;
|
|
1036
|
-
status.messages.push(create(AgentMessageSchema, {
|
|
1037
|
-
type: MessageType.MESSAGE_SYSTEM,
|
|
1038
|
-
content: "Execution paused by user. Use resume to continue.",
|
|
1039
|
-
timestamp: utcTimestamp(),
|
|
1040
|
-
}));
|
|
1041
|
-
await persist(status);
|
|
1042
|
-
console.log(`ExecuteCursor paused: execution=${executionId}, events=${eventCount}`);
|
|
1043
|
-
throw new CancelledFailure("Activity paused by orchestrator");
|
|
1044
|
-
}
|
|
1045
|
-
// If cancellation arrived without pauseDetected (e.g. heartbeat timeout
|
|
1046
|
-
// that slipped past the periodic heartbeat, or worker shutdown), report
|
|
1047
|
-
// as failed rather than misleadingly labeling it as user-paused.
|
|
1048
|
-
if (Context.current().cancellationSignal.aborted) {
|
|
1049
|
-
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
1050
|
-
status.error = "Execution interrupted: agent was unresponsive (heartbeat timeout). Retry or resume.";
|
|
1051
|
-
status.completedAt = utcTimestamp();
|
|
1052
|
-
status.messages.push(create(AgentMessageSchema, {
|
|
1053
|
-
type: MessageType.MESSAGE_SYSTEM,
|
|
1054
|
-
content: "Execution interrupted: the agent was unresponsive for too long. You can retry or resume.",
|
|
1055
|
-
timestamp: utcTimestamp(),
|
|
1056
|
-
}));
|
|
1057
|
-
await persist(status);
|
|
1058
|
-
console.log(`ExecuteCursor interrupted (infrastructure cancel): execution=${executionId}, events=${eventCount}`);
|
|
1059
|
-
throw new CancelledFailure("Activity cancelled (heartbeat timeout, not user pause)");
|
|
1060
|
-
}
|
|
1061
|
-
// Phase 11b: Handle platform stop signal early exit
|
|
1062
|
-
if (platformStopSignaled) {
|
|
1063
|
-
status.phase = ExecutionPhase.EXECUTION_COMPLETED;
|
|
1064
|
-
status.completedAt = utcTimestamp();
|
|
1065
|
-
status.messages.push(create(AgentMessageSchema, {
|
|
1066
|
-
type: MessageType.MESSAGE_SYSTEM,
|
|
1067
|
-
content: "Execution stopped by the platform.",
|
|
1068
|
-
timestamp: utcTimestamp(),
|
|
1069
|
-
}));
|
|
849
|
+
// Post-stream finalize, shared by the primary turn and both recovery retries:
|
|
850
|
+
// finalize the transcript + streaming flags, mark any in-flight sub-agent
|
|
851
|
+
// CANCELLED on an aborted turn, snapshot usage, flush the recorder, and
|
|
852
|
+
// persist so the UI sees the settled rows. The unified loop applies delta
|
|
853
|
+
// enrichments per-iteration, so — unlike the old bare retry path — no
|
|
854
|
+
// compensating applyEnrichments() is needed here.
|
|
855
|
+
const finalizeStreamPhase = async () => {
|
|
856
|
+
accumulator.finalize();
|
|
857
|
+
deltaEnricher.finalize(status.messages);
|
|
858
|
+
// A pause / cancel / worker shutdown aborts the Cursor SDK run, so any
|
|
859
|
+
// sub-agent the parent had delegated is no longer executing. Mark it
|
|
860
|
+
// CANCELLED rather than leaving a permanent IN_PROGRESS "zombie" in the
|
|
861
|
+
// final snapshot (parity with the native harness's cancelSubAgents()).
|
|
862
|
+
if (turnState.pauseDetected ||
|
|
863
|
+
workerShutdownDetected ||
|
|
864
|
+
turnState.stallDetected ||
|
|
865
|
+
Context.current().cancellationSignal.aborted) {
|
|
866
|
+
accumulator.cancelInProgressSubAgents();
|
|
867
|
+
}
|
|
868
|
+
status.subAgentExecutions = accumulator.subAgentExecutions;
|
|
869
|
+
await eventRecorder?.flush();
|
|
870
|
+
if (usageAccumulator.hasTurns) {
|
|
871
|
+
status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
|
|
872
|
+
}
|
|
873
|
+
console.log(`ExecuteCursor stream ended: execution=${executionId}, events=${turnState.eventCount}, messages=${status.messages.length}, subAgents=${status.subAgentExecutions.length}`);
|
|
874
|
+
// Persist immediately after finalize so the UI sees correct tool-call
|
|
875
|
+
// statuses before the boundary / run.wait() / structured-output extraction.
|
|
1070
876
|
await persist(status);
|
|
1071
|
-
|
|
1072
|
-
|
|
877
|
+
heartbeat();
|
|
878
|
+
};
|
|
879
|
+
const resolvePreBoundaryTerminal = async () => {
|
|
880
|
+
// Stall: the watchdog cancelled a turn that made no progress. RETURN (not
|
|
881
|
+
// throw): re-running the identical prompt via Temporal retry would very
|
|
882
|
+
// likely wedge again.
|
|
883
|
+
if (turnState.stallDetected) {
|
|
884
|
+
const err = turnState.stallError ?? new StallTimeoutError(config.cursorStreamStallTimeoutMs);
|
|
885
|
+
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
886
|
+
status.error = formatStallFailure(err);
|
|
887
|
+
status.completedAt = utcTimestamp();
|
|
888
|
+
status.messages.push(create(AgentMessageSchema, {
|
|
889
|
+
type: MessageType.MESSAGE_SYSTEM,
|
|
890
|
+
content: `Execution failed: the agent made no progress for too long and was stopped (${err.message}). You can retry or resume.`,
|
|
891
|
+
timestamp: utcTimestamp(),
|
|
892
|
+
}));
|
|
893
|
+
await persist(status);
|
|
894
|
+
console.warn(`ExecuteCursor stalled: execution=${executionId}, events=${turnState.eventCount}, error=${status.error}`);
|
|
895
|
+
return { kind: "return" };
|
|
896
|
+
}
|
|
897
|
+
// Worker shutdown: the runner-manager aborted the shutdown signal. NOT a
|
|
898
|
+
// user pause. Checked via the shutdown signal directly so a retry (whose
|
|
899
|
+
// periodic heartbeat is already stopped) still classifies it correctly.
|
|
900
|
+
if (workerShutdownDetected || (shutdownSignal?.aborted ?? false)) {
|
|
901
|
+
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
902
|
+
status.error = "Execution interrupted: runner worker was shut down. Retry or resume.";
|
|
903
|
+
status.completedAt = utcTimestamp();
|
|
904
|
+
status.messages.push(create(AgentMessageSchema, {
|
|
905
|
+
type: MessageType.MESSAGE_SYSTEM,
|
|
906
|
+
content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
|
|
907
|
+
timestamp: utcTimestamp(),
|
|
908
|
+
}));
|
|
909
|
+
await persist(status);
|
|
910
|
+
console.log(`ExecuteCursor interrupted (worker shutdown): execution=${executionId}, events=${turnState.eventCount}`);
|
|
911
|
+
return { kind: "throw", message: "Activity cancelled (worker shutdown, not user pause)" };
|
|
912
|
+
}
|
|
913
|
+
// pauseDetected is only true if a heartbeat() call threw CancelledFailure,
|
|
914
|
+
// confirming the orchestrator explicitly requested a pause.
|
|
915
|
+
if (turnState.pauseDetected) {
|
|
916
|
+
status.phase = ExecutionPhase.EXECUTION_PAUSED;
|
|
917
|
+
status.messages.push(create(AgentMessageSchema, {
|
|
918
|
+
type: MessageType.MESSAGE_SYSTEM,
|
|
919
|
+
content: "Execution paused by user. Use resume to continue.",
|
|
920
|
+
timestamp: utcTimestamp(),
|
|
921
|
+
}));
|
|
922
|
+
await persist(status);
|
|
923
|
+
console.log(`ExecuteCursor paused: execution=${executionId}, events=${turnState.eventCount}`);
|
|
924
|
+
return { kind: "throw", message: "Activity paused by orchestrator" };
|
|
925
|
+
}
|
|
926
|
+
// Cancellation without pauseDetected (e.g. heartbeat timeout): report as
|
|
927
|
+
// failed rather than misleadingly labeling it a user pause.
|
|
928
|
+
if (Context.current().cancellationSignal.aborted) {
|
|
929
|
+
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
930
|
+
status.error = "Execution interrupted: agent was unresponsive (heartbeat timeout). Retry or resume.";
|
|
931
|
+
status.completedAt = utcTimestamp();
|
|
932
|
+
status.messages.push(create(AgentMessageSchema, {
|
|
933
|
+
type: MessageType.MESSAGE_SYSTEM,
|
|
934
|
+
content: "Execution interrupted: the agent was unresponsive for too long. You can retry or resume.",
|
|
935
|
+
timestamp: utcTimestamp(),
|
|
936
|
+
}));
|
|
937
|
+
await persist(status);
|
|
938
|
+
console.log(`ExecuteCursor interrupted (infrastructure cancel): execution=${executionId}, events=${turnState.eventCount}`);
|
|
939
|
+
return { kind: "throw", message: "Activity cancelled (heartbeat timeout, not user pause)" };
|
|
1073
940
|
}
|
|
1074
|
-
|
|
1075
|
-
|
|
941
|
+
// Platform stop signal: a clean COMPLETED early exit.
|
|
942
|
+
if (turnState.platformStopSignaled) {
|
|
943
|
+
status.phase = ExecutionPhase.EXECUTION_COMPLETED;
|
|
944
|
+
status.completedAt = utcTimestamp();
|
|
945
|
+
status.messages.push(create(AgentMessageSchema, {
|
|
946
|
+
type: MessageType.MESSAGE_SYSTEM,
|
|
947
|
+
content: "Execution stopped by the platform.",
|
|
948
|
+
timestamp: utcTimestamp(),
|
|
949
|
+
}));
|
|
950
|
+
await persist(status);
|
|
951
|
+
try {
|
|
952
|
+
resolution.agent.close();
|
|
953
|
+
}
|
|
954
|
+
catch { /* best effort */ }
|
|
955
|
+
console.log(`ExecuteCursor completed (platform stop): execution=${executionId}`);
|
|
956
|
+
return { kind: "return" };
|
|
957
|
+
}
|
|
958
|
+
return { kind: "proceed" };
|
|
959
|
+
};
|
|
960
|
+
await finalizeStreamPhase();
|
|
961
|
+
const primaryTerminal = await resolvePreBoundaryTerminal();
|
|
962
|
+
if (primaryTerminal.kind === "return")
|
|
1076
963
|
return slimStatus(status);
|
|
1077
|
-
|
|
964
|
+
if (primaryTerminal.kind === "throw")
|
|
965
|
+
throw new CancelledFailure(primaryTerminal.message);
|
|
1078
966
|
// Phase 12: The turn boundary — author this turn's change set to the
|
|
1079
967
|
// file_review ledger (CANDIDATE_CAPTURED) and overlay the hook's denials as
|
|
1080
968
|
// WAITING_APPROVAL gate rows. The full pipeline and its ordering rationale
|
|
@@ -1110,43 +998,49 @@ turnSeq) {
|
|
|
1110
998
|
`${boundary.capturedChangeCount} file card(s) pending`);
|
|
1111
999
|
return slimStatus(status);
|
|
1112
1000
|
};
|
|
1113
|
-
//
|
|
1114
|
-
//
|
|
1115
|
-
//
|
|
1116
|
-
//
|
|
1117
|
-
//
|
|
1118
|
-
//
|
|
1119
|
-
//
|
|
1120
|
-
//
|
|
1121
|
-
//
|
|
1122
|
-
// and re-stamp completedAt (stamped below BEFORE the retry ran);
|
|
1123
|
-
// - re-enter the turn boundary so edits made by the retry reach the
|
|
1124
|
-
// file_review ledger / approval gates — without this, a retry's file
|
|
1125
|
-
// edits silently escape review (production case
|
|
1126
|
-
// aex_01kws27q1e2esvkqjpvectttxf).
|
|
1127
|
-
// Returns undefined for a cancelled retry: parity with the primary path,
|
|
1128
|
-
// where cancellation exits before the boundary — there is no review to open.
|
|
1001
|
+
// Re-enter the turn boundary for a recovery retry: author the retry's net
|
|
1002
|
+
// change set to the file_review ledger and overlay any denials as gates —
|
|
1003
|
+
// without this a retry's file edits silently escape review (production case
|
|
1004
|
+
// aex_01kws27q1e2esvkqjpvectttxf). The stream finalize now runs through the
|
|
1005
|
+
// shared finalizeStreamPhase (in runRecoveryStream), so this is only the
|
|
1006
|
+
// boundary + completedAt. Returns undefined for a cancelled retry — there is
|
|
1007
|
+
// no review to open. Passes denialCancelSettled so a first denial that stopped
|
|
1008
|
+
// the RETRY waits for run.cancel() before the ledger read, exactly like the
|
|
1009
|
+
// primary path.
|
|
1129
1010
|
const settleRetryTurn = async (retryResultStatus) => {
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
status.subAgentExecutions = accumulator.subAgentExecutions;
|
|
1134
|
-
if (usageAccumulator.hasTurns) {
|
|
1135
|
-
status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
|
|
1136
|
-
}
|
|
1137
|
-
// Re-flush the (dev-only) event recorder: flush rewrites the full JSONL,
|
|
1138
|
-
// so the recorded trace now includes the retry's events too.
|
|
1139
|
-
await eventRecorder?.flush();
|
|
1140
|
-
const retryBoundary = retryResultStatus === "cancelled" ? undefined : await runBoundary();
|
|
1011
|
+
const retryBoundary = retryResultStatus === "cancelled"
|
|
1012
|
+
? undefined
|
|
1013
|
+
: await runBoundary(turnState.firstDenialDetected ? turnState.denialCancelSettled : undefined);
|
|
1141
1014
|
// Phase 13 stamped completedAt BEFORE the retry ran. A terminal outcome
|
|
1142
1015
|
// re-stamps it to the true end; a review pause CLEARS it — the primary
|
|
1143
1016
|
// pause path never stamps it (a waiting turn is not complete).
|
|
1144
1017
|
status.completedAt = retryBoundary?.waiting ? "" : utcTimestamp();
|
|
1145
1018
|
return retryBoundary;
|
|
1146
1019
|
};
|
|
1020
|
+
const runRecoveryStream = async (freshAgent, retryPrompt) => {
|
|
1021
|
+
// The fresh agent is now the live handle: point resolution at it so the
|
|
1022
|
+
// terminal close() (platform stop, or Phase 14 success) frees THIS agent's
|
|
1023
|
+
// executor lease rather than the disposed one it replaced. (Without this the
|
|
1024
|
+
// poisoned-handle path leaked the fresh agent — it closed the stale one.)
|
|
1025
|
+
resolution = { ...resolution, agent: freshAgent, agentId: freshAgent.agentId, isNew: true };
|
|
1026
|
+
turnState.streamErrorMessage = undefined;
|
|
1027
|
+
const retryRun = await freshAgent.send(retryPrompt, {
|
|
1028
|
+
onDelta: makeCursorTurnOnDelta(onDeltaDeps),
|
|
1029
|
+
});
|
|
1030
|
+
await consumeCursorTurnStream(retryRun, streamDeps);
|
|
1031
|
+
await finalizeStreamPhase();
|
|
1032
|
+
const terminal = await resolvePreBoundaryTerminal();
|
|
1033
|
+
if (terminal.kind !== "proceed")
|
|
1034
|
+
return { proceeded: false, terminal };
|
|
1035
|
+
const retryResult = await retryRun.wait();
|
|
1036
|
+
console.log(`ExecuteCursor retry run.wait(): execution=${executionId}, ` +
|
|
1037
|
+
`retryResult=${JSON.stringify(retryResult)}`);
|
|
1038
|
+
const retryBoundary = await settleRetryTurn(retryResult.status);
|
|
1039
|
+
return { proceeded: true, retryRun, retryResult, retryBoundary };
|
|
1040
|
+
};
|
|
1147
1041
|
// The denial-settle wait applies only when a first denial stopped THIS run;
|
|
1148
|
-
//
|
|
1149
|
-
const boundary = await runBoundary(firstDenialDetected ? denialCancelSettled : undefined);
|
|
1042
|
+
// a normal completion passes no promise.
|
|
1043
|
+
const boundary = await runBoundary(turnState.firstDenialDetected ? turnState.denialCancelSettled : undefined);
|
|
1150
1044
|
if (boundary.waiting) {
|
|
1151
1045
|
return enterApprovalPause(boundary);
|
|
1152
1046
|
}
|
|
@@ -1179,7 +1073,7 @@ turnSeq) {
|
|
|
1179
1073
|
clearCapturedRejection(executionId);
|
|
1180
1074
|
const classified = synthesizeError({
|
|
1181
1075
|
sdkResultFields: sdkErrorStr,
|
|
1182
|
-
streamErrorMessage,
|
|
1076
|
+
streamErrorMessage: turnState.streamErrorMessage,
|
|
1183
1077
|
capturedRejection,
|
|
1184
1078
|
conversationErrorText,
|
|
1185
1079
|
isResumedHandle: resolution.reason === "resumed_successfully",
|
|
@@ -1234,53 +1128,13 @@ turnSeq) {
|
|
|
1234
1128
|
catch (updateErr) {
|
|
1235
1129
|
console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
|
|
1236
1130
|
}
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
usageAccumulator.addTurn(update.usage);
|
|
1243
|
-
}
|
|
1244
|
-
deltaEnricher.processDelta(update);
|
|
1245
|
-
try {
|
|
1246
|
-
heartbeat();
|
|
1247
|
-
}
|
|
1248
|
-
catch { /* swallow during retry */ }
|
|
1249
|
-
},
|
|
1250
|
-
});
|
|
1251
|
-
// Mirror the primary stream's stall protection: a wedged retry must
|
|
1252
|
-
// not hang the activity. On stall, cancel the run so the loop ends and
|
|
1253
|
-
// retryRun.wait() resolves down the existing non-finished failure path.
|
|
1254
|
-
retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
1255
|
-
console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
|
|
1256
|
-
if (retryRun.supports?.("cancel"))
|
|
1257
|
-
void retryRun.cancel().catch(() => { });
|
|
1258
|
-
});
|
|
1259
|
-
streamErrorMessage = undefined;
|
|
1260
|
-
try {
|
|
1261
|
-
for await (const retryEvent of retryRun.stream()) {
|
|
1262
|
-
if (Context.current().cancellationSignal.aborted)
|
|
1263
|
-
break;
|
|
1264
|
-
retryWatchdog.recordActivity();
|
|
1265
|
-
eventRecorder?.record(retryEvent, eventCount);
|
|
1266
|
-
accumulator.processEvent(retryEvent);
|
|
1267
|
-
eventCount++;
|
|
1268
|
-
if (retryEvent.type === "status") {
|
|
1269
|
-
const retryStatusEvent = retryEvent;
|
|
1270
|
-
if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
|
|
1271
|
-
streamErrorMessage = retryStatusEvent.message;
|
|
1272
|
-
}
|
|
1273
|
-
}
|
|
1274
|
-
heartbeat();
|
|
1275
|
-
}
|
|
1276
|
-
}
|
|
1277
|
-
finally {
|
|
1278
|
-
retryWatchdog.stop();
|
|
1131
|
+
const outcome = await runRecoveryStream(freshAgent, freshPrompt);
|
|
1132
|
+
if (!outcome.proceeded) {
|
|
1133
|
+
if (outcome.terminal.kind === "return")
|
|
1134
|
+
return slimStatus(status);
|
|
1135
|
+
throw new CancelledFailure(outcome.terminal.message);
|
|
1279
1136
|
}
|
|
1280
|
-
const retryResult =
|
|
1281
|
-
console.log(`ExecuteCursor retry run.wait(): execution=${executionId}, ` +
|
|
1282
|
-
`retryResult=${JSON.stringify(retryResult)}`);
|
|
1283
|
-
const retryBoundary = await settleRetryTurn(retryResult.status);
|
|
1137
|
+
const { retryRun, retryResult, retryBoundary } = outcome;
|
|
1284
1138
|
if (retryBoundary?.waiting) {
|
|
1285
1139
|
// The retry's edits/denials armed the gate — pause for review. On a
|
|
1286
1140
|
// retry error this supersedes the failure, exactly as on the primary
|
|
@@ -1304,7 +1158,7 @@ turnSeq) {
|
|
|
1304
1158
|
const retryConversationErrorText = await introspectConversation(retryRun, executionId);
|
|
1305
1159
|
const retryClassified = synthesizeError({
|
|
1306
1160
|
sdkResultFields: retryResult.result ? String(retryResult.result) : undefined,
|
|
1307
|
-
streamErrorMessage,
|
|
1161
|
+
streamErrorMessage: turnState.streamErrorMessage,
|
|
1308
1162
|
capturedRejection: retryRejection,
|
|
1309
1163
|
conversationErrorText: retryConversationErrorText,
|
|
1310
1164
|
isResumedHandle: false,
|
|
@@ -1342,61 +1196,21 @@ turnSeq) {
|
|
|
1342
1196
|
catch (updateErr) {
|
|
1343
1197
|
console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
|
|
1344
1198
|
}
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
usageAccumulator.addTurn(update.usage);
|
|
1351
|
-
}
|
|
1352
|
-
deltaEnricher.processDelta(update);
|
|
1353
|
-
try {
|
|
1354
|
-
heartbeat();
|
|
1355
|
-
}
|
|
1356
|
-
catch { /* swallow during retry */ }
|
|
1357
|
-
},
|
|
1358
|
-
});
|
|
1359
|
-
// Mirror the primary stream's stall protection: a wedged retry must
|
|
1360
|
-
// not hang the activity. On stall, cancel the run so the loop ends and
|
|
1361
|
-
// retryRun.wait() resolves down the existing non-finished failure path.
|
|
1362
|
-
retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
1363
|
-
console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
|
|
1364
|
-
if (retryRun.supports?.("cancel"))
|
|
1365
|
-
void retryRun.cancel().catch(() => { });
|
|
1366
|
-
});
|
|
1367
|
-
streamErrorMessage = undefined;
|
|
1368
|
-
try {
|
|
1369
|
-
for await (const retryEvent of retryRun.stream()) {
|
|
1370
|
-
if (Context.current().cancellationSignal.aborted)
|
|
1371
|
-
break;
|
|
1372
|
-
retryWatchdog.recordActivity();
|
|
1373
|
-
eventRecorder?.record(retryEvent, eventCount);
|
|
1374
|
-
accumulator.processEvent(retryEvent);
|
|
1375
|
-
eventCount++;
|
|
1376
|
-
if (retryEvent.type === "status") {
|
|
1377
|
-
const retryStatusEvent = retryEvent;
|
|
1378
|
-
if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
|
|
1379
|
-
streamErrorMessage = retryStatusEvent.message;
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
heartbeat();
|
|
1383
|
-
}
|
|
1384
|
-
}
|
|
1385
|
-
finally {
|
|
1386
|
-
retryWatchdog.stop();
|
|
1199
|
+
const outcome = await runRecoveryStream(freshAgent, effectivePrompt);
|
|
1200
|
+
if (!outcome.proceeded) {
|
|
1201
|
+
if (outcome.terminal.kind === "return")
|
|
1202
|
+
return slimStatus(status);
|
|
1203
|
+
throw new CancelledFailure(outcome.terminal.message);
|
|
1387
1204
|
}
|
|
1388
|
-
const retryResult =
|
|
1389
|
-
const retryBoundary = await settleRetryTurn(retryResult.status);
|
|
1205
|
+
const { retryResult, retryBoundary } = outcome;
|
|
1390
1206
|
if (retryBoundary?.waiting) {
|
|
1391
1207
|
// The retry's edits/denials armed the gate — pause for review (see
|
|
1392
1208
|
// the poisoned-handle branch above for the precedence rationale).
|
|
1393
1209
|
console.log(`ExecuteCursor transport-timeout recovery paused for review: execution=${executionId}`);
|
|
1394
|
-
resolution = { ...resolution, agent: freshAgent, agentId: freshAgent.agentId, isNew: true };
|
|
1395
1210
|
return enterApprovalPause(retryBoundary);
|
|
1396
1211
|
}
|
|
1397
1212
|
if (retryResult.status === "finished") {
|
|
1398
1213
|
status.phase = ExecutionPhase.EXECUTION_COMPLETED;
|
|
1399
|
-
resolution = { ...resolution, agent: freshAgent, agentId: freshAgent.agentId, isNew: true };
|
|
1400
1214
|
break;
|
|
1401
1215
|
}
|
|
1402
1216
|
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
@@ -1515,7 +1329,7 @@ turnSeq) {
|
|
|
1515
1329
|
timestamp: utcTimestamp(),
|
|
1516
1330
|
}));
|
|
1517
1331
|
}
|
|
1518
|
-
else if (pauseDetected) {
|
|
1332
|
+
else if (turnState.pauseDetected) {
|
|
1519
1333
|
console.log(`ExecuteCursor cancelled (pause) for execution ${executionId}`);
|
|
1520
1334
|
status.phase = ExecutionPhase.EXECUTION_PAUSED;
|
|
1521
1335
|
status.messages.push(create(AgentMessageSchema, {
|
|
@@ -1545,7 +1359,7 @@ turnSeq) {
|
|
|
1545
1359
|
// treat the execution as paused rather than failed. The error was likely
|
|
1546
1360
|
// caused by the cancellation (e.g. SDK stream teardown) and should not
|
|
1547
1361
|
// overwrite the PAUSED state that the Pause RPC already set in the DB.
|
|
1548
|
-
if (pauseDetected) {
|
|
1362
|
+
if (turnState.pauseDetected) {
|
|
1549
1363
|
const errDetail = err instanceof Error ? err.message : String(err);
|
|
1550
1364
|
console.log(`ExecuteCursor error during pause (treating as pause): execution=${executionId}, error=${errDetail}`);
|
|
1551
1365
|
status.phase = ExecutionPhase.EXECUTION_PAUSED;
|
|
@@ -1636,10 +1450,15 @@ turnSeq) {
|
|
|
1636
1450
|
return slimStatus(status);
|
|
1637
1451
|
}
|
|
1638
1452
|
finally {
|
|
1639
|
-
//
|
|
1640
|
-
// path
|
|
1641
|
-
//
|
|
1642
|
-
|
|
1453
|
+
// End the OTel turn span + record metrics with the final token snapshot on
|
|
1454
|
+
// EVERY exit path (idempotent). Placed here so the span covers any recovery
|
|
1455
|
+
// retry (whose tokens accrue after the primary stream) and never leaks on an
|
|
1456
|
+
// early return or throw. A no-op when OTel is off or the span never opened.
|
|
1457
|
+
await finishTurnTelemetry?.();
|
|
1458
|
+
// Disarm the stall watchdog on EVERY exit path (idempotent). consumeCursorTurnStream
|
|
1459
|
+
// stops the one it armed; this covers throws before that point so no orphaned
|
|
1460
|
+
// timer survives the activity.
|
|
1461
|
+
turnState.stallWatchdog?.stop();
|
|
1643
1462
|
// Close the denial-ledger watcher on EVERY exit path (idempotent) so no
|
|
1644
1463
|
// orphaned fs.watch handle survives the activity.
|
|
1645
1464
|
stopDenialWatcher?.();
|