@stigmer/runner 3.0.8-dev.20260613085218 → 3.0.9-dev.20260615145121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/activities/execute-cursor/hook-script.d.ts +23 -12
- package/dist/activities/execute-cursor/hook-script.js +85 -51
- package/dist/activities/execute-cursor/hook-script.js.map +1 -1
- package/dist/activities/execute-cursor/index.js +220 -79
- package/dist/activities/execute-cursor/index.js.map +1 -1
- package/dist/activities/execute-cursor/message-translator.d.ts +51 -9
- package/dist/activities/execute-cursor/message-translator.js +146 -20
- package/dist/activities/execute-cursor/message-translator.js.map +1 -1
- package/dist/activities/execute-cursor/persist-decision.d.ts +42 -0
- package/dist/activities/execute-cursor/persist-decision.js +30 -0
- package/dist/activities/execute-cursor/persist-decision.js.map +1 -0
- package/dist/activities/execute-cursor/prompt-builder.d.ts +25 -0
- package/dist/activities/execute-cursor/prompt-builder.js +54 -0
- package/dist/activities/execute-cursor/prompt-builder.js.map +1 -1
- package/dist/activities/execute-cursor/workspace-setup.d.ts +8 -2
- package/dist/activities/execute-cursor/workspace-setup.js +62 -30
- package/dist/activities/execute-cursor/workspace-setup.js.map +1 -1
- package/dist/activities/execute-deep-agent/index.js +15 -5
- package/dist/activities/execute-deep-agent/index.js.map +1 -1
- package/dist/activities/execute-deep-agent/status-builder-shared.d.ts +0 -1
- package/dist/activities/execute-deep-agent/status-builder-shared.js +32 -8
- package/dist/activities/execute-deep-agent/status-builder-shared.js.map +1 -1
- package/dist/activities/execute-deep-agent/status-builder.js +4 -5
- package/dist/activities/execute-deep-agent/status-builder.js.map +1 -1
- package/dist/activities/execute-deep-agent/streaming-v3.js +4 -5
- package/dist/activities/execute-deep-agent/streaming-v3.js.map +1 -1
- package/dist/activities/execute-deep-agent/streaming.d.ts +9 -1
- package/dist/activities/execute-deep-agent/streaming.js +4 -5
- package/dist/activities/execute-deep-agent/streaming.js.map +1 -1
- package/dist/activities/execute-deep-agent/subagent-tracker.js +4 -5
- package/dist/activities/execute-deep-agent/subagent-tracker.js.map +1 -1
- package/dist/activities/execute-deep-agent/v3-status-builder.js +6 -5
- package/dist/activities/execute-deep-agent/v3-status-builder.js.map +1 -1
- package/dist/config.d.ts +21 -0
- package/dist/config.js +12 -0
- package/dist/config.js.map +1 -1
- package/dist/in-flight.d.ts +35 -0
- package/dist/in-flight.js +61 -0
- package/dist/in-flight.js.map +1 -0
- package/dist/runner-manager.d.ts +2 -0
- package/dist/runner-manager.js +90 -29
- package/dist/runner-manager.js.map +1 -1
- package/dist/runner.d.ts +2 -0
- package/dist/runner.js +2 -0
- package/dist/runner.js.map +1 -1
- package/dist/shared/grpc-retry.d.ts +9 -20
- package/dist/shared/grpc-retry.js +9 -52
- package/dist/shared/grpc-retry.js.map +1 -1
- package/dist/shared/stall-watchdog.d.ts +68 -0
- package/dist/shared/stall-watchdog.js +102 -0
- package/dist/shared/stall-watchdog.js.map +1 -0
- package/dist/shared/status-offload.d.ts +84 -0
- package/dist/shared/status-offload.js +292 -0
- package/dist/shared/status-offload.js.map +1 -0
- package/dist/shared/status.d.ts +34 -3
- package/dist/shared/status.js +102 -9
- package/dist/shared/status.js.map +1 -1
- package/dist/{activities/execute-deep-agent → shared}/streaming-scheduler.d.ts +4 -0
- package/dist/{activities/execute-deep-agent → shared}/streaming-scheduler.js +4 -0
- package/dist/shared/streaming-scheduler.js.map +1 -0
- package/package.json +2 -2
- package/src/__tests__/config.test.ts +8 -0
- package/src/__tests__/in-flight.test.ts +84 -0
- package/src/activities/__tests__/classify-tool-approvals.test.ts +1 -0
- package/src/activities/__tests__/discover-mcp-server.test.ts +1 -0
- package/src/activities/execute-cursor/__tests__/build-prompt.test.ts +74 -0
- package/src/activities/execute-cursor/__tests__/hook-script.test.ts +90 -15
- package/src/activities/execute-cursor/__tests__/message-translator.test.ts +124 -12
- package/src/activities/execute-cursor/__tests__/persist-decision.test.ts +99 -0
- package/src/activities/execute-cursor/__tests__/tool-result-image.test.ts +244 -0
- package/src/activities/execute-cursor/__tests__/workspace-setup.test.ts +53 -4
- package/src/activities/execute-cursor/hook-script.ts +85 -51
- package/src/activities/execute-cursor/index.ts +187 -38
- package/src/activities/execute-cursor/message-translator.ts +146 -20
- package/src/activities/execute-cursor/persist-decision.ts +54 -0
- package/src/activities/execute-cursor/prompt-builder.ts +59 -0
- package/src/activities/execute-cursor/workspace-setup.ts +76 -44
- package/src/activities/execute-deep-agent/__tests__/index.test.ts +1 -0
- package/src/activities/execute-deep-agent/__tests__/status-builder-shared.test.ts +66 -0
- package/src/activities/execute-deep-agent/__tests__/status-builder.test.ts +6 -3
- package/src/activities/execute-deep-agent/__tests__/streaming-v3.test.ts +70 -0
- package/src/activities/execute-deep-agent/index.ts +17 -5
- package/src/activities/execute-deep-agent/status-builder-shared.ts +27 -5
- package/src/activities/execute-deep-agent/status-builder.ts +3 -5
- package/src/activities/execute-deep-agent/streaming-v3.ts +5 -5
- package/src/activities/execute-deep-agent/streaming.ts +14 -5
- package/src/activities/execute-deep-agent/subagent-tracker.ts +4 -5
- package/src/activities/execute-deep-agent/v3-status-builder.ts +5 -5
- package/src/config.ts +27 -0
- package/src/in-flight.ts +71 -0
- package/src/runner-manager.ts +127 -33
- package/src/runner.ts +6 -0
- package/src/shared/__tests__/artifact-storage.test.ts +1 -0
- package/src/shared/__tests__/grpc-retry-extended.test.ts +6 -144
- package/src/shared/__tests__/grpc-retry.test.ts +5 -134
- package/src/shared/__tests__/stall-watchdog.test.ts +193 -0
- package/src/shared/__tests__/status-offload.test.ts +256 -0
- package/src/shared/__tests__/status.test.ts +199 -0
- package/src/shared/grpc-retry.ts +9 -72
- package/src/shared/stall-watchdog.ts +122 -0
- package/src/shared/status-offload.ts +342 -0
- package/src/shared/status.ts +142 -8
- package/src/{activities/execute-deep-agent → shared}/streaming-scheduler.ts +4 -0
- package/dist/activities/execute-deep-agent/streaming-scheduler.js.map +0 -1
- /package/src/{activities/execute-deep-agent → shared}/__tests__/streaming-scheduler.test.ts +0 -0
|
@@ -36,10 +36,13 @@ import { CursorMode } from "@stigmer/protos/ai/stigmer/agentic/session/v1/enum_p
|
|
|
36
36
|
import { determineCursorMode, isCloudMode } from "./cursor-mode.js";
|
|
37
37
|
import { MessageAccumulator, reconcileDeniedToolCalls, cancelInProgressSubAgentProtos } from "./message-translator.js";
|
|
38
38
|
import { utcTimestamp, persistStatus, reportSetupProgress, slimStatus } from "../../shared/status.js";
|
|
39
|
+
import { startStallWatchdog, StallTimeoutError, formatStallFailure } from "../../shared/stall-watchdog.js";
|
|
39
40
|
import { createArtifactStorage, loadArtifactStorageConfig } from "../../shared/artifact-storage.js";
|
|
40
41
|
import { publishPlanArtifact } from "../../shared/plan-artifact.js";
|
|
41
42
|
import { DeltaEnricher } from "./delta-enricher.js";
|
|
42
43
|
import { TodoTracker } from "./todo-tracker.js";
|
|
44
|
+
import { shouldPersistStreamingStatus } from "./persist-decision.js";
|
|
45
|
+
import { StreamingUpdateScheduler, loadStreamingConfig } from "../../shared/streaming-scheduler.js";
|
|
43
46
|
import { createCursorEventRecorder } from "./cursor-event-recorder.js";
|
|
44
47
|
import { resolveMcpServers, validateMcpServerEnv } from "./mcp-resolver.js";
|
|
45
48
|
import { mergeApprovalPolicies } from "./approval-policy.js";
|
|
@@ -102,11 +105,40 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
102
105
|
phase: ExecutionPhase.EXECUTION_IN_PROGRESS,
|
|
103
106
|
startedAt: utcTimestamp(),
|
|
104
107
|
});
|
|
108
|
+
// Artifact storage for offloading oversized tool outputs (screenshots, giant
|
|
109
|
+
// dumps) out of the persisted status, and for publishing the plan artifact.
|
|
110
|
+
// Created once here so it is available to EVERY persist below. Best-effort:
|
|
111
|
+
// if it can't be built (e.g. proxy mode without a token), offload is disabled
|
|
112
|
+
// but persistStatus still enforces the aggregate size cap, so persistence can
|
|
113
|
+
// never silently blow past the gRPC limit.
|
|
114
|
+
let artifactStorage;
|
|
115
|
+
try {
|
|
116
|
+
artifactStorage = createArtifactStorage(loadArtifactStorageConfig(config));
|
|
117
|
+
}
|
|
118
|
+
catch (storageErr) {
|
|
119
|
+
console.warn(`ExecuteCursor artifact storage unavailable — tool-output offload disabled ` +
|
|
120
|
+
`(aggregate size guard still active): execution=${executionId}, error=${storageErr}`);
|
|
121
|
+
}
|
|
122
|
+
const statusOffload = artifactStorage
|
|
123
|
+
? { artifactStorage, executionId }
|
|
124
|
+
: undefined;
|
|
125
|
+
// ALL status persistence in this activity flows through `persist`, so the
|
|
126
|
+
// single size-bounding guard (offload + aggregate elision) is unforgeable and
|
|
127
|
+
// a future call site cannot accidentally skip it.
|
|
128
|
+
const persist = (s = status) => persistStatus(client, executionId, s, { offload: statusOffload });
|
|
105
129
|
let sessionId;
|
|
106
130
|
let session;
|
|
107
131
|
let pauseDetected = false;
|
|
108
132
|
let workerShutdownDetected = false;
|
|
133
|
+
// Set by the stall watchdog when the SDK stream makes no progress for
|
|
134
|
+
// config.cursorStreamStallTimeoutMs. stallError carries the recognizable
|
|
135
|
+
// message surfaced to the user; both feed the Phase 11a stall branch.
|
|
136
|
+
let stallDetected = false;
|
|
137
|
+
let stallError;
|
|
109
138
|
let periodicHeartbeat;
|
|
139
|
+
// Progress-based stall watchdog (see ../../shared/stall-watchdog.ts). Stopped
|
|
140
|
+
// in the finally on every exit path; complements the liveness heartbeat.
|
|
141
|
+
let stallWatchdog;
|
|
110
142
|
// Session HITL directory (runner-owned, outside the workspace) where the hook
|
|
111
143
|
// script, approval-state file, and denial ledger live. Set once the gate is
|
|
112
144
|
// installed; the WAITING_FOR_APPROVAL path reads the denial ledger from here.
|
|
@@ -186,7 +218,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
186
218
|
content: "Execution was rejected by the user during tool approval.",
|
|
187
219
|
timestamp: utcTimestamp(),
|
|
188
220
|
}));
|
|
189
|
-
await
|
|
221
|
+
await persist(status);
|
|
190
222
|
return slimStatus(status);
|
|
191
223
|
}
|
|
192
224
|
}
|
|
@@ -384,6 +416,9 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
384
416
|
let firstTurnAttributionLogged = false;
|
|
385
417
|
let streamErrorMessage;
|
|
386
418
|
let alreadyRetriedWithFreshAgent = false;
|
|
419
|
+
// Most recent tool name observed on the stream, used only to enrich the
|
|
420
|
+
// stall message ("last tool: …") so a wedged turn names the likely culprit.
|
|
421
|
+
let lastToolName;
|
|
387
422
|
// Periodic heartbeat keeps Temporal informed during silent SDK operations
|
|
388
423
|
// (e.g. long tool calls, MCP requests, model thinking). Without this,
|
|
389
424
|
// the 2-minute heartbeat timeout can cancel the activity and mislabel
|
|
@@ -411,6 +446,10 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
411
446
|
}
|
|
412
447
|
const run = await resolution.agent.send(effectivePrompt, {
|
|
413
448
|
onDelta: ({ update }) => {
|
|
449
|
+
// Reset the stall timer on the delta channel too: a long model
|
|
450
|
+
// generation emits token deltas but few discrete stream events, so
|
|
451
|
+
// resetting only in the stream loop would false-positive a stall.
|
|
452
|
+
stallWatchdog?.recordActivity();
|
|
414
453
|
if (update.type === "turn-ended" && update.usage) {
|
|
415
454
|
usageAccumulator.addTurn(update.usage);
|
|
416
455
|
if (!firstTurnAttributionLogged) {
|
|
@@ -435,57 +474,103 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
435
474
|
}
|
|
436
475
|
},
|
|
437
476
|
});
|
|
477
|
+
// Arm the stall watchdog now that the run exists. The periodic heartbeat
|
|
478
|
+
// above proves the process is alive, not that the agent is progressing: if
|
|
479
|
+
// the stream wedges (a tool call or model connection that never returns),
|
|
480
|
+
// no event/delta arrives, Phase 12 below is never reached, and the
|
|
481
|
+
// execution hangs at EXECUTION_IN_PROGRESS forever. On stall we end the run
|
|
482
|
+
// cleanly via the SDK's run.cancel() (guarded by supports("cancel")), which
|
|
483
|
+
// unblocks the for-await; the stallDetected branch in Phase 11a then
|
|
484
|
+
// reports EXECUTION_FAILED with a recognizable, actionable message.
|
|
485
|
+
stallWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
486
|
+
stallDetected = true;
|
|
487
|
+
stallError = new StallTimeoutError(idleMs, lastToolName ? `last tool: ${lastToolName}` : undefined);
|
|
488
|
+
console.warn(`ExecuteCursor stall detected: execution=${executionId}, idleMs=${idleMs}, lastTool=${lastToolName ?? "none"}`);
|
|
489
|
+
if (run.supports?.("cancel")) {
|
|
490
|
+
void run.cancel().catch((cancelErr) => {
|
|
491
|
+
console.warn(`ExecuteCursor run.cancel() after stall failed (non-fatal): execution=${executionId}, ` +
|
|
492
|
+
`error=${cancelErr instanceof Error ? cancelErr.message : cancelErr}`);
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
});
|
|
438
496
|
const accumulator = new MessageAccumulator(status.messages, { mergedPolicies });
|
|
497
|
+
// Shared cadence with the native harness: discrete state changes force a
|
|
498
|
+
// flush; high-frequency token deltas ride this scheduler's time cadence
|
|
499
|
+
// (env-tunable via STREAMING_* — see loadStreamingConfig).
|
|
500
|
+
const scheduler = new StreamingUpdateScheduler(loadStreamingConfig());
|
|
439
501
|
let eventCount = 0;
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
pauseDetected
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
eventRecorder?.record(event, eventCount);
|
|
446
|
-
accumulator.processEvent(event);
|
|
447
|
-
todoTracker.processEvent(event);
|
|
448
|
-
if (event.type === "tool_call" && event.name === "task") {
|
|
449
|
-
accumulator.trackSubAgentExecution(event);
|
|
450
|
-
}
|
|
451
|
-
deltaEnricher.applyEnrichments(status.messages);
|
|
452
|
-
eventCount++;
|
|
453
|
-
if (event.type === "status") {
|
|
454
|
-
console.log(`ExecuteCursor stream status: execution=${executionId}, status=${JSON.stringify(event)}`);
|
|
455
|
-
const statusEvent = event;
|
|
456
|
-
if (statusEvent.status === "ERROR" && statusEvent.message) {
|
|
457
|
-
streamErrorMessage = statusEvent.message;
|
|
502
|
+
try {
|
|
503
|
+
for await (const event of run.stream()) {
|
|
504
|
+
if (pauseDetected || Context.current().cancellationSignal.aborted) {
|
|
505
|
+
pauseDetected = true;
|
|
506
|
+
break;
|
|
458
507
|
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
status.
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
508
|
+
if (stallDetected)
|
|
509
|
+
break;
|
|
510
|
+
// Progress: reset the stall timer on every stream event.
|
|
511
|
+
stallWatchdog.recordActivity();
|
|
512
|
+
if (event.type === "tool_call" && typeof event.name === "string") {
|
|
513
|
+
lastToolName = event.name;
|
|
514
|
+
}
|
|
515
|
+
eventRecorder?.record(event, eventCount);
|
|
516
|
+
accumulator.processEvent(event);
|
|
517
|
+
todoTracker.processEvent(event);
|
|
518
|
+
if (event.type === "tool_call" && event.name === "task") {
|
|
519
|
+
accumulator.trackSubAgentExecution(event);
|
|
520
|
+
}
|
|
521
|
+
deltaEnricher.applyEnrichments(status.messages);
|
|
522
|
+
eventCount++;
|
|
523
|
+
if (event.type === "status") {
|
|
524
|
+
console.log(`ExecuteCursor stream status: execution=${executionId}, status=${JSON.stringify(event)}`);
|
|
525
|
+
const statusEvent = event;
|
|
526
|
+
if (statusEvent.status === "ERROR" && statusEvent.message) {
|
|
527
|
+
streamErrorMessage = statusEvent.message;
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
const shouldPersist = shouldPersistStreamingStatus({
|
|
531
|
+
deltaEnricherDirty: deltaEnricher.isDirty,
|
|
532
|
+
todosDirty: todoTracker.isDirty,
|
|
533
|
+
contentDirty: accumulator.isDirty,
|
|
534
|
+
}, scheduler, eventCount);
|
|
535
|
+
if (usageAccumulator.hasTurns) {
|
|
536
|
+
status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
|
|
537
|
+
}
|
|
538
|
+
if (shouldPersist) {
|
|
539
|
+
// Sync sub-agent executions into status before every persist so the
|
|
540
|
+
// live UI reflects delegation (including the IN_PROGRESS state) while
|
|
541
|
+
// the parent is still running — matching the native harness, which
|
|
542
|
+
// calls syncSubAgentExecutions() on each persist. Without this, the
|
|
543
|
+
// accumulator tracked sub-agents in memory but they only reached the
|
|
544
|
+
// status (and the subscriber stream) after the loop ended.
|
|
545
|
+
status.subAgentExecutions = accumulator.subAgentExecutions;
|
|
546
|
+
const signal = await persist(status);
|
|
547
|
+
deltaEnricher.markPersisted();
|
|
548
|
+
todoTracker.markPersisted();
|
|
549
|
+
accumulator.markPersisted();
|
|
550
|
+
scheduler.markUpdateSent(eventCount);
|
|
551
|
+
heartbeat();
|
|
552
|
+
if (signal === ExecutionControlSignal.STOP) {
|
|
553
|
+
platformStopSignaled = true;
|
|
554
|
+
console.warn(`ExecuteCursor platform stop signal received: execution=${executionId}`);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
if (platformStopSignaled) {
|
|
558
|
+
console.log(`ExecuteCursor stopping stream due to platform stop signal: execution=${executionId}`);
|
|
559
|
+
break;
|
|
481
560
|
}
|
|
482
|
-
}
|
|
483
|
-
if (platformStopSignaled) {
|
|
484
|
-
console.log(`ExecuteCursor stopping stream due to platform stop signal: execution=${executionId}`);
|
|
485
|
-
break;
|
|
486
561
|
}
|
|
487
562
|
}
|
|
563
|
+
catch (streamErr) {
|
|
564
|
+
// run.cancel() from the stall watchdog can make the stream iterator
|
|
565
|
+
// reject; that is the expected teardown, so swallow it and fall through
|
|
566
|
+
// to the stallDetected branch in Phase 11a. Anything else is a genuine
|
|
567
|
+
// stream failure — rethrow it to the outer error handler.
|
|
568
|
+
if (!stallDetected)
|
|
569
|
+
throw streamErr;
|
|
570
|
+
console.warn(`ExecuteCursor stream ended via stall cancel: execution=${executionId}`);
|
|
571
|
+
}
|
|
488
572
|
periodicHeartbeat.stop();
|
|
573
|
+
stallWatchdog.stop();
|
|
489
574
|
// Check both the heartbeat flag AND the shutdown signal directly.
|
|
490
575
|
// Race condition: the heartbeat timer may detect Temporal's CancelledFailure
|
|
491
576
|
// (from worker.shutdown()) before the AbortSignal microtask propagates,
|
|
@@ -505,7 +590,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
505
590
|
// sub-agent the parent had delegated is no longer executing. Mark it
|
|
506
591
|
// CANCELLED rather than leaving a permanent IN_PROGRESS "zombie" in the
|
|
507
592
|
// final snapshot (parity with the native harness's cancelSubAgents()).
|
|
508
|
-
if (pauseDetected || workerShutdownDetected || Context.current().cancellationSignal.aborted) {
|
|
593
|
+
if (pauseDetected || workerShutdownDetected || stallDetected || Context.current().cancellationSignal.aborted) {
|
|
509
594
|
accumulator.cancelInProgressSubAgents();
|
|
510
595
|
}
|
|
511
596
|
status.subAgentExecutions = accumulator.subAgentExecutions;
|
|
@@ -518,7 +603,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
518
603
|
// call statuses before run.wait() / structured output extraction.
|
|
519
604
|
// This is unconditional (not throttled) because finalize is a
|
|
520
605
|
// once-per-execution correctness boundary.
|
|
521
|
-
await
|
|
606
|
+
await persist(status);
|
|
522
607
|
heartbeat();
|
|
523
608
|
// End OTel turn span with accumulated token usage
|
|
524
609
|
const usageSnapshot = usageAccumulator.snapshot();
|
|
@@ -539,7 +624,28 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
539
624
|
catch {
|
|
540
625
|
// Metrics not initialized — silently skip.
|
|
541
626
|
}
|
|
542
|
-
// Phase 11a: Handle worker shutdown, pause, or infrastructure cancellation.
|
|
627
|
+
// Phase 11a: Handle stall, worker shutdown, pause, or infrastructure cancellation.
|
|
628
|
+
// Stall: the watchdog cancelled a turn that made no progress for longer
|
|
629
|
+
// than config.cursorStreamStallTimeoutMs (a wedged tool call or a dead
|
|
630
|
+
// model connection). The keep-alive heartbeat proves liveness, so Temporal
|
|
631
|
+
// never reaps this on its own — this branch is the only clean exit. We
|
|
632
|
+
// RETURN (not throw): re-running the identical prompt via Temporal retry
|
|
633
|
+
// would very likely wedge again. accumulator.finalize() above already
|
|
634
|
+
// cleared isStreaming, so the UI spinners stop.
|
|
635
|
+
if (stallDetected) {
|
|
636
|
+
const err = stallError ?? new StallTimeoutError(config.cursorStreamStallTimeoutMs);
|
|
637
|
+
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
638
|
+
status.error = formatStallFailure(err);
|
|
639
|
+
status.completedAt = utcTimestamp();
|
|
640
|
+
status.messages.push(create(AgentMessageSchema, {
|
|
641
|
+
type: MessageType.MESSAGE_SYSTEM,
|
|
642
|
+
content: `Execution failed: the agent made no progress for too long and was stopped (${err.message}). You can retry or resume.`,
|
|
643
|
+
timestamp: utcTimestamp(),
|
|
644
|
+
}));
|
|
645
|
+
await persist(status);
|
|
646
|
+
console.warn(`ExecuteCursor stalled: execution=${executionId}, events=${eventCount}, error=${status.error}`);
|
|
647
|
+
return slimStatus(status);
|
|
648
|
+
}
|
|
543
649
|
// Worker shutdown: the runner-manager aborted the shutdown signal before
|
|
544
650
|
// calling worker.shutdown(). This is NOT a user-initiated pause — it's
|
|
545
651
|
// an infrastructure event (e.g., premature removal from UI race).
|
|
@@ -552,7 +658,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
552
658
|
content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
|
|
553
659
|
timestamp: utcTimestamp(),
|
|
554
660
|
}));
|
|
555
|
-
await
|
|
661
|
+
await persist(status);
|
|
556
662
|
console.log(`ExecuteCursor interrupted (worker shutdown): execution=${executionId}, events=${eventCount}`);
|
|
557
663
|
throw new CancelledFailure("Activity cancelled (worker shutdown, not user pause)");
|
|
558
664
|
}
|
|
@@ -565,7 +671,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
565
671
|
content: "Execution paused by user. Use resume to continue.",
|
|
566
672
|
timestamp: utcTimestamp(),
|
|
567
673
|
}));
|
|
568
|
-
await
|
|
674
|
+
await persist(status);
|
|
569
675
|
console.log(`ExecuteCursor paused: execution=${executionId}, events=${eventCount}`);
|
|
570
676
|
throw new CancelledFailure("Activity paused by orchestrator");
|
|
571
677
|
}
|
|
@@ -581,7 +687,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
581
687
|
content: "Execution interrupted: the agent was unresponsive for too long. You can retry or resume.",
|
|
582
688
|
timestamp: utcTimestamp(),
|
|
583
689
|
}));
|
|
584
|
-
await
|
|
690
|
+
await persist(status);
|
|
585
691
|
console.log(`ExecuteCursor interrupted (infrastructure cancel): execution=${executionId}, events=${eventCount}`);
|
|
586
692
|
throw new CancelledFailure("Activity cancelled (heartbeat timeout, not user pause)");
|
|
587
693
|
}
|
|
@@ -594,7 +700,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
594
700
|
content: "Execution stopped by the platform.",
|
|
595
701
|
timestamp: utcTimestamp(),
|
|
596
702
|
}));
|
|
597
|
-
await
|
|
703
|
+
await persist(status);
|
|
598
704
|
try {
|
|
599
705
|
resolution.agent.close();
|
|
600
706
|
}
|
|
@@ -614,7 +720,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
614
720
|
const deniedToolCalls = reconcileDeniedToolCalls(status.messages, deniedLedger, mergedPolicies);
|
|
615
721
|
if (deniedToolCalls.length > 0) {
|
|
616
722
|
status.phase = ExecutionPhase.EXECUTION_WAITING_FOR_APPROVAL;
|
|
617
|
-
await
|
|
723
|
+
await persist(status);
|
|
618
724
|
console.log(`ExecuteCursor returning WAITING_FOR_APPROVAL: ${deniedToolCalls.length} tools pending`);
|
|
619
725
|
return slimStatus(status);
|
|
620
726
|
}
|
|
@@ -702,8 +808,10 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
702
808
|
catch (updateErr) {
|
|
703
809
|
console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
|
|
704
810
|
}
|
|
811
|
+
let retryWatchdog;
|
|
705
812
|
const retryRun = await freshAgent.send(freshPrompt, {
|
|
706
813
|
onDelta: ({ update }) => {
|
|
814
|
+
retryWatchdog?.recordActivity();
|
|
707
815
|
if (update.type === "turn-ended" && update.usage) {
|
|
708
816
|
usageAccumulator.addTurn(update.usage);
|
|
709
817
|
}
|
|
@@ -714,18 +822,32 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
714
822
|
catch { /* swallow during retry */ }
|
|
715
823
|
},
|
|
716
824
|
});
|
|
825
|
+
// Mirror the primary stream's stall protection: a wedged retry must
|
|
826
|
+
// not hang the activity. On stall, cancel the run so the loop ends and
|
|
827
|
+
// retryRun.wait() resolves down the existing non-finished failure path.
|
|
828
|
+
retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
829
|
+
console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
|
|
830
|
+
if (retryRun.supports?.("cancel"))
|
|
831
|
+
void retryRun.cancel().catch(() => { });
|
|
832
|
+
});
|
|
717
833
|
streamErrorMessage = undefined;
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
if (
|
|
725
|
-
|
|
834
|
+
try {
|
|
835
|
+
for await (const retryEvent of retryRun.stream()) {
|
|
836
|
+
if (Context.current().cancellationSignal.aborted)
|
|
837
|
+
break;
|
|
838
|
+
retryWatchdog.recordActivity();
|
|
839
|
+
accumulator.processEvent(retryEvent);
|
|
840
|
+
if (retryEvent.type === "status") {
|
|
841
|
+
const retryStatusEvent = retryEvent;
|
|
842
|
+
if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
|
|
843
|
+
streamErrorMessage = retryStatusEvent.message;
|
|
844
|
+
}
|
|
726
845
|
}
|
|
846
|
+
heartbeat();
|
|
727
847
|
}
|
|
728
|
-
|
|
848
|
+
}
|
|
849
|
+
finally {
|
|
850
|
+
retryWatchdog.stop();
|
|
729
851
|
}
|
|
730
852
|
const retryResult = await retryRun.wait();
|
|
731
853
|
console.log(`ExecuteCursor retry run.wait(): execution=${executionId}, ` +
|
|
@@ -783,8 +905,10 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
783
905
|
catch (updateErr) {
|
|
784
906
|
console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
|
|
785
907
|
}
|
|
908
|
+
let retryWatchdog;
|
|
786
909
|
const retryRun = await freshAgent.send(effectivePrompt, {
|
|
787
910
|
onDelta: ({ update }) => {
|
|
911
|
+
retryWatchdog?.recordActivity();
|
|
788
912
|
if (update.type === "turn-ended" && update.usage) {
|
|
789
913
|
usageAccumulator.addTurn(update.usage);
|
|
790
914
|
}
|
|
@@ -795,18 +919,32 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
795
919
|
catch { /* swallow during retry */ }
|
|
796
920
|
},
|
|
797
921
|
});
|
|
922
|
+
// Mirror the primary stream's stall protection: a wedged retry must
|
|
923
|
+
// not hang the activity. On stall, cancel the run so the loop ends and
|
|
924
|
+
// retryRun.wait() resolves down the existing non-finished failure path.
|
|
925
|
+
retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
926
|
+
console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
|
|
927
|
+
if (retryRun.supports?.("cancel"))
|
|
928
|
+
void retryRun.cancel().catch(() => { });
|
|
929
|
+
});
|
|
798
930
|
streamErrorMessage = undefined;
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
if (
|
|
806
|
-
|
|
931
|
+
try {
|
|
932
|
+
for await (const retryEvent of retryRun.stream()) {
|
|
933
|
+
if (Context.current().cancellationSignal.aborted)
|
|
934
|
+
break;
|
|
935
|
+
retryWatchdog.recordActivity();
|
|
936
|
+
accumulator.processEvent(retryEvent);
|
|
937
|
+
if (retryEvent.type === "status") {
|
|
938
|
+
const retryStatusEvent = retryEvent;
|
|
939
|
+
if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
|
|
940
|
+
streamErrorMessage = retryStatusEvent.message;
|
|
941
|
+
}
|
|
807
942
|
}
|
|
943
|
+
heartbeat();
|
|
808
944
|
}
|
|
809
|
-
|
|
945
|
+
}
|
|
946
|
+
finally {
|
|
947
|
+
retryWatchdog.stop();
|
|
810
948
|
}
|
|
811
949
|
const retryResult = await retryRun.wait();
|
|
812
950
|
if (retryResult.status === "finished") {
|
|
@@ -872,9 +1010,8 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
872
1010
|
// Cursor harness has no auto-publish pipeline, so this is the only
|
|
873
1011
|
// artifact path; build storage from the same config-driven factory the
|
|
874
1012
|
// native harness uses.
|
|
875
|
-
if (interactionMode === InteractionMode.PLAN && finalText) {
|
|
1013
|
+
if (interactionMode === InteractionMode.PLAN && finalText && artifactStorage) {
|
|
876
1014
|
try {
|
|
877
|
-
const artifactStorage = createArtifactStorage(loadArtifactStorageConfig(config));
|
|
878
1015
|
await publishPlanArtifact({ status, executionId, planText: finalText, artifactStorage });
|
|
879
1016
|
}
|
|
880
1017
|
catch (err) {
|
|
@@ -884,7 +1021,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
884
1021
|
}
|
|
885
1022
|
}
|
|
886
1023
|
// NOW persist — subscriber sees COMPLETED + structured_output atomically
|
|
887
|
-
await
|
|
1024
|
+
await persist(status);
|
|
888
1025
|
console.log(`ExecuteCursor completed: execution=${executionId}, phase=${ExecutionPhase[status.phase]}, ` +
|
|
889
1026
|
`hasStructuredOutput=${structuredOutput !== undefined}` +
|
|
890
1027
|
(status.error ? `, error=${status.error}` : ""));
|
|
@@ -941,7 +1078,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
941
1078
|
// The aborted Cursor run leaves no live sub-agent — mark any in-flight
|
|
942
1079
|
// delegation CANCELLED so the final snapshot has no zombie sub-agent.
|
|
943
1080
|
cancelInProgressSubAgentProtos(status.subAgentExecutions);
|
|
944
|
-
await
|
|
1081
|
+
await persist(status).catch(() => { });
|
|
945
1082
|
throw err;
|
|
946
1083
|
}
|
|
947
1084
|
// If a non-CancelledFailure error occurs while a pause is in progress,
|
|
@@ -958,7 +1095,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
958
1095
|
timestamp: utcTimestamp(),
|
|
959
1096
|
}));
|
|
960
1097
|
cancelInProgressSubAgentProtos(status.subAgentExecutions);
|
|
961
|
-
await
|
|
1098
|
+
await persist(status).catch(() => { });
|
|
962
1099
|
throw new CancelledFailure("Activity paused by orchestrator (error during pause)");
|
|
963
1100
|
}
|
|
964
1101
|
// Infrastructure cancellation (e.g. heartbeat timeout) with a
|
|
@@ -975,7 +1112,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
975
1112
|
timestamp: utcTimestamp(),
|
|
976
1113
|
}));
|
|
977
1114
|
cancelInProgressSubAgentProtos(status.subAgentExecutions);
|
|
978
|
-
await
|
|
1115
|
+
await persist(status).catch(() => { });
|
|
979
1116
|
throw new CancelledFailure("Activity cancelled (infrastructure, not user pause)");
|
|
980
1117
|
}
|
|
981
1118
|
// A thrown CursorSdkError carries structured fields (code/status/endpoint/
|
|
@@ -1008,7 +1145,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
1008
1145
|
timestamp: utcTimestamp(),
|
|
1009
1146
|
}));
|
|
1010
1147
|
try {
|
|
1011
|
-
await
|
|
1148
|
+
await persist(status);
|
|
1012
1149
|
}
|
|
1013
1150
|
catch (persistErr) {
|
|
1014
1151
|
console.error("Failed to persist error status (best-effort):", persistErr);
|
|
@@ -1031,7 +1168,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
1031
1168
|
timestamp: utcTimestamp(),
|
|
1032
1169
|
}));
|
|
1033
1170
|
try {
|
|
1034
|
-
await
|
|
1171
|
+
await persist(status);
|
|
1035
1172
|
}
|
|
1036
1173
|
catch (persistErr) {
|
|
1037
1174
|
console.error("Failed to persist error status (best-effort):", persistErr);
|
|
@@ -1039,6 +1176,10 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
1039
1176
|
return slimStatus(status);
|
|
1040
1177
|
}
|
|
1041
1178
|
finally {
|
|
1179
|
+
// Disarm the stall watchdog on EVERY exit path (idempotent). The happy
|
|
1180
|
+
// path stops it after the stream loop; this covers throws before that
|
|
1181
|
+
// point so no orphaned timer survives the activity.
|
|
1182
|
+
stallWatchdog?.stop();
|
|
1042
1183
|
// Tear down the HITL gate on EVERY exit path (success, error, approval
|
|
1043
1184
|
// pause, cancellation) so attaching a real repo leaves the user's
|
|
1044
1185
|
// .cursor/hooks.json and workspace untouched between turns (issue #173).
|