@stigmer/runner 3.0.8-dev.20260613074252 → 3.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/activities/execute-cursor/hook-script.d.ts +23 -12
- package/dist/activities/execute-cursor/hook-script.js +85 -51
- package/dist/activities/execute-cursor/hook-script.js.map +1 -1
- package/dist/activities/execute-cursor/index.js +210 -79
- package/dist/activities/execute-cursor/index.js.map +1 -1
- package/dist/activities/execute-cursor/message-translator.d.ts +35 -0
- package/dist/activities/execute-cursor/message-translator.js +114 -6
- package/dist/activities/execute-cursor/message-translator.js.map +1 -1
- package/dist/activities/execute-cursor/prompt-builder.d.ts +25 -0
- package/dist/activities/execute-cursor/prompt-builder.js +54 -0
- package/dist/activities/execute-cursor/prompt-builder.js.map +1 -1
- package/dist/activities/execute-cursor/workspace-setup.d.ts +8 -2
- package/dist/activities/execute-cursor/workspace-setup.js +62 -30
- package/dist/activities/execute-cursor/workspace-setup.js.map +1 -1
- package/dist/activities/execute-deep-agent/index.js +14 -4
- package/dist/activities/execute-deep-agent/index.js.map +1 -1
- package/dist/activities/execute-deep-agent/status-builder-shared.d.ts +0 -1
- package/dist/activities/execute-deep-agent/status-builder-shared.js +32 -8
- package/dist/activities/execute-deep-agent/status-builder-shared.js.map +1 -1
- package/dist/activities/execute-deep-agent/status-builder.js +4 -5
- package/dist/activities/execute-deep-agent/status-builder.js.map +1 -1
- package/dist/activities/execute-deep-agent/streaming-v3.js +3 -4
- package/dist/activities/execute-deep-agent/streaming-v3.js.map +1 -1
- package/dist/activities/execute-deep-agent/streaming.d.ts +8 -0
- package/dist/activities/execute-deep-agent/streaming.js +3 -4
- package/dist/activities/execute-deep-agent/streaming.js.map +1 -1
- package/dist/activities/execute-deep-agent/subagent-tracker.js +4 -5
- package/dist/activities/execute-deep-agent/subagent-tracker.js.map +1 -1
- package/dist/activities/execute-deep-agent/v3-status-builder.js +6 -5
- package/dist/activities/execute-deep-agent/v3-status-builder.js.map +1 -1
- package/dist/config.d.ts +21 -0
- package/dist/config.js +12 -0
- package/dist/config.js.map +1 -1
- package/dist/in-flight.d.ts +35 -0
- package/dist/in-flight.js +61 -0
- package/dist/in-flight.js.map +1 -0
- package/dist/main.js +6 -3
- package/dist/main.js.map +1 -1
- package/dist/runner-manager.d.ts +2 -0
- package/dist/runner-manager.js +90 -29
- package/dist/runner-manager.js.map +1 -1
- package/dist/runner.d.ts +2 -0
- package/dist/runner.js +2 -0
- package/dist/runner.js.map +1 -1
- package/dist/shared/grpc-retry.d.ts +9 -20
- package/dist/shared/grpc-retry.js +9 -52
- package/dist/shared/grpc-retry.js.map +1 -1
- package/dist/shared/stall-watchdog.d.ts +68 -0
- package/dist/shared/stall-watchdog.js +102 -0
- package/dist/shared/stall-watchdog.js.map +1 -0
- package/dist/shared/status-offload.d.ts +84 -0
- package/dist/shared/status-offload.js +292 -0
- package/dist/shared/status-offload.js.map +1 -0
- package/dist/shared/status.d.ts +34 -3
- package/dist/shared/status.js +102 -9
- package/dist/shared/status.js.map +1 -1
- package/package.json +2 -2
- package/src/__tests__/config.test.ts +8 -0
- package/src/__tests__/in-flight.test.ts +84 -0
- package/src/activities/__tests__/classify-tool-approvals.test.ts +1 -0
- package/src/activities/__tests__/discover-mcp-server.test.ts +1 -0
- package/src/activities/execute-cursor/__tests__/build-prompt.test.ts +74 -0
- package/src/activities/execute-cursor/__tests__/hook-script.test.ts +90 -15
- package/src/activities/execute-cursor/__tests__/tool-result-image.test.ts +244 -0
- package/src/activities/execute-cursor/__tests__/workspace-setup.test.ts +53 -4
- package/src/activities/execute-cursor/hook-script.ts +85 -51
- package/src/activities/execute-cursor/index.ts +170 -35
- package/src/activities/execute-cursor/message-translator.ts +113 -6
- package/src/activities/execute-cursor/prompt-builder.ts +59 -0
- package/src/activities/execute-cursor/workspace-setup.ts +76 -44
- package/src/activities/execute-deep-agent/__tests__/index.test.ts +1 -0
- package/src/activities/execute-deep-agent/__tests__/status-builder-shared.test.ts +66 -0
- package/src/activities/execute-deep-agent/__tests__/status-builder.test.ts +6 -3
- package/src/activities/execute-deep-agent/__tests__/streaming-v3.test.ts +70 -0
- package/src/activities/execute-deep-agent/index.ts +16 -4
- package/src/activities/execute-deep-agent/status-builder-shared.ts +27 -5
- package/src/activities/execute-deep-agent/status-builder.ts +3 -5
- package/src/activities/execute-deep-agent/streaming-v3.ts +4 -4
- package/src/activities/execute-deep-agent/streaming.ts +13 -4
- package/src/activities/execute-deep-agent/subagent-tracker.ts +4 -5
- package/src/activities/execute-deep-agent/v3-status-builder.ts +5 -5
- package/src/config.ts +27 -0
- package/src/in-flight.ts +71 -0
- package/src/main.ts +7 -2
- package/src/runner-manager.ts +127 -33
- package/src/runner.ts +6 -0
- package/src/shared/__tests__/artifact-storage.test.ts +1 -0
- package/src/shared/__tests__/grpc-retry-extended.test.ts +6 -144
- package/src/shared/__tests__/grpc-retry.test.ts +5 -134
- package/src/shared/__tests__/stall-watchdog.test.ts +193 -0
- package/src/shared/__tests__/status-offload.test.ts +256 -0
- package/src/shared/__tests__/status.test.ts +199 -0
- package/src/shared/grpc-retry.ts +9 -72
- package/src/shared/stall-watchdog.ts +122 -0
- package/src/shared/status-offload.ts +342 -0
- package/src/shared/status.ts +142 -8
|
@@ -36,6 +36,7 @@ import { CursorMode } from "@stigmer/protos/ai/stigmer/agentic/session/v1/enum_p
|
|
|
36
36
|
import { determineCursorMode, isCloudMode } from "./cursor-mode.js";
|
|
37
37
|
import { MessageAccumulator, reconcileDeniedToolCalls, cancelInProgressSubAgentProtos } from "./message-translator.js";
|
|
38
38
|
import { utcTimestamp, persistStatus, reportSetupProgress, slimStatus } from "../../shared/status.js";
|
|
39
|
+
import { startStallWatchdog, StallTimeoutError, formatStallFailure } from "../../shared/stall-watchdog.js";
|
|
39
40
|
import { createArtifactStorage, loadArtifactStorageConfig } from "../../shared/artifact-storage.js";
|
|
40
41
|
import { publishPlanArtifact } from "../../shared/plan-artifact.js";
|
|
41
42
|
import { DeltaEnricher } from "./delta-enricher.js";
|
|
@@ -102,11 +103,40 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
102
103
|
phase: ExecutionPhase.EXECUTION_IN_PROGRESS,
|
|
103
104
|
startedAt: utcTimestamp(),
|
|
104
105
|
});
|
|
106
|
+
// Artifact storage for offloading oversized tool outputs (screenshots, giant
|
|
107
|
+
// dumps) out of the persisted status, and for publishing the plan artifact.
|
|
108
|
+
// Created once here so it is available to EVERY persist below. Best-effort:
|
|
109
|
+
// if it can't be built (e.g. proxy mode without a token), offload is disabled
|
|
110
|
+
// but persistStatus still enforces the aggregate size cap, so persistence can
|
|
111
|
+
// never silently blow past the gRPC limit.
|
|
112
|
+
let artifactStorage;
|
|
113
|
+
try {
|
|
114
|
+
artifactStorage = createArtifactStorage(loadArtifactStorageConfig(config));
|
|
115
|
+
}
|
|
116
|
+
catch (storageErr) {
|
|
117
|
+
console.warn(`ExecuteCursor artifact storage unavailable — tool-output offload disabled ` +
|
|
118
|
+
`(aggregate size guard still active): execution=${executionId}, error=${storageErr}`);
|
|
119
|
+
}
|
|
120
|
+
const statusOffload = artifactStorage
|
|
121
|
+
? { artifactStorage, executionId }
|
|
122
|
+
: undefined;
|
|
123
|
+
// ALL status persistence in this activity flows through `persist`, so the
|
|
124
|
+
// single size-bounding guard (offload + aggregate elision) is unforgeable and
|
|
125
|
+
// a future call site cannot accidentally skip it.
|
|
126
|
+
const persist = (s = status) => persistStatus(client, executionId, s, { offload: statusOffload });
|
|
105
127
|
let sessionId;
|
|
106
128
|
let session;
|
|
107
129
|
let pauseDetected = false;
|
|
108
130
|
let workerShutdownDetected = false;
|
|
131
|
+
// Set by the stall watchdog when the SDK stream makes no progress for
|
|
132
|
+
// config.cursorStreamStallTimeoutMs. stallError carries the recognizable
|
|
133
|
+
// message surfaced to the user; both feed the Phase 11a stall branch.
|
|
134
|
+
let stallDetected = false;
|
|
135
|
+
let stallError;
|
|
109
136
|
let periodicHeartbeat;
|
|
137
|
+
// Progress-based stall watchdog (see ../../shared/stall-watchdog.ts). Stopped
|
|
138
|
+
// in the finally on every exit path; complements the liveness heartbeat.
|
|
139
|
+
let stallWatchdog;
|
|
110
140
|
// Session HITL directory (runner-owned, outside the workspace) where the hook
|
|
111
141
|
// script, approval-state file, and denial ledger live. Set once the gate is
|
|
112
142
|
// installed; the WAITING_FOR_APPROVAL path reads the denial ledger from here.
|
|
@@ -186,7 +216,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
186
216
|
content: "Execution was rejected by the user during tool approval.",
|
|
187
217
|
timestamp: utcTimestamp(),
|
|
188
218
|
}));
|
|
189
|
-
await
|
|
219
|
+
await persist(status);
|
|
190
220
|
return slimStatus(status);
|
|
191
221
|
}
|
|
192
222
|
}
|
|
@@ -384,6 +414,9 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
384
414
|
let firstTurnAttributionLogged = false;
|
|
385
415
|
let streamErrorMessage;
|
|
386
416
|
let alreadyRetriedWithFreshAgent = false;
|
|
417
|
+
// Most recent tool name observed on the stream, used only to enrich the
|
|
418
|
+
// stall message ("last tool: …") so a wedged turn names the likely culprit.
|
|
419
|
+
let lastToolName;
|
|
387
420
|
// Periodic heartbeat keeps Temporal informed during silent SDK operations
|
|
388
421
|
// (e.g. long tool calls, MCP requests, model thinking). Without this,
|
|
389
422
|
// the 2-minute heartbeat timeout can cancel the activity and mislabel
|
|
@@ -411,6 +444,10 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
411
444
|
}
|
|
412
445
|
const run = await resolution.agent.send(effectivePrompt, {
|
|
413
446
|
onDelta: ({ update }) => {
|
|
447
|
+
// Reset the stall timer on the delta channel too: a long model
|
|
448
|
+
// generation emits token deltas but few discrete stream events, so
|
|
449
|
+
// resetting only in the stream loop would false-positive a stall.
|
|
450
|
+
stallWatchdog?.recordActivity();
|
|
414
451
|
if (update.type === "turn-ended" && update.usage) {
|
|
415
452
|
usageAccumulator.addTurn(update.usage);
|
|
416
453
|
if (!firstTurnAttributionLogged) {
|
|
@@ -435,57 +472,95 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
435
472
|
}
|
|
436
473
|
},
|
|
437
474
|
});
|
|
475
|
+
// Arm the stall watchdog now that the run exists. The periodic heartbeat
|
|
476
|
+
// above proves the process is alive, not that the agent is progressing: if
|
|
477
|
+
// the stream wedges (a tool call or model connection that never returns),
|
|
478
|
+
// no event/delta arrives, Phase 12 below is never reached, and the
|
|
479
|
+
// execution hangs at EXECUTION_IN_PROGRESS forever. On stall we end the run
|
|
480
|
+
// cleanly via the SDK's run.cancel() (guarded by supports("cancel")), which
|
|
481
|
+
// unblocks the for-await; the stallDetected branch in Phase 11a then
|
|
482
|
+
// reports EXECUTION_FAILED with a recognizable, actionable message.
|
|
483
|
+
stallWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
484
|
+
stallDetected = true;
|
|
485
|
+
stallError = new StallTimeoutError(idleMs, lastToolName ? `last tool: ${lastToolName}` : undefined);
|
|
486
|
+
console.warn(`ExecuteCursor stall detected: execution=${executionId}, idleMs=${idleMs}, lastTool=${lastToolName ?? "none"}`);
|
|
487
|
+
if (run.supports?.("cancel")) {
|
|
488
|
+
void run.cancel().catch((cancelErr) => {
|
|
489
|
+
console.warn(`ExecuteCursor run.cancel() after stall failed (non-fatal): execution=${executionId}, ` +
|
|
490
|
+
`error=${cancelErr instanceof Error ? cancelErr.message : cancelErr}`);
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
});
|
|
438
494
|
const accumulator = new MessageAccumulator(status.messages, { mergedPolicies });
|
|
439
495
|
let eventCount = 0;
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
pauseDetected
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
eventRecorder?.record(event, eventCount);
|
|
446
|
-
accumulator.processEvent(event);
|
|
447
|
-
todoTracker.processEvent(event);
|
|
448
|
-
if (event.type === "tool_call" && event.name === "task") {
|
|
449
|
-
accumulator.trackSubAgentExecution(event);
|
|
450
|
-
}
|
|
451
|
-
deltaEnricher.applyEnrichments(status.messages);
|
|
452
|
-
eventCount++;
|
|
453
|
-
if (event.type === "status") {
|
|
454
|
-
console.log(`ExecuteCursor stream status: execution=${executionId}, status=${JSON.stringify(event)}`);
|
|
455
|
-
const statusEvent = event;
|
|
456
|
-
if (statusEvent.status === "ERROR" && statusEvent.message) {
|
|
457
|
-
streamErrorMessage = statusEvent.message;
|
|
496
|
+
try {
|
|
497
|
+
for await (const event of run.stream()) {
|
|
498
|
+
if (pauseDetected || Context.current().cancellationSignal.aborted) {
|
|
499
|
+
pauseDetected = true;
|
|
500
|
+
break;
|
|
458
501
|
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
status.
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
502
|
+
if (stallDetected)
|
|
503
|
+
break;
|
|
504
|
+
// Progress: reset the stall timer on every stream event.
|
|
505
|
+
stallWatchdog.recordActivity();
|
|
506
|
+
if (event.type === "tool_call" && typeof event.name === "string") {
|
|
507
|
+
lastToolName = event.name;
|
|
508
|
+
}
|
|
509
|
+
eventRecorder?.record(event, eventCount);
|
|
510
|
+
accumulator.processEvent(event);
|
|
511
|
+
todoTracker.processEvent(event);
|
|
512
|
+
if (event.type === "tool_call" && event.name === "task") {
|
|
513
|
+
accumulator.trackSubAgentExecution(event);
|
|
514
|
+
}
|
|
515
|
+
deltaEnricher.applyEnrichments(status.messages);
|
|
516
|
+
eventCount++;
|
|
517
|
+
if (event.type === "status") {
|
|
518
|
+
console.log(`ExecuteCursor stream status: execution=${executionId}, status=${JSON.stringify(event)}`);
|
|
519
|
+
const statusEvent = event;
|
|
520
|
+
if (statusEvent.status === "ERROR" && statusEvent.message) {
|
|
521
|
+
streamErrorMessage = statusEvent.message;
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
const shouldPersist = eventCount % 20 === 0 || deltaEnricher.isDirty ||
|
|
525
|
+
todoTracker.isDirty || accumulator.subAgentDirty;
|
|
526
|
+
if (usageAccumulator.hasTurns) {
|
|
527
|
+
status.streamingUsage = create(StreamingUsageSummarySchema, usageAccumulator.snapshot());
|
|
528
|
+
}
|
|
529
|
+
if (shouldPersist) {
|
|
530
|
+
// Sync sub-agent executions into status before every persist so the
|
|
531
|
+
// live UI reflects delegation (including the IN_PROGRESS state) while
|
|
532
|
+
// the parent is still running — matching the native harness, which
|
|
533
|
+
// calls syncSubAgentExecutions() on each persist. Without this, the
|
|
534
|
+
// accumulator tracked sub-agents in memory but they only reached the
|
|
535
|
+
// status (and the subscriber stream) after the loop ended.
|
|
536
|
+
status.subAgentExecutions = accumulator.subAgentExecutions;
|
|
537
|
+
const signal = await persist(status);
|
|
538
|
+
deltaEnricher.markPersisted();
|
|
539
|
+
todoTracker.markPersisted();
|
|
540
|
+
accumulator.markSubAgentPersisted();
|
|
541
|
+
heartbeat();
|
|
542
|
+
if (signal === ExecutionControlSignal.STOP) {
|
|
543
|
+
platformStopSignaled = true;
|
|
544
|
+
console.warn(`ExecuteCursor platform stop signal received: execution=${executionId}`);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
if (platformStopSignaled) {
|
|
548
|
+
console.log(`ExecuteCursor stopping stream due to platform stop signal: execution=${executionId}`);
|
|
549
|
+
break;
|
|
481
550
|
}
|
|
482
|
-
}
|
|
483
|
-
if (platformStopSignaled) {
|
|
484
|
-
console.log(`ExecuteCursor stopping stream due to platform stop signal: execution=${executionId}`);
|
|
485
|
-
break;
|
|
486
551
|
}
|
|
487
552
|
}
|
|
553
|
+
catch (streamErr) {
|
|
554
|
+
// run.cancel() from the stall watchdog can make the stream iterator
|
|
555
|
+
// reject; that is the expected teardown, so swallow it and fall through
|
|
556
|
+
// to the stallDetected branch in Phase 11a. Anything else is a genuine
|
|
557
|
+
// stream failure — rethrow it to the outer error handler.
|
|
558
|
+
if (!stallDetected)
|
|
559
|
+
throw streamErr;
|
|
560
|
+
console.warn(`ExecuteCursor stream ended via stall cancel: execution=${executionId}`);
|
|
561
|
+
}
|
|
488
562
|
periodicHeartbeat.stop();
|
|
563
|
+
stallWatchdog.stop();
|
|
489
564
|
// Check both the heartbeat flag AND the shutdown signal directly.
|
|
490
565
|
// Race condition: the heartbeat timer may detect Temporal's CancelledFailure
|
|
491
566
|
// (from worker.shutdown()) before the AbortSignal microtask propagates,
|
|
@@ -505,7 +580,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
505
580
|
// sub-agent the parent had delegated is no longer executing. Mark it
|
|
506
581
|
// CANCELLED rather than leaving a permanent IN_PROGRESS "zombie" in the
|
|
507
582
|
// final snapshot (parity with the native harness's cancelSubAgents()).
|
|
508
|
-
if (pauseDetected || workerShutdownDetected || Context.current().cancellationSignal.aborted) {
|
|
583
|
+
if (pauseDetected || workerShutdownDetected || stallDetected || Context.current().cancellationSignal.aborted) {
|
|
509
584
|
accumulator.cancelInProgressSubAgents();
|
|
510
585
|
}
|
|
511
586
|
status.subAgentExecutions = accumulator.subAgentExecutions;
|
|
@@ -518,7 +593,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
518
593
|
// call statuses before run.wait() / structured output extraction.
|
|
519
594
|
// This is unconditional (not throttled) because finalize is a
|
|
520
595
|
// once-per-execution correctness boundary.
|
|
521
|
-
await
|
|
596
|
+
await persist(status);
|
|
522
597
|
heartbeat();
|
|
523
598
|
// End OTel turn span with accumulated token usage
|
|
524
599
|
const usageSnapshot = usageAccumulator.snapshot();
|
|
@@ -539,7 +614,28 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
539
614
|
catch {
|
|
540
615
|
// Metrics not initialized — silently skip.
|
|
541
616
|
}
|
|
542
|
-
// Phase 11a: Handle worker shutdown, pause, or infrastructure cancellation.
|
|
617
|
+
// Phase 11a: Handle stall, worker shutdown, pause, or infrastructure cancellation.
|
|
618
|
+
// Stall: the watchdog cancelled a turn that made no progress for longer
|
|
619
|
+
// than config.cursorStreamStallTimeoutMs (a wedged tool call or a dead
|
|
620
|
+
// model connection). The keep-alive heartbeat proves liveness, so Temporal
|
|
621
|
+
// never reaps this on its own — this branch is the only clean exit. We
|
|
622
|
+
// RETURN (not throw): re-running the identical prompt via Temporal retry
|
|
623
|
+
// would very likely wedge again. accumulator.finalize() above already
|
|
624
|
+
// cleared isStreaming, so the UI spinners stop.
|
|
625
|
+
if (stallDetected) {
|
|
626
|
+
const err = stallError ?? new StallTimeoutError(config.cursorStreamStallTimeoutMs);
|
|
627
|
+
status.phase = ExecutionPhase.EXECUTION_FAILED;
|
|
628
|
+
status.error = formatStallFailure(err);
|
|
629
|
+
status.completedAt = utcTimestamp();
|
|
630
|
+
status.messages.push(create(AgentMessageSchema, {
|
|
631
|
+
type: MessageType.MESSAGE_SYSTEM,
|
|
632
|
+
content: `Execution failed: the agent made no progress for too long and was stopped (${err.message}). You can retry or resume.`,
|
|
633
|
+
timestamp: utcTimestamp(),
|
|
634
|
+
}));
|
|
635
|
+
await persist(status);
|
|
636
|
+
console.warn(`ExecuteCursor stalled: execution=${executionId}, events=${eventCount}, error=${status.error}`);
|
|
637
|
+
return slimStatus(status);
|
|
638
|
+
}
|
|
543
639
|
// Worker shutdown: the runner-manager aborted the shutdown signal before
|
|
544
640
|
// calling worker.shutdown(). This is NOT a user-initiated pause — it's
|
|
545
641
|
// an infrastructure event (e.g., premature removal from UI race).
|
|
@@ -552,7 +648,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
552
648
|
content: "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
|
|
553
649
|
timestamp: utcTimestamp(),
|
|
554
650
|
}));
|
|
555
|
-
await
|
|
651
|
+
await persist(status);
|
|
556
652
|
console.log(`ExecuteCursor interrupted (worker shutdown): execution=${executionId}, events=${eventCount}`);
|
|
557
653
|
throw new CancelledFailure("Activity cancelled (worker shutdown, not user pause)");
|
|
558
654
|
}
|
|
@@ -565,7 +661,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
565
661
|
content: "Execution paused by user. Use resume to continue.",
|
|
566
662
|
timestamp: utcTimestamp(),
|
|
567
663
|
}));
|
|
568
|
-
await
|
|
664
|
+
await persist(status);
|
|
569
665
|
console.log(`ExecuteCursor paused: execution=${executionId}, events=${eventCount}`);
|
|
570
666
|
throw new CancelledFailure("Activity paused by orchestrator");
|
|
571
667
|
}
|
|
@@ -581,7 +677,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
581
677
|
content: "Execution interrupted: the agent was unresponsive for too long. You can retry or resume.",
|
|
582
678
|
timestamp: utcTimestamp(),
|
|
583
679
|
}));
|
|
584
|
-
await
|
|
680
|
+
await persist(status);
|
|
585
681
|
console.log(`ExecuteCursor interrupted (infrastructure cancel): execution=${executionId}, events=${eventCount}`);
|
|
586
682
|
throw new CancelledFailure("Activity cancelled (heartbeat timeout, not user pause)");
|
|
587
683
|
}
|
|
@@ -594,7 +690,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
594
690
|
content: "Execution stopped by the platform.",
|
|
595
691
|
timestamp: utcTimestamp(),
|
|
596
692
|
}));
|
|
597
|
-
await
|
|
693
|
+
await persist(status);
|
|
598
694
|
try {
|
|
599
695
|
resolution.agent.close();
|
|
600
696
|
}
|
|
@@ -614,7 +710,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
614
710
|
const deniedToolCalls = reconcileDeniedToolCalls(status.messages, deniedLedger, mergedPolicies);
|
|
615
711
|
if (deniedToolCalls.length > 0) {
|
|
616
712
|
status.phase = ExecutionPhase.EXECUTION_WAITING_FOR_APPROVAL;
|
|
617
|
-
await
|
|
713
|
+
await persist(status);
|
|
618
714
|
console.log(`ExecuteCursor returning WAITING_FOR_APPROVAL: ${deniedToolCalls.length} tools pending`);
|
|
619
715
|
return slimStatus(status);
|
|
620
716
|
}
|
|
@@ -702,8 +798,10 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
702
798
|
catch (updateErr) {
|
|
703
799
|
console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
|
|
704
800
|
}
|
|
801
|
+
let retryWatchdog;
|
|
705
802
|
const retryRun = await freshAgent.send(freshPrompt, {
|
|
706
803
|
onDelta: ({ update }) => {
|
|
804
|
+
retryWatchdog?.recordActivity();
|
|
707
805
|
if (update.type === "turn-ended" && update.usage) {
|
|
708
806
|
usageAccumulator.addTurn(update.usage);
|
|
709
807
|
}
|
|
@@ -714,18 +812,32 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
714
812
|
catch { /* swallow during retry */ }
|
|
715
813
|
},
|
|
716
814
|
});
|
|
815
|
+
// Mirror the primary stream's stall protection: a wedged retry must
|
|
816
|
+
// not hang the activity. On stall, cancel the run so the loop ends and
|
|
817
|
+
// retryRun.wait() resolves down the existing non-finished failure path.
|
|
818
|
+
retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
819
|
+
console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
|
|
820
|
+
if (retryRun.supports?.("cancel"))
|
|
821
|
+
void retryRun.cancel().catch(() => { });
|
|
822
|
+
});
|
|
717
823
|
streamErrorMessage = undefined;
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
if (
|
|
725
|
-
|
|
824
|
+
try {
|
|
825
|
+
for await (const retryEvent of retryRun.stream()) {
|
|
826
|
+
if (Context.current().cancellationSignal.aborted)
|
|
827
|
+
break;
|
|
828
|
+
retryWatchdog.recordActivity();
|
|
829
|
+
accumulator.processEvent(retryEvent);
|
|
830
|
+
if (retryEvent.type === "status") {
|
|
831
|
+
const retryStatusEvent = retryEvent;
|
|
832
|
+
if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
|
|
833
|
+
streamErrorMessage = retryStatusEvent.message;
|
|
834
|
+
}
|
|
726
835
|
}
|
|
836
|
+
heartbeat();
|
|
727
837
|
}
|
|
728
|
-
|
|
838
|
+
}
|
|
839
|
+
finally {
|
|
840
|
+
retryWatchdog.stop();
|
|
729
841
|
}
|
|
730
842
|
const retryResult = await retryRun.wait();
|
|
731
843
|
console.log(`ExecuteCursor retry run.wait(): execution=${executionId}, ` +
|
|
@@ -783,8 +895,10 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
783
895
|
catch (updateErr) {
|
|
784
896
|
console.warn("Failed to update session with fresh agentId (non-fatal):", updateErr);
|
|
785
897
|
}
|
|
898
|
+
let retryWatchdog;
|
|
786
899
|
const retryRun = await freshAgent.send(effectivePrompt, {
|
|
787
900
|
onDelta: ({ update }) => {
|
|
901
|
+
retryWatchdog?.recordActivity();
|
|
788
902
|
if (update.type === "turn-ended" && update.usage) {
|
|
789
903
|
usageAccumulator.addTurn(update.usage);
|
|
790
904
|
}
|
|
@@ -795,18 +909,32 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
795
909
|
catch { /* swallow during retry */ }
|
|
796
910
|
},
|
|
797
911
|
});
|
|
912
|
+
// Mirror the primary stream's stall protection: a wedged retry must
|
|
913
|
+
// not hang the activity. On stall, cancel the run so the loop ends and
|
|
914
|
+
// retryRun.wait() resolves down the existing non-finished failure path.
|
|
915
|
+
retryWatchdog = startStallWatchdog(config.cursorStreamStallTimeoutMs, (idleMs) => {
|
|
916
|
+
console.warn(`ExecuteCursor retry stall detected: execution=${executionId}, idleMs=${idleMs}`);
|
|
917
|
+
if (retryRun.supports?.("cancel"))
|
|
918
|
+
void retryRun.cancel().catch(() => { });
|
|
919
|
+
});
|
|
798
920
|
streamErrorMessage = undefined;
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
if (
|
|
806
|
-
|
|
921
|
+
try {
|
|
922
|
+
for await (const retryEvent of retryRun.stream()) {
|
|
923
|
+
if (Context.current().cancellationSignal.aborted)
|
|
924
|
+
break;
|
|
925
|
+
retryWatchdog.recordActivity();
|
|
926
|
+
accumulator.processEvent(retryEvent);
|
|
927
|
+
if (retryEvent.type === "status") {
|
|
928
|
+
const retryStatusEvent = retryEvent;
|
|
929
|
+
if (retryStatusEvent.status === "ERROR" && retryStatusEvent.message) {
|
|
930
|
+
streamErrorMessage = retryStatusEvent.message;
|
|
931
|
+
}
|
|
807
932
|
}
|
|
933
|
+
heartbeat();
|
|
808
934
|
}
|
|
809
|
-
|
|
935
|
+
}
|
|
936
|
+
finally {
|
|
937
|
+
retryWatchdog.stop();
|
|
810
938
|
}
|
|
811
939
|
const retryResult = await retryRun.wait();
|
|
812
940
|
if (retryResult.status === "finished") {
|
|
@@ -872,9 +1000,8 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
872
1000
|
// Cursor harness has no auto-publish pipeline, so this is the only
|
|
873
1001
|
// artifact path; build storage from the same config-driven factory the
|
|
874
1002
|
// native harness uses.
|
|
875
|
-
if (interactionMode === InteractionMode.PLAN && finalText) {
|
|
1003
|
+
if (interactionMode === InteractionMode.PLAN && finalText && artifactStorage) {
|
|
876
1004
|
try {
|
|
877
|
-
const artifactStorage = createArtifactStorage(loadArtifactStorageConfig(config));
|
|
878
1005
|
await publishPlanArtifact({ status, executionId, planText: finalText, artifactStorage });
|
|
879
1006
|
}
|
|
880
1007
|
catch (err) {
|
|
@@ -884,7 +1011,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
884
1011
|
}
|
|
885
1012
|
}
|
|
886
1013
|
// NOW persist — subscriber sees COMPLETED + structured_output atomically
|
|
887
|
-
await
|
|
1014
|
+
await persist(status);
|
|
888
1015
|
console.log(`ExecuteCursor completed: execution=${executionId}, phase=${ExecutionPhase[status.phase]}, ` +
|
|
889
1016
|
`hasStructuredOutput=${structuredOutput !== undefined}` +
|
|
890
1017
|
(status.error ? `, error=${status.error}` : ""));
|
|
@@ -941,7 +1068,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
941
1068
|
// The aborted Cursor run leaves no live sub-agent — mark any in-flight
|
|
942
1069
|
// delegation CANCELLED so the final snapshot has no zombie sub-agent.
|
|
943
1070
|
cancelInProgressSubAgentProtos(status.subAgentExecutions);
|
|
944
|
-
await
|
|
1071
|
+
await persist(status).catch(() => { });
|
|
945
1072
|
throw err;
|
|
946
1073
|
}
|
|
947
1074
|
// If a non-CancelledFailure error occurs while a pause is in progress,
|
|
@@ -958,7 +1085,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
958
1085
|
timestamp: utcTimestamp(),
|
|
959
1086
|
}));
|
|
960
1087
|
cancelInProgressSubAgentProtos(status.subAgentExecutions);
|
|
961
|
-
await
|
|
1088
|
+
await persist(status).catch(() => { });
|
|
962
1089
|
throw new CancelledFailure("Activity paused by orchestrator (error during pause)");
|
|
963
1090
|
}
|
|
964
1091
|
// Infrastructure cancellation (e.g. heartbeat timeout) with a
|
|
@@ -975,7 +1102,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
975
1102
|
timestamp: utcTimestamp(),
|
|
976
1103
|
}));
|
|
977
1104
|
cancelInProgressSubAgentProtos(status.subAgentExecutions);
|
|
978
|
-
await
|
|
1105
|
+
await persist(status).catch(() => { });
|
|
979
1106
|
throw new CancelledFailure("Activity cancelled (infrastructure, not user pause)");
|
|
980
1107
|
}
|
|
981
1108
|
// A thrown CursorSdkError carries structured fields (code/status/endpoint/
|
|
@@ -1008,7 +1135,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
1008
1135
|
timestamp: utcTimestamp(),
|
|
1009
1136
|
}));
|
|
1010
1137
|
try {
|
|
1011
|
-
await
|
|
1138
|
+
await persist(status);
|
|
1012
1139
|
}
|
|
1013
1140
|
catch (persistErr) {
|
|
1014
1141
|
console.error("Failed to persist error status (best-effort):", persistErr);
|
|
@@ -1031,7 +1158,7 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
1031
1158
|
timestamp: utcTimestamp(),
|
|
1032
1159
|
}));
|
|
1033
1160
|
try {
|
|
1034
|
-
await
|
|
1161
|
+
await persist(status);
|
|
1035
1162
|
}
|
|
1036
1163
|
catch (persistErr) {
|
|
1037
1164
|
console.error("Failed to persist error status (best-effort):", persistErr);
|
|
@@ -1039,6 +1166,10 @@ async function executeCursorInner(config, client, executionId, threadId) {
|
|
|
1039
1166
|
return slimStatus(status);
|
|
1040
1167
|
}
|
|
1041
1168
|
finally {
|
|
1169
|
+
// Disarm the stall watchdog on EVERY exit path (idempotent). The happy
|
|
1170
|
+
// path stops it after the stream loop; this covers throws before that
|
|
1171
|
+
// point so no orphaned timer survives the activity.
|
|
1172
|
+
stallWatchdog?.stop();
|
|
1042
1173
|
// Tear down the HITL gate on EVERY exit path (success, error, approval
|
|
1043
1174
|
// pause, cancellation) so attaching a real repo leaves the user's
|
|
1044
1175
|
// .cursor/hooks.json and workspace untouched between turns (issue #173).
|