@stigmer/runner 3.0.8-dev.20260613074252 → 3.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/activities/execute-cursor/hook-script.d.ts +23 -12
- package/dist/activities/execute-cursor/hook-script.js +85 -51
- package/dist/activities/execute-cursor/hook-script.js.map +1 -1
- package/dist/activities/execute-cursor/index.js +210 -79
- package/dist/activities/execute-cursor/index.js.map +1 -1
- package/dist/activities/execute-cursor/message-translator.d.ts +35 -0
- package/dist/activities/execute-cursor/message-translator.js +114 -6
- package/dist/activities/execute-cursor/message-translator.js.map +1 -1
- package/dist/activities/execute-cursor/prompt-builder.d.ts +25 -0
- package/dist/activities/execute-cursor/prompt-builder.js +54 -0
- package/dist/activities/execute-cursor/prompt-builder.js.map +1 -1
- package/dist/activities/execute-cursor/workspace-setup.d.ts +8 -2
- package/dist/activities/execute-cursor/workspace-setup.js +62 -30
- package/dist/activities/execute-cursor/workspace-setup.js.map +1 -1
- package/dist/activities/execute-deep-agent/index.js +14 -4
- package/dist/activities/execute-deep-agent/index.js.map +1 -1
- package/dist/activities/execute-deep-agent/status-builder-shared.d.ts +0 -1
- package/dist/activities/execute-deep-agent/status-builder-shared.js +32 -8
- package/dist/activities/execute-deep-agent/status-builder-shared.js.map +1 -1
- package/dist/activities/execute-deep-agent/status-builder.js +4 -5
- package/dist/activities/execute-deep-agent/status-builder.js.map +1 -1
- package/dist/activities/execute-deep-agent/streaming-v3.js +3 -4
- package/dist/activities/execute-deep-agent/streaming-v3.js.map +1 -1
- package/dist/activities/execute-deep-agent/streaming.d.ts +8 -0
- package/dist/activities/execute-deep-agent/streaming.js +3 -4
- package/dist/activities/execute-deep-agent/streaming.js.map +1 -1
- package/dist/activities/execute-deep-agent/subagent-tracker.js +4 -5
- package/dist/activities/execute-deep-agent/subagent-tracker.js.map +1 -1
- package/dist/activities/execute-deep-agent/v3-status-builder.js +6 -5
- package/dist/activities/execute-deep-agent/v3-status-builder.js.map +1 -1
- package/dist/config.d.ts +21 -0
- package/dist/config.js +12 -0
- package/dist/config.js.map +1 -1
- package/dist/in-flight.d.ts +35 -0
- package/dist/in-flight.js +61 -0
- package/dist/in-flight.js.map +1 -0
- package/dist/main.js +6 -3
- package/dist/main.js.map +1 -1
- package/dist/runner-manager.d.ts +2 -0
- package/dist/runner-manager.js +90 -29
- package/dist/runner-manager.js.map +1 -1
- package/dist/runner.d.ts +2 -0
- package/dist/runner.js +2 -0
- package/dist/runner.js.map +1 -1
- package/dist/shared/grpc-retry.d.ts +9 -20
- package/dist/shared/grpc-retry.js +9 -52
- package/dist/shared/grpc-retry.js.map +1 -1
- package/dist/shared/stall-watchdog.d.ts +68 -0
- package/dist/shared/stall-watchdog.js +102 -0
- package/dist/shared/stall-watchdog.js.map +1 -0
- package/dist/shared/status-offload.d.ts +84 -0
- package/dist/shared/status-offload.js +292 -0
- package/dist/shared/status-offload.js.map +1 -0
- package/dist/shared/status.d.ts +34 -3
- package/dist/shared/status.js +102 -9
- package/dist/shared/status.js.map +1 -1
- package/package.json +2 -2
- package/src/__tests__/config.test.ts +8 -0
- package/src/__tests__/in-flight.test.ts +84 -0
- package/src/activities/__tests__/classify-tool-approvals.test.ts +1 -0
- package/src/activities/__tests__/discover-mcp-server.test.ts +1 -0
- package/src/activities/execute-cursor/__tests__/build-prompt.test.ts +74 -0
- package/src/activities/execute-cursor/__tests__/hook-script.test.ts +90 -15
- package/src/activities/execute-cursor/__tests__/tool-result-image.test.ts +244 -0
- package/src/activities/execute-cursor/__tests__/workspace-setup.test.ts +53 -4
- package/src/activities/execute-cursor/hook-script.ts +85 -51
- package/src/activities/execute-cursor/index.ts +170 -35
- package/src/activities/execute-cursor/message-translator.ts +113 -6
- package/src/activities/execute-cursor/prompt-builder.ts +59 -0
- package/src/activities/execute-cursor/workspace-setup.ts +76 -44
- package/src/activities/execute-deep-agent/__tests__/index.test.ts +1 -0
- package/src/activities/execute-deep-agent/__tests__/status-builder-shared.test.ts +66 -0
- package/src/activities/execute-deep-agent/__tests__/status-builder.test.ts +6 -3
- package/src/activities/execute-deep-agent/__tests__/streaming-v3.test.ts +70 -0
- package/src/activities/execute-deep-agent/index.ts +16 -4
- package/src/activities/execute-deep-agent/status-builder-shared.ts +27 -5
- package/src/activities/execute-deep-agent/status-builder.ts +3 -5
- package/src/activities/execute-deep-agent/streaming-v3.ts +4 -4
- package/src/activities/execute-deep-agent/streaming.ts +13 -4
- package/src/activities/execute-deep-agent/subagent-tracker.ts +4 -5
- package/src/activities/execute-deep-agent/v3-status-builder.ts +5 -5
- package/src/config.ts +27 -0
- package/src/in-flight.ts +71 -0
- package/src/main.ts +7 -2
- package/src/runner-manager.ts +127 -33
- package/src/runner.ts +6 -0
- package/src/shared/__tests__/artifact-storage.test.ts +1 -0
- package/src/shared/__tests__/grpc-retry-extended.test.ts +6 -144
- package/src/shared/__tests__/grpc-retry.test.ts +5 -134
- package/src/shared/__tests__/stall-watchdog.test.ts +193 -0
- package/src/shared/__tests__/status-offload.test.ts +256 -0
- package/src/shared/__tests__/status.test.ts +199 -0
- package/src/shared/grpc-retry.ts +9 -72
- package/src/shared/stall-watchdog.ts +122 -0
- package/src/shared/status-offload.ts +342 -0
- package/src/shared/status.ts +142 -8
|
@@ -16,6 +16,7 @@ import { AgentMessageSchema, ToolCallSchema } from "@stigmer/protos/ai/stigmer/a
|
|
|
16
16
|
import { ExecutionPhase, InteractionMode, MessageType, ToolCallStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
|
|
17
17
|
import { activityStarted, activityFinished } from "../../idle-watchdog.js";
|
|
18
18
|
import { persistStatus, slimStatus, utcTimestamp } from "../../shared/status.js";
|
|
19
|
+
import type { ToolOutputOffloadContext } from "../../shared/status-offload.js";
|
|
19
20
|
import { publishPlanArtifact } from "../../shared/plan-artifact.js";
|
|
20
21
|
import { classifyTool } from "../../shared/tool-kind.js";
|
|
21
22
|
import type { Config } from "../../config.js";
|
|
@@ -48,6 +49,16 @@ export function createDeepAgentActivities(config: Config) {
|
|
|
48
49
|
|
|
49
50
|
setup = await performSetup({ config, client, executionId, threadId });
|
|
50
51
|
|
|
52
|
+
// Single offload context for every persist in this execution: spill
|
|
53
|
+
// oversized tool outputs (e.g. computer-use screenshots) to artifact
|
|
54
|
+
// storage so the UI can render them, and keep the status under the
|
|
55
|
+
// gRPC cap. Threaded into the streaming loop and reused for the
|
|
56
|
+
// terminal persists below so the guard is never skipped.
|
|
57
|
+
const statusOffload: ToolOutputOffloadContext = {
|
|
58
|
+
artifactStorage: setup.artifactStorage,
|
|
59
|
+
executionId,
|
|
60
|
+
};
|
|
61
|
+
|
|
51
62
|
const initialStatus = create(AgentExecutionStatusSchema, {});
|
|
52
63
|
const statusBuilder = new StatusBuilder(executionId, initialStatus);
|
|
53
64
|
|
|
@@ -77,7 +88,7 @@ export function createDeepAgentActivities(config: Config) {
|
|
|
77
88
|
}),
|
|
78
89
|
],
|
|
79
90
|
});
|
|
80
|
-
await persistStatus(client, executionId, failedStatus);
|
|
91
|
+
await persistStatus(client, executionId, failedStatus, { offload: statusOffload });
|
|
81
92
|
return slimStatus(failedStatus);
|
|
82
93
|
}
|
|
83
94
|
|
|
@@ -113,6 +124,7 @@ export function createDeepAgentActivities(config: Config) {
|
|
|
113
124
|
client,
|
|
114
125
|
initialStatus,
|
|
115
126
|
streamingConfig,
|
|
127
|
+
offload: statusOffload,
|
|
116
128
|
gracefulStop: setup.gracefulStop,
|
|
117
129
|
inlinePublisher,
|
|
118
130
|
writebackCoordinator: writebackCoordinator ?? undefined,
|
|
@@ -137,7 +149,7 @@ export function createDeepAgentActivities(config: Config) {
|
|
|
137
149
|
|
|
138
150
|
if (result.terminalStatus) {
|
|
139
151
|
if (initialStatus.phase === ExecutionPhase.EXECUTION_PAUSED) {
|
|
140
|
-
await persistStatus(client, executionId, initialStatus);
|
|
152
|
+
await persistStatus(client, executionId, initialStatus, { offload: statusOffload });
|
|
141
153
|
console.log(`[ExecuteDeepAgent] Paused for execution ${executionId}: events=${result.eventsProcessed}`);
|
|
142
154
|
throw new CancelledFailure("Activity paused by orchestrator");
|
|
143
155
|
}
|
|
@@ -189,7 +201,7 @@ export function createDeepAgentActivities(config: Config) {
|
|
|
189
201
|
}
|
|
190
202
|
initialStatus.messages.push(aiMsg);
|
|
191
203
|
|
|
192
|
-
await persistStatus(client, executionId, initialStatus);
|
|
204
|
+
await persistStatus(client, executionId, initialStatus, { offload: statusOffload });
|
|
193
205
|
return slimStatus(initialStatus);
|
|
194
206
|
}
|
|
195
207
|
}
|
|
@@ -257,7 +269,7 @@ export function createDeepAgentActivities(config: Config) {
|
|
|
257
269
|
});
|
|
258
270
|
}
|
|
259
271
|
|
|
260
|
-
await persistStatus(client, executionId, initialStatus);
|
|
272
|
+
await persistStatus(client, executionId, initialStatus, { offload: statusOffload });
|
|
261
273
|
|
|
262
274
|
console.log(
|
|
263
275
|
`[ExecuteDeepAgent] Completed for execution ${executionId}: ` +
|
|
@@ -69,14 +69,34 @@ export function toBigInt(value: unknown): bigint {
|
|
|
69
69
|
|
|
70
70
|
// ── Tool Result Extraction ─────────────────────────────────────────
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
/**
|
|
73
|
+
* Serialize a LangChain message `content` field into the canonical tool-result
|
|
74
|
+
* string.
|
|
75
|
+
*
|
|
76
|
+
* Text-only content is a plain string and passes through unchanged. Multimodal
|
|
77
|
+
* content (image, or mixed text+image — e.g. a computer-use screenshot) is an
|
|
78
|
+
* array of content blocks; we serialize the blocks array ITSELF, not the
|
|
79
|
+
* surrounding message envelope, so the result lands in the exact top-level-array
|
|
80
|
+
* shape the persist-time offload (`detectImagePayload`/`contentBlocks` in
|
|
81
|
+
* status-offload.ts) consumes to lift the image out into a renderable
|
|
82
|
+
* `ToolCallOutputRef`. Serializing the envelope instead would bury the base64
|
|
83
|
+
* one level deeper and defeat that detection.
|
|
84
|
+
*
|
|
85
|
+
* Returns undefined when `content` is neither a string nor an array, letting the
|
|
86
|
+
* caller fall back to serializing whatever else it holds.
|
|
87
|
+
*/
|
|
88
|
+
function serializeToolContent(content: unknown): string | undefined {
|
|
89
|
+
if (typeof content === "string") return content;
|
|
90
|
+
if (Array.isArray(content)) return JSON.stringify(content);
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
73
93
|
|
|
74
94
|
export function extractToolResult(data: Record<string, unknown>): string {
|
|
75
95
|
const output = data.output;
|
|
76
96
|
if (typeof output === "string") return output;
|
|
77
97
|
if (typeof output === "object" && output !== null) {
|
|
78
|
-
const
|
|
79
|
-
if (
|
|
98
|
+
const fromContent = serializeToolContent((output as Record<string, unknown>).content);
|
|
99
|
+
if (fromContent !== undefined) return fromContent;
|
|
80
100
|
}
|
|
81
101
|
try {
|
|
82
102
|
return JSON.stringify(output ?? data);
|
|
@@ -96,9 +116,11 @@ export function extractToolResultV3(output: unknown): string {
|
|
|
96
116
|
const obj = output as Record<string, unknown>;
|
|
97
117
|
const kwargs = obj.kwargs as Record<string, unknown> | undefined;
|
|
98
118
|
if (kwargs) {
|
|
99
|
-
|
|
119
|
+
const fromKwargs = serializeToolContent(kwargs.content);
|
|
120
|
+
if (fromKwargs !== undefined) return fromKwargs;
|
|
100
121
|
}
|
|
101
|
-
|
|
122
|
+
const fromContent = serializeToolContent(obj.content);
|
|
123
|
+
if (fromContent !== undefined) return fromContent;
|
|
102
124
|
}
|
|
103
125
|
try {
|
|
104
126
|
return JSON.stringify(output);
|
|
@@ -34,7 +34,6 @@ import {
|
|
|
34
34
|
UsageAccumulator,
|
|
35
35
|
extractToolResult,
|
|
36
36
|
sanitizeArgsPreview,
|
|
37
|
-
MAX_TOOL_RESULT_CHARS,
|
|
38
37
|
} from "./status-builder-shared.js";
|
|
39
38
|
|
|
40
39
|
/** Minimal LangGraph streamEvents v2 event shape. */
|
|
@@ -290,11 +289,10 @@ export class StatusBuilder {
|
|
|
290
289
|
tc.status = ToolCallStatus.TOOL_CALL_FAILED;
|
|
291
290
|
tc.error = errorMsg;
|
|
292
291
|
} else {
|
|
292
|
+
// Faithful result only; payload size is bounded at the persist chokepoint
|
|
293
|
+
// (see v3 builder note). Truncating here would corrupt image base64.
|
|
293
294
|
tc.status = ToolCallStatus.TOOL_CALL_COMPLETED;
|
|
294
|
-
|
|
295
|
-
tc.result = result.length > MAX_TOOL_RESULT_CHARS
|
|
296
|
-
? result.slice(0, MAX_TOOL_RESULT_CHARS) + `\n[truncated: ${result.length} chars total]`
|
|
297
|
-
: result;
|
|
295
|
+
tc.result = extractToolResult(event.data);
|
|
298
296
|
}
|
|
299
297
|
|
|
300
298
|
tc.completedAt = utcTimestamp();
|
|
@@ -17,8 +17,7 @@ import { createV3EventRecorder, type V3ProtocolEvent } from "./v3-event-recorder
|
|
|
17
17
|
import { normalize } from "./v3-protocol-normalizer.js";
|
|
18
18
|
import { V3StatusBuilder } from "./v3-status-builder.js";
|
|
19
19
|
import { StreamingUpdateScheduler } from "./streaming-scheduler.js";
|
|
20
|
-
import {
|
|
21
|
-
import { slimStatus } from "../../shared/status.js";
|
|
20
|
+
import { persistStatus, slimStatus } from "../../shared/status.js";
|
|
22
21
|
import { StreamingSideEffects } from "./streaming-side-effects.js";
|
|
23
22
|
import {
|
|
24
23
|
handlePause,
|
|
@@ -49,6 +48,7 @@ export async function streamExecutionV3(
|
|
|
49
48
|
initialStatus,
|
|
50
49
|
streamingConfig,
|
|
51
50
|
retryOptions,
|
|
51
|
+
offload,
|
|
52
52
|
stallTimeoutMs = DEFAULT_STALL_TIMEOUT_MS,
|
|
53
53
|
heartbeatFn,
|
|
54
54
|
isCancelledFn,
|
|
@@ -118,11 +118,11 @@ export async function streamExecutionV3(
|
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
statusBuilder.syncSubAgentExecutions();
|
|
121
|
-
const signal = await
|
|
121
|
+
const signal = await persistStatus(
|
|
122
122
|
client,
|
|
123
123
|
executionId,
|
|
124
124
|
statusBuilder.currentStatus,
|
|
125
|
-
retryOptions,
|
|
125
|
+
{ offload, retry: retryOptions },
|
|
126
126
|
);
|
|
127
127
|
scheduler.markUpdateSent(eventsProcessed);
|
|
128
128
|
|
|
@@ -33,8 +33,9 @@ import {
|
|
|
33
33
|
StreamingUpdateScheduler,
|
|
34
34
|
type StreamingConfig,
|
|
35
35
|
} from "./streaming-scheduler.js";
|
|
36
|
-
import {
|
|
37
|
-
import { slimStatus, utcTimestamp } from "../../shared/status.js";
|
|
36
|
+
import { type RetryOptions } from "../../shared/grpc-retry.js";
|
|
37
|
+
import { persistStatus, slimStatus, utcTimestamp } from "../../shared/status.js";
|
|
38
|
+
import type { ToolOutputOffloadContext } from "../../shared/status-offload.js";
|
|
38
39
|
import type { StigmerClient } from "../../client/stigmer-client.js";
|
|
39
40
|
import type { GracefulStopMiddleware } from "../../middleware/index.js";
|
|
40
41
|
import type { InlinePublisher } from "./inline-publisher.js";
|
|
@@ -59,6 +60,13 @@ export interface StreamDependencies {
|
|
|
59
60
|
readonly initialStatus: AgentExecutionStatus;
|
|
60
61
|
readonly streamingConfig?: StreamingConfig;
|
|
61
62
|
readonly retryOptions?: RetryOptions;
|
|
63
|
+
/**
|
|
64
|
+
* Offload context for the persist chokepoint. When set, oversized tool
|
|
65
|
+
* outputs (e.g. computer-use screenshots) are spilled to artifact storage so
|
|
66
|
+
* the UI can render them; when omitted, the aggregate size backstop still
|
|
67
|
+
* keeps the payload under the gRPC cap.
|
|
68
|
+
*/
|
|
69
|
+
readonly offload?: ToolOutputOffloadContext;
|
|
62
70
|
readonly stallTimeoutMs?: number;
|
|
63
71
|
/** Temporal heartbeat. Injected so the loop is testable without Temporal. */
|
|
64
72
|
readonly heartbeatFn?: (details: Record<string, unknown>) => void;
|
|
@@ -112,6 +120,7 @@ async function streamExecutionV2(
|
|
|
112
120
|
initialStatus,
|
|
113
121
|
streamingConfig,
|
|
114
122
|
retryOptions,
|
|
123
|
+
offload,
|
|
115
124
|
stallTimeoutMs = DEFAULT_STALL_TIMEOUT_MS,
|
|
116
125
|
heartbeatFn,
|
|
117
126
|
isCancelledFn,
|
|
@@ -179,11 +188,11 @@ async function streamExecutionV2(
|
|
|
179
188
|
statusBuilder.clearForceFlag();
|
|
180
189
|
}
|
|
181
190
|
|
|
182
|
-
const signal = await
|
|
191
|
+
const signal = await persistStatus(
|
|
183
192
|
client,
|
|
184
193
|
executionId,
|
|
185
194
|
statusBuilder.currentStatus,
|
|
186
|
-
retryOptions,
|
|
195
|
+
{ offload, retry: retryOptions },
|
|
187
196
|
);
|
|
188
197
|
scheduler.markUpdateSent(eventsProcessed);
|
|
189
198
|
|
|
@@ -29,7 +29,7 @@ import {
|
|
|
29
29
|
} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
|
|
30
30
|
import { utcTimestamp } from "../../shared/status.js";
|
|
31
31
|
import { classifyTool } from "../../shared/tool-kind.js";
|
|
32
|
-
import { extractToolResultV3
|
|
32
|
+
import { extractToolResultV3 } from "./status-builder-shared.js";
|
|
33
33
|
import type { StigmerRunEvent, V3UsagePayload } from "./v3-events.js";
|
|
34
34
|
|
|
35
35
|
// ── Per-SubAgent State ───────────────────────────────────────────────────────
|
|
@@ -299,11 +299,10 @@ export class SubAgentTracker {
|
|
|
299
299
|
const tc = state.toolCalls.get(callId);
|
|
300
300
|
if (!tc) return;
|
|
301
301
|
|
|
302
|
-
|
|
302
|
+
// Faithful result only; payload size is bounded at the persist chokepoint
|
|
303
|
+
// (see v3 builder note). Truncating here would corrupt image base64.
|
|
303
304
|
tc.status = ToolCallStatus.TOOL_CALL_COMPLETED;
|
|
304
|
-
tc.result =
|
|
305
|
-
? result.slice(0, MAX_TOOL_RESULT_CHARS) + `\n[truncated: ${result.length} chars total]`
|
|
306
|
-
: result;
|
|
305
|
+
tc.result = extractToolResultV3(output);
|
|
307
306
|
tc.completedAt = utcTimestamp();
|
|
308
307
|
tc.isStreaming = false;
|
|
309
308
|
state.toolArgBuffers.delete(callId);
|
|
@@ -36,7 +36,6 @@ import {
|
|
|
36
36
|
UsageAccumulator,
|
|
37
37
|
extractToolResultV3,
|
|
38
38
|
sanitizeArgsPreview,
|
|
39
|
-
MAX_TOOL_RESULT_CHARS,
|
|
40
39
|
} from "./status-builder-shared.js";
|
|
41
40
|
import { SubAgentTracker } from "./subagent-tracker.js";
|
|
42
41
|
|
|
@@ -296,11 +295,12 @@ export class V3StatusBuilder implements ExecutionStatusWriter {
|
|
|
296
295
|
const tc = this.state.toolCalls.get(callId);
|
|
297
296
|
if (!tc) return;
|
|
298
297
|
|
|
299
|
-
|
|
298
|
+
// Store the faithful result; bounding the gRPC payload is owned solely by
|
|
299
|
+
// the persist chokepoint (offload + enforce in status.ts/status-offload.ts).
|
|
300
|
+
// Truncating here would corrupt binary content (e.g. a screenshot's base64)
|
|
301
|
+
// before offload can lift it into a renderable ToolCallOutputRef.
|
|
300
302
|
tc.status = ToolCallStatus.TOOL_CALL_COMPLETED;
|
|
301
|
-
tc.result =
|
|
302
|
-
? result.slice(0, MAX_TOOL_RESULT_CHARS) + `\n[truncated: ${result.length} chars total]`
|
|
303
|
-
: result;
|
|
303
|
+
tc.result = extractToolResultV3(output);
|
|
304
304
|
tc.completedAt = utcTimestamp();
|
|
305
305
|
tc.isStreaming = false;
|
|
306
306
|
this.state.toolStartTimes.delete(callId);
|
package/src/config.ts
CHANGED
|
@@ -28,6 +28,15 @@ import { mkdirSync } from "node:fs";
|
|
|
28
28
|
import { join } from "node:path";
|
|
29
29
|
import { homedir, tmpdir } from "node:os";
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Default no-progress bound for the Cursor harness stream (ms). Larger than the
|
|
33
|
+
* shared DEFAULT_STALL_TIMEOUT_MS (120s) because opaque MCP / GUI tool calls
|
|
34
|
+
* can run for minutes while emitting no stream activity. Single source of truth
|
|
35
|
+
* for env-loaded ({@link loadConfig}) and options-mapped (runner / manager)
|
|
36
|
+
* config so the three construction sites never drift.
|
|
37
|
+
*/
|
|
38
|
+
export const DEFAULT_CURSOR_STREAM_STALL_TIMEOUT_MS = 180_000;
|
|
39
|
+
|
|
31
40
|
export interface Config {
|
|
32
41
|
readonly taskQueue: string;
|
|
33
42
|
readonly temporalAddress: string;
|
|
@@ -61,6 +70,19 @@ export interface Config {
|
|
|
61
70
|
readonly checkpointerType: "memory" | "http";
|
|
62
71
|
readonly checkpointerProxyEndpoint: string | null;
|
|
63
72
|
readonly primaryModel: string;
|
|
73
|
+
/**
|
|
74
|
+
* No-progress bound for the Cursor harness stream (milliseconds). If no
|
|
75
|
+
* stream event or token delta arrives for this long, the stall watchdog
|
|
76
|
+
* (see activities/execute-cursor + shared/stall-watchdog.ts) cancels the run
|
|
77
|
+
* and fails the execution with a StallTimeoutError rather than hanging at
|
|
78
|
+
* EXECUTION_IN_PROGRESS forever.
|
|
79
|
+
*
|
|
80
|
+
* Larger than the shared DEFAULT_STALL_TIMEOUT_MS (120s) because opaque MCP /
|
|
81
|
+
* GUI tool calls can legitimately run for minutes while emitting no stream
|
|
82
|
+
* activity. This bounds no-progress time only; it is orthogonal to Temporal's
|
|
83
|
+
* heartbeatTimeout (process liveness) and the 30s keep-alive heartbeat.
|
|
84
|
+
*/
|
|
85
|
+
readonly cursorStreamStallTimeoutMs: number;
|
|
64
86
|
/** Shared mutable token reference for dynamic token updates (manager mode). */
|
|
65
87
|
readonly stigmerTokenRef?: { current: string | null };
|
|
66
88
|
}
|
|
@@ -119,6 +141,10 @@ export function loadConfig(): Config {
|
|
|
119
141
|
|
|
120
142
|
const primaryModel = process.env.STIGMER_PRIMARY_MODEL ?? "gpt-4.1";
|
|
121
143
|
|
|
144
|
+
const cursorStreamStallTimeoutMs = process.env.CURSOR_STREAM_STALL_TIMEOUT_MS
|
|
145
|
+
? parseInt(process.env.CURSOR_STREAM_STALL_TIMEOUT_MS, 10)
|
|
146
|
+
: DEFAULT_CURSOR_STREAM_STALL_TIMEOUT_MS;
|
|
147
|
+
|
|
122
148
|
return {
|
|
123
149
|
taskQueue,
|
|
124
150
|
temporalAddress,
|
|
@@ -135,6 +161,7 @@ export function loadConfig(): Config {
|
|
|
135
161
|
checkpointerType,
|
|
136
162
|
checkpointerProxyEndpoint,
|
|
137
163
|
primaryModel,
|
|
164
|
+
cursorStreamStallTimeoutMs,
|
|
138
165
|
};
|
|
139
166
|
}
|
|
140
167
|
|
package/src/in-flight.ts
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-task-queue in-flight activity tracking.
|
|
3
|
+
*
|
|
4
|
+
* A session/wfexec worker must NOT be torn down while one of its activities
|
|
5
|
+
* (notably the long-running ExecuteCursor) is still running: doing so abandons
|
|
6
|
+
* the activity and removes the only poller on its queue, turning a harmless UI
|
|
7
|
+
* navigation into an "Activity task timed out" dead-end. The activity inbound
|
|
8
|
+
* interceptor increments on every activity start and decrements on finish;
|
|
9
|
+
* runner-manager's removeSession/removeWorkflowExecution consult the count and
|
|
10
|
+
* defer teardown until it drains.
|
|
11
|
+
*
|
|
12
|
+
* This lives in its own module (rather than inside the runner-manager closure)
|
|
13
|
+
* so the interceptor — which has no handle to that closure — and unit tests can
|
|
14
|
+
* both reach it. Keyed by task queue (`session:{id}` / `wfexec:{id}`), it is
|
|
15
|
+
* process-global, mirroring the shutdown-signal registry.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
interface InFlightEntry {
|
|
19
|
+
count: number;
|
|
20
|
+
/** Invoked once when count returns to 0; used to run a deferred teardown. */
|
|
21
|
+
onDrained?: () => void;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const registry = new Map<string, InFlightEntry>();
|
|
25
|
+
|
|
26
|
+
/** Record that an activity started on the given task queue. */
|
|
27
|
+
export function activityStartedOnQueue(taskQueue: string): void {
|
|
28
|
+
const entry = registry.get(taskQueue) ?? { count: 0 };
|
|
29
|
+
entry.count++;
|
|
30
|
+
registry.set(taskQueue, entry);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Record that an activity finished on the given task queue. When the count
|
|
35
|
+
* returns to zero, any registered drain callback fires exactly once.
|
|
36
|
+
*/
|
|
37
|
+
export function activityFinishedOnQueue(taskQueue: string): void {
|
|
38
|
+
const entry = registry.get(taskQueue);
|
|
39
|
+
if (!entry) return;
|
|
40
|
+
entry.count = Math.max(0, entry.count - 1);
|
|
41
|
+
if (entry.count === 0 && entry.onDrained) {
|
|
42
|
+
const onDrained = entry.onDrained;
|
|
43
|
+
entry.onDrained = undefined;
|
|
44
|
+
onDrained();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Current number of in-flight activities on the task queue (0 if unknown). */
|
|
49
|
+
export function inFlightCountForQueue(taskQueue: string): number {
|
|
50
|
+
return registry.get(taskQueue)?.count ?? 0;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Register (or clear) a callback to run when the queue next drains to zero
|
|
55
|
+
* in-flight activities. No-op if the queue has no entry (count already 0); the
|
|
56
|
+
* caller handles the already-idle case by tearing down immediately.
|
|
57
|
+
*/
|
|
58
|
+
export function setQueueDrainCallback(taskQueue: string, cb: (() => void) | undefined): void {
|
|
59
|
+
const entry = registry.get(taskQueue);
|
|
60
|
+
if (entry) entry.onDrained = cb;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Forget all tracking for a queue (called after its worker is torn down). */
|
|
64
|
+
export function forgetQueue(taskQueue: string): void {
|
|
65
|
+
registry.delete(taskQueue);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Test-only: clear the entire registry between cases. */
|
|
69
|
+
export function __resetInFlightRegistryForTests(): void {
|
|
70
|
+
registry.clear();
|
|
71
|
+
}
|
package/src/main.ts
CHANGED
|
@@ -319,9 +319,14 @@ async function main(): Promise<void> {
|
|
|
319
319
|
|
|
320
320
|
if (runnerMode === "manager") {
|
|
321
321
|
await runManagerMode(config);
|
|
322
|
-
|
|
323
|
-
|
|
322
|
+
// Manager mode is driven by the host over stdin. Once it returns — via the IPC `shutdown`
|
|
323
|
+
// command or stdin EOF when the host process dies — exit deterministically so a stray open
|
|
324
|
+
// handle can't keep a shut-down runner alive as an orphan (issue #177). Static mode is left
|
|
325
|
+
// to exit naturally so its OTel flush completes.
|
|
326
|
+
process.exit(0);
|
|
324
327
|
}
|
|
328
|
+
|
|
329
|
+
await runStaticMode(config);
|
|
325
330
|
}
|
|
326
331
|
|
|
327
332
|
main().catch((err) => {
|
package/src/runner-manager.ts
CHANGED
|
@@ -24,10 +24,22 @@ import {
|
|
|
24
24
|
} from "@temporalio/worker";
|
|
25
25
|
import type { PayloadCodec } from "@temporalio/common";
|
|
26
26
|
import type { Config } from "./config.js";
|
|
27
|
+
import { DEFAULT_CURSOR_STREAM_STALL_TIMEOUT_MS } from "./config.js";
|
|
27
28
|
import type { WorkerActivities } from "./worker.js";
|
|
28
29
|
import { resolveWorkflowSource, OTEL_WORKFLOW_INTERCEPTOR_MODULE } from "./workflow-source.js";
|
|
29
30
|
import { resolveRunnerBootstrap, refreshRunnerAccessToken } from "./bootstrap.js";
|
|
30
31
|
import { createRunnerTokenCoordinator } from "./runner-token-coordinator.js";
|
|
32
|
+
// Per-task-queue in-flight activity tracking lives in ./in-flight.ts so the
|
|
33
|
+
// activity interceptor (no manager-closure handle) and unit tests can reach it.
|
|
34
|
+
// This is what keeps a session worker alive while ExecuteCursor is running, so a
|
|
35
|
+
// view close can no longer reap the worker mid-run.
|
|
36
|
+
import {
|
|
37
|
+
activityStartedOnQueue,
|
|
38
|
+
activityFinishedOnQueue,
|
|
39
|
+
inFlightCountForQueue,
|
|
40
|
+
setQueueDrainCallback,
|
|
41
|
+
forgetQueue,
|
|
42
|
+
} from "./in-flight.js";
|
|
31
43
|
|
|
32
44
|
const SESSION_QUEUE_PREFIX = "session:";
|
|
33
45
|
const WFEXEC_QUEUE_PREFIX = "wfexec:";
|
|
@@ -76,6 +88,9 @@ export interface RunnerManagerOptions {
|
|
|
76
88
|
/** Default LLM model identifier. @default "gpt-4.1" */
|
|
77
89
|
readonly primaryModel?: string;
|
|
78
90
|
|
|
91
|
+
/** No-progress bound for the Cursor harness stream (ms). @default 180000 */
|
|
92
|
+
readonly cursorStreamStallTimeoutMs?: number;
|
|
93
|
+
|
|
79
94
|
/** Checkpointer type for LangGraph agent state. @default "memory" (or "http" if proxyEndpoint is set) */
|
|
80
95
|
readonly checkpointerType?: "memory" | "http";
|
|
81
96
|
|
|
@@ -141,6 +156,12 @@ interface ManagedSession {
|
|
|
141
156
|
worker: Worker;
|
|
142
157
|
runPromise: Promise<void>;
|
|
143
158
|
shutdownController: AbortController;
|
|
159
|
+
/**
|
|
160
|
+
* Set when a remove was requested while an activity was still in flight, so
|
|
161
|
+
* the worker is kept alive in the background and torn down only once the last
|
|
162
|
+
* activity finishes. Cleared if the session is re-opened before it drains.
|
|
163
|
+
*/
|
|
164
|
+
pendingClose: boolean;
|
|
144
165
|
}
|
|
145
166
|
|
|
146
167
|
/**
|
|
@@ -324,7 +345,84 @@ export async function createStigmerRunnerManager(
|
|
|
324
345
|
);
|
|
325
346
|
});
|
|
326
347
|
|
|
327
|
-
return { worker, runPromise, shutdownController };
|
|
348
|
+
return { worker, runPromise, shutdownController, pendingClose: false };
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Re-opening a session/execution whose teardown was deferred: cancel the
|
|
353
|
+
* pending close so the worker keeps serving the reused queue. Returns true
|
|
354
|
+
* when an existing managed worker was found (caller should not recreate one).
|
|
355
|
+
*/
|
|
356
|
+
function reuseExistingWorker(
|
|
357
|
+
registry: Map<string, ManagedSession>,
|
|
358
|
+
id: string,
|
|
359
|
+
taskQueue: string,
|
|
360
|
+
kind: string,
|
|
361
|
+
): boolean {
|
|
362
|
+
const existing = registry.get(id);
|
|
363
|
+
if (!existing) return false;
|
|
364
|
+
if (existing.pendingClose) {
|
|
365
|
+
existing.pendingClose = false;
|
|
366
|
+
setQueueDrainCallback(taskQueue, undefined);
|
|
367
|
+
console.log(`[runner-manager] Re-opened ${kind} ${id}; cancelled deferred teardown`);
|
|
368
|
+
}
|
|
369
|
+
return true;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* Graceful teardown of a managed worker. Never aborts: callers only reach the
|
|
374
|
+
* actual teardown once no activity is in flight, so a plain worker.shutdown()
|
|
375
|
+
* drains the (idle) queue cleanly. The abort-before-shutdown that used to live
|
|
376
|
+
* here is what killed running activities on a view close.
|
|
377
|
+
*/
|
|
378
|
+
async function teardownManaged(
|
|
379
|
+
registry: Map<string, ManagedSession>,
|
|
380
|
+
id: string,
|
|
381
|
+
taskQueue: string,
|
|
382
|
+
kind: string,
|
|
383
|
+
): Promise<void> {
|
|
384
|
+
const managed = registry.get(id);
|
|
385
|
+
if (!managed) return;
|
|
386
|
+
managed.worker.shutdown();
|
|
387
|
+
await managed.runPromise;
|
|
388
|
+
registry.delete(id);
|
|
389
|
+
shutdownSignals.delete(taskQueue);
|
|
390
|
+
_shutdownSignalRegistry.delete(taskQueue);
|
|
391
|
+
forgetQueue(taskQueue);
|
|
392
|
+
console.log(`[runner-manager] Removed ${kind} ${id} (active=${registry.size})`);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Remove a managed worker, deferring teardown while activities are in flight.
|
|
397
|
+
* This is the server-side safety invariant that lets a run continue in the
|
|
398
|
+
* background after its session view closes: the worker is reaped only when the
|
|
399
|
+
* last activity finishes (or immediately if the queue is already idle).
|
|
400
|
+
*/
|
|
401
|
+
async function removeManaged(
|
|
402
|
+
registry: Map<string, ManagedSession>,
|
|
403
|
+
id: string,
|
|
404
|
+
taskQueue: string,
|
|
405
|
+
kind: string,
|
|
406
|
+
): Promise<void> {
|
|
407
|
+
const managed = registry.get(id);
|
|
408
|
+
if (!managed) return;
|
|
409
|
+
|
|
410
|
+
if (inFlightCountForQueue(taskQueue) > 0) {
|
|
411
|
+
managed.pendingClose = true;
|
|
412
|
+
setQueueDrainCallback(taskQueue, () => {
|
|
413
|
+
// Skip if a full shutdown() is already reaping every worker, so we
|
|
414
|
+
// never call worker.shutdown() twice on the same worker.
|
|
415
|
+
if (shuttingDown) return;
|
|
416
|
+
void teardownManaged(registry, id, taskQueue, kind);
|
|
417
|
+
});
|
|
418
|
+
console.log(
|
|
419
|
+
`[runner-manager] Deferring teardown of ${kind} ${id} — ` +
|
|
420
|
+
`${inFlightCountForQueue(taskQueue)} activity(ies) still in flight (runs in background)`,
|
|
421
|
+
);
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
await teardownManaged(registry, id, taskQueue, kind);
|
|
328
426
|
}
|
|
329
427
|
|
|
330
428
|
return {
|
|
@@ -332,11 +430,11 @@ export async function createStigmerRunnerManager(
|
|
|
332
430
|
if (shuttingDown) {
|
|
333
431
|
throw new Error("RunnerManager is shutting down");
|
|
334
432
|
}
|
|
335
|
-
|
|
433
|
+
const taskQueue = SESSION_QUEUE_PREFIX + sessionId;
|
|
434
|
+
if (reuseExistingWorker(sessions, sessionId, taskQueue, "session")) {
|
|
336
435
|
return;
|
|
337
436
|
}
|
|
338
437
|
|
|
339
|
-
const taskQueue = SESSION_QUEUE_PREFIX + sessionId;
|
|
340
438
|
const managed = await createWorkerOnQueue(taskQueue);
|
|
341
439
|
sessions.set(sessionId, managed);
|
|
342
440
|
console.log(
|
|
@@ -345,20 +443,8 @@ export async function createStigmerRunnerManager(
|
|
|
345
443
|
},
|
|
346
444
|
|
|
347
445
|
async removeSession(sessionId: string): Promise<void> {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
return;
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
const taskQueue = SESSION_QUEUE_PREFIX + sessionId;
|
|
354
|
-
session.shutdownController.abort();
|
|
355
|
-
session.worker.shutdown();
|
|
356
|
-
await session.runPromise;
|
|
357
|
-
sessions.delete(sessionId);
|
|
358
|
-
shutdownSignals.delete(taskQueue);
|
|
359
|
-
_shutdownSignalRegistry.delete(taskQueue);
|
|
360
|
-
console.log(
|
|
361
|
-
`[runner-manager] Removed session ${sessionId} (active=${sessions.size})`,
|
|
446
|
+
await removeManaged(
|
|
447
|
+
sessions, sessionId, SESSION_QUEUE_PREFIX + sessionId, "session",
|
|
362
448
|
);
|
|
363
449
|
},
|
|
364
450
|
|
|
@@ -370,11 +456,11 @@ export async function createStigmerRunnerManager(
|
|
|
370
456
|
if (shuttingDown) {
|
|
371
457
|
throw new Error("RunnerManager is shutting down");
|
|
372
458
|
}
|
|
373
|
-
|
|
459
|
+
const taskQueue = WFEXEC_QUEUE_PREFIX + executionId;
|
|
460
|
+
if (reuseExistingWorker(workflowExecutions, executionId, taskQueue, "workflow execution")) {
|
|
374
461
|
return;
|
|
375
462
|
}
|
|
376
463
|
|
|
377
|
-
const taskQueue = WFEXEC_QUEUE_PREFIX + executionId;
|
|
378
464
|
const managed = await createWorkerOnQueue(taskQueue);
|
|
379
465
|
workflowExecutions.set(executionId, managed);
|
|
380
466
|
console.log(
|
|
@@ -383,20 +469,9 @@ export async function createStigmerRunnerManager(
|
|
|
383
469
|
},
|
|
384
470
|
|
|
385
471
|
async removeWorkflowExecution(executionId: string): Promise<void> {
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
const taskQueue = WFEXEC_QUEUE_PREFIX + executionId;
|
|
392
|
-
execution.shutdownController.abort();
|
|
393
|
-
execution.worker.shutdown();
|
|
394
|
-
await execution.runPromise;
|
|
395
|
-
workflowExecutions.delete(executionId);
|
|
396
|
-
shutdownSignals.delete(taskQueue);
|
|
397
|
-
_shutdownSignalRegistry.delete(taskQueue);
|
|
398
|
-
console.log(
|
|
399
|
-
`[runner-manager] Removed workflow execution ${executionId} (active=${workflowExecutions.size})`,
|
|
472
|
+
await removeManaged(
|
|
473
|
+
workflowExecutions, executionId, WFEXEC_QUEUE_PREFIX + executionId,
|
|
474
|
+
"workflow execution",
|
|
400
475
|
);
|
|
401
476
|
},
|
|
402
477
|
|
|
@@ -499,6 +574,8 @@ export function mapManagerOptionsToConfig(
|
|
|
499
574
|
checkpointerProxyEndpoint:
|
|
500
575
|
options.checkpointerProxyEndpoint ?? options.proxyEndpoint ?? null,
|
|
501
576
|
primaryModel: options.primaryModel ?? "gpt-4.1",
|
|
577
|
+
cursorStreamStallTimeoutMs:
|
|
578
|
+
options.cursorStreamStallTimeoutMs ?? DEFAULT_CURSOR_STREAM_STALL_TIMEOUT_MS,
|
|
502
579
|
};
|
|
503
580
|
}
|
|
504
581
|
|
|
@@ -600,6 +677,23 @@ async function buildInterceptorConfig(): Promise<InterceptorConfig> {
|
|
|
600
677
|
let sinks: InjectedSinks<any> = { ...createWorkflowMetricsSinks() };
|
|
601
678
|
const workflowInterceptorModules: string[] = [];
|
|
602
679
|
|
|
680
|
+
// In-flight activity counter: keeps a session/wfexec worker alive while one of
|
|
681
|
+
// its activities (notably the long ExecuteCursor) is running, so a view close
|
|
682
|
+
// can no longer reap the worker mid-run. Always installed; cheap and global.
|
|
683
|
+
activityInterceptors.push((ctx) => ({
|
|
684
|
+
inbound: {
|
|
685
|
+
async execute(input, next) {
|
|
686
|
+
const taskQueue = ctx.info.taskQueue;
|
|
687
|
+
activityStartedOnQueue(taskQueue);
|
|
688
|
+
try {
|
|
689
|
+
return await next(input);
|
|
690
|
+
} finally {
|
|
691
|
+
activityFinishedOnQueue(taskQueue);
|
|
692
|
+
}
|
|
693
|
+
},
|
|
694
|
+
},
|
|
695
|
+
}));
|
|
696
|
+
|
|
603
697
|
if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) {
|
|
604
698
|
const { OpenTelemetryActivityInboundInterceptor, makeWorkflowExporter } =
|
|
605
699
|
await import("@temporalio/interceptors-opentelemetry");
|