@code-yeongyu/senpi-codemode 2026.9.9 → 2026.9.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +17 -2
- package/package.json +4 -4
- package/src/config/settings.ts +35 -15
- package/src/index.ts +13 -2
- package/src/prompt/eval-prompt-template.ts +76 -0
- package/src/prompt/eval-prompt.ts +6 -77
- package/src/timeouts/run-budget.ts +107 -0
- package/src/tool/cell-deadlines.ts +76 -0
- package/src/tool/cell-execution.ts +25 -14
- package/src/tool/detached-cell-contract.ts +4 -0
- package/src/tool/detached-cell-manager.ts +47 -40
- package/src/tool/detached-cell-notification.ts +2 -0
- package/src/tool/detached-cell-snapshot.ts +5 -0
- package/src/tool/eval-tool-options.ts +8 -3
- package/src/tool/eval-tool.ts +17 -187
- package/src/tool/run-eval-cell.ts +189 -0
- package/src/tool/types.ts +66 -56
|
@@ -13,21 +13,28 @@ export const defaultTimeoutFactory: EvalTimeoutFactory = {
|
|
|
13
13
|
},
|
|
14
14
|
};
|
|
15
15
|
|
|
16
|
+
export interface CellIdleWatchdogOptions {
|
|
17
|
+
readonly timeoutMs: number;
|
|
18
|
+
/** Caps how long a host-bridge pause may suspend the idle watchdog (the foreground window for a detaching cell). */
|
|
19
|
+
readonly maxPauseGraceMs: number;
|
|
20
|
+
readonly onTimeout: (error: Error) => void;
|
|
21
|
+
}
|
|
22
|
+
|
|
16
23
|
export interface CellExecutionOptions {
|
|
17
24
|
readonly callerSignal: AbortSignal;
|
|
18
25
|
readonly cellId: string;
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
* Caps how long a host-bridge pause may suspend the idle watchdog. When the cell will detach on
|
|
22
|
-
* timeout this is set to the foreground window so a bridge-parked cell still frees the turn at the
|
|
23
|
-
* window; left undefined (error mode) it keeps the idle-timeout default grace.
|
|
24
|
-
*/
|
|
25
|
-
readonly maxPauseGraceMs?: number;
|
|
26
|
+
/** Interactive calls detach at this idle watchdog; a call that never detaches is bounded by its deadlines alone. */
|
|
27
|
+
readonly idle?: CellIdleWatchdogOptions;
|
|
26
28
|
readonly timeoutFactory: EvalTimeoutFactory;
|
|
27
|
-
readonly onTimeout: (error: Error) => void;
|
|
28
29
|
readonly onAbort: (error: Error) => void;
|
|
29
30
|
}
|
|
30
31
|
|
|
32
|
+
const NO_WATCHDOG: TimeoutPauseHandle & { dispose(): void } = {
|
|
33
|
+
pause(): void {},
|
|
34
|
+
resume(): void {},
|
|
35
|
+
dispose(): void {},
|
|
36
|
+
};
|
|
37
|
+
|
|
31
38
|
export class CellExecution {
|
|
32
39
|
readonly #callerSignal: AbortSignal;
|
|
33
40
|
readonly #onAbort: (error: Error) => void;
|
|
@@ -49,12 +56,16 @@ export class CellExecution {
|
|
|
49
56
|
this.#detachedPromise = new Promise<void>((resolve) => {
|
|
50
57
|
this.#resolveDetached = resolve;
|
|
51
58
|
});
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
59
|
+
const idle = options.idle;
|
|
60
|
+
this.#watchdog =
|
|
61
|
+
idle === undefined
|
|
62
|
+
? NO_WATCHDOG
|
|
63
|
+
: options.timeoutFactory.create({
|
|
64
|
+
cellId: options.cellId,
|
|
65
|
+
timeoutMs: idle.timeoutMs,
|
|
66
|
+
maxPauseGraceMs: idle.maxPauseGraceMs,
|
|
67
|
+
onTimeout: ({ error }) => idle.onTimeout(error),
|
|
68
|
+
});
|
|
58
69
|
this.#callerSignal.addEventListener("abort", this.#handleCallerAbort, {
|
|
59
70
|
once: true,
|
|
60
71
|
});
|
|
@@ -15,6 +15,8 @@ export interface EvalDetachedCellSnapshot {
|
|
|
15
15
|
readonly interruptNote?: string;
|
|
16
16
|
/** Set only when the wall-clock kill deadline ended this cell. */
|
|
17
17
|
readonly hardLimitSeconds?: number;
|
|
18
|
+
/** Set only when the cell's own execution time exhausted its run budget. */
|
|
19
|
+
readonly runBudgetSeconds?: number;
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
export interface EvalDetachedCellNotification {
|
|
@@ -38,6 +40,8 @@ export interface EvalDetachedCellManagerOptions {
|
|
|
38
40
|
readonly notifier?: EvalDetachedCellNotifier;
|
|
39
41
|
/** Wall-clock kill deadline in seconds; defaults to the bash-parity 1800s. */
|
|
40
42
|
readonly hardLimitSeconds?: number;
|
|
43
|
+
/** Kill deadline for a cell's own execution time in seconds; a per-call `timeout` replaces it. Defaults to 300s. */
|
|
44
|
+
readonly runBudgetSeconds?: number;
|
|
41
45
|
readonly onStatusChange?: (entries: readonly EvalDetachedCellStatusEntry[]) => void;
|
|
42
46
|
/** Receives a full per-source liveness snapshot on every detached-cell transition; used by the goal builtin. */
|
|
43
47
|
readonly onWakeSourceState?: (state: WakeSourceState) => void;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AgentToolResult } from "@code-yeongyu/senpi";
|
|
2
|
-
import { DEFAULT_HARD_LIMIT_SECONDS } from "../config/settings.ts";
|
|
2
|
+
import { DEFAULT_HARD_LIMIT_SECONDS, DEFAULT_RUN_BUDGET_SECONDS } from "../config/settings.ts";
|
|
3
3
|
import type { WakeSourceState } from "../extension/wake-source-state.ts";
|
|
4
|
+
import { type CellDeadlineExpiry, CellDeadlines } from "./cell-deadlines.ts";
|
|
4
5
|
import type {
|
|
5
6
|
EvalDetachedCellManagerOptions,
|
|
6
7
|
EvalDetachedCellSnapshot,
|
|
@@ -46,18 +47,15 @@ type ManagedCell = {
|
|
|
46
47
|
liveResult: LiveResultProvider | undefined;
|
|
47
48
|
terminalResult: AgentToolResult<EvalToolDetails> | undefined;
|
|
48
49
|
notificationQueued: boolean;
|
|
49
|
-
|
|
50
|
-
|
|
50
|
+
readonly deadlines: CellDeadlines;
|
|
51
|
+
readonly hardLimitSeconds: number;
|
|
52
|
+
readonly runBudgetSeconds: number;
|
|
51
53
|
hardLimited: boolean;
|
|
52
|
-
|
|
54
|
+
runBudgetExhausted: boolean;
|
|
55
|
+
/** Foreground killer: the still-awaited CellExecution owns interrupting and rejecting its own call; bound from creation so a deadline firing during kernel boot still ends it. */
|
|
56
|
+
onKill: ((error: Error) => void) | undefined;
|
|
53
57
|
};
|
|
54
58
|
|
|
55
|
-
export function hardLimitError(cellId: string, hardLimitSeconds: number): Error {
|
|
56
|
-
const error = new Error(`Eval cell ${cellId} was killed at the ${hardLimitSeconds}s hard limit.`);
|
|
57
|
-
error.name = "TimeoutError";
|
|
58
|
-
return error;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
59
|
export class EvalDetachedCellManager {
|
|
62
60
|
readonly #artifactsDir: string | undefined;
|
|
63
61
|
readonly #onStatusChange: ((entries: readonly EvalDetachedCellStatusEntry[]) => void) | undefined;
|
|
@@ -67,6 +65,7 @@ export class EvalDetachedCellManager {
|
|
|
67
65
|
readonly #notificationQueue: DetachedNotificationQueue;
|
|
68
66
|
readonly #now: () => number;
|
|
69
67
|
readonly #hardLimitSeconds: number;
|
|
68
|
+
readonly #runBudgetSeconds: number;
|
|
70
69
|
|
|
71
70
|
constructor(options: EvalDetachedCellManagerOptions = {}) {
|
|
72
71
|
this.#artifactsDir = options.artifactsDir;
|
|
@@ -75,14 +74,18 @@ export class EvalDetachedCellManager {
|
|
|
75
74
|
this.#notificationQueue = new DetachedNotificationQueue(options.notifier);
|
|
76
75
|
this.#now = options.now ?? Date.now;
|
|
77
76
|
this.#hardLimitSeconds = options.hardLimitSeconds ?? DEFAULT_HARD_LIMIT_SECONDS;
|
|
77
|
+
this.#runBudgetSeconds = options.runBudgetSeconds ?? DEFAULT_RUN_BUDGET_SECONDS;
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
-
create(cellId: string, input: EvalToolInput): ManagedCell {
|
|
80
|
+
create(cellId: string, input: EvalToolInput, onKill?: (error: Error) => void): ManagedCell {
|
|
81
81
|
const existing = this.#cells.get(cellId);
|
|
82
82
|
if (existing !== undefined) {
|
|
83
83
|
if (detachedCellIsActive(existing.state)) throw activeDetachedCellReuseError(existing);
|
|
84
84
|
this.#cells.delete(cellId);
|
|
85
85
|
}
|
|
86
|
+
// An explicit longer per-call timeout raises the deadline, mirroring bash keeping explicit timeouts.
|
|
87
|
+
const hardLimitSeconds = Math.max(this.#hardLimitSeconds, input.timeout ?? 0);
|
|
88
|
+
const runBudgetSeconds = input.timeout ?? this.#runBudgetSeconds;
|
|
86
89
|
const cell: ManagedCell = {
|
|
87
90
|
cellId,
|
|
88
91
|
input,
|
|
@@ -98,15 +101,23 @@ export class EvalDetachedCellManager {
|
|
|
98
101
|
liveResult: undefined,
|
|
99
102
|
terminalResult: undefined,
|
|
100
103
|
notificationQueued: false,
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
+
deadlines: new CellDeadlines({
|
|
105
|
+
cellId,
|
|
106
|
+
hardLimitSeconds,
|
|
107
|
+
runBudgetSeconds,
|
|
108
|
+
onExpire: (expiry) => {
|
|
109
|
+
const managed = this.#cells.get(cellId);
|
|
110
|
+
if (managed !== undefined) void this.#expireDeadline(managed, expiry);
|
|
111
|
+
},
|
|
112
|
+
}),
|
|
113
|
+
hardLimitSeconds,
|
|
114
|
+
runBudgetSeconds,
|
|
104
115
|
hardLimited: false,
|
|
105
|
-
|
|
116
|
+
runBudgetExhausted: false,
|
|
117
|
+
onKill,
|
|
106
118
|
terminal: Promise.withResolvers<EvalDetachedCellSnapshot>(),
|
|
107
119
|
};
|
|
108
120
|
this.#cells.set(cellId, cell);
|
|
109
|
-
this.#armHardLimit(cell);
|
|
110
121
|
return cell;
|
|
111
122
|
}
|
|
112
123
|
|
|
@@ -114,16 +125,24 @@ export class EvalDetachedCellManager {
|
|
|
114
125
|
cell: ManagedCell,
|
|
115
126
|
kernel: EvalKernel,
|
|
116
127
|
liveResult: LiveResultProvider,
|
|
117
|
-
|
|
118
|
-
onHardLimit?: (error: Error) => void,
|
|
128
|
+
onKill?: (error: Error) => void,
|
|
119
129
|
): void {
|
|
120
130
|
if (cell.state !== "running") return;
|
|
121
|
-
cell.
|
|
131
|
+
cell.onKill = onKill ?? cell.onKill;
|
|
122
132
|
cell.kernel = kernel;
|
|
123
133
|
cell.liveResult = liveResult;
|
|
124
134
|
cell.canDetach = true;
|
|
125
135
|
}
|
|
126
136
|
|
|
137
|
+
/** A host bridge call is in flight for this cell; its run budget stops charging until {@link resume}. */
|
|
138
|
+
pause(cell: ManagedCell): void {
|
|
139
|
+
cell.deadlines.pause();
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
resume(cell: ManagedCell): void {
|
|
143
|
+
cell.deadlines.resume();
|
|
144
|
+
}
|
|
145
|
+
|
|
127
146
|
detach(cell: ManagedCell): boolean {
|
|
128
147
|
if (!cell.canDetach || !allowsDetachedCellTransition(cell.state, "detached")) return false;
|
|
129
148
|
cell.state = "detached";
|
|
@@ -184,7 +203,7 @@ export class EvalDetachedCellManager {
|
|
|
184
203
|
result: AgentToolResult<EvalToolDetails>,
|
|
185
204
|
): boolean {
|
|
186
205
|
if (!allowsDetachedCellTransition(cell.state, state)) return false;
|
|
187
|
-
|
|
206
|
+
cell.deadlines.clear();
|
|
188
207
|
cell.state = state;
|
|
189
208
|
cell.terminalResult = result;
|
|
190
209
|
cell.liveResult = undefined;
|
|
@@ -207,32 +226,20 @@ export class EvalDetachedCellManager {
|
|
|
207
226
|
return true;
|
|
208
227
|
}
|
|
209
228
|
|
|
210
|
-
#armHardLimit(cell: ManagedCell): void {
|
|
211
|
-
const timer = setTimeout(() => void this.#expireHardLimit(cell), cell.hardLimitSeconds * 1_000);
|
|
212
|
-
timer.unref?.();
|
|
213
|
-
cell.hardLimitTimer = timer;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
#clearHardLimit(cell: ManagedCell): void {
|
|
217
|
-
if (cell.hardLimitTimer === undefined) return;
|
|
218
|
-
clearTimeout(cell.hardLimitTimer);
|
|
219
|
-
cell.hardLimitTimer = undefined;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
229
|
/**
|
|
223
|
-
*
|
|
224
|
-
*
|
|
230
|
+
* A kill deadline fired (see {@link CellDeadlines}). A foreground cell is killed through the
|
|
231
|
+
* CellExecution that still awaits it; a detached cell is cancelled here, which interrupts its kernel.
|
|
225
232
|
*/
|
|
226
|
-
async #
|
|
233
|
+
async #expireDeadline(cell: ManagedCell, expiry: CellDeadlineExpiry): Promise<void> {
|
|
227
234
|
if (!detachedCellIsActive(cell.state)) return;
|
|
228
|
-
const foreground = cell.state === "running" && cell.
|
|
229
|
-
cell.hardLimited =
|
|
230
|
-
|
|
235
|
+
const foreground = cell.state === "running" && cell.onKill !== undefined;
|
|
236
|
+
cell.hardLimited = expiry.kind === "hard-limit";
|
|
237
|
+
cell.runBudgetExhausted = expiry.kind === "run-budget";
|
|
231
238
|
if (foreground) {
|
|
232
|
-
if (this.#settle(cell, "cancelled", currentDetachedResult(cell))) cell.
|
|
239
|
+
if (this.#settle(cell, "cancelled", currentDetachedResult(cell))) cell.onKill?.(expiry.error);
|
|
233
240
|
return;
|
|
234
241
|
}
|
|
235
|
-
await this.#cancel(cell, error.message);
|
|
242
|
+
await this.#cancel(cell, expiry.error.message);
|
|
236
243
|
}
|
|
237
244
|
|
|
238
245
|
async #cancel(cell: ManagedCell, reason: string): Promise<void> {
|
|
@@ -69,6 +69,8 @@ function textContent(cell: EvalDetachedCellSnapshot): string {
|
|
|
69
69
|
|
|
70
70
|
function outcomeOf(cell: EvalDetachedCellSnapshot): string {
|
|
71
71
|
if (cell.hardLimitSeconds !== undefined) return `was killed at the ${cell.hardLimitSeconds}s hard limit`;
|
|
72
|
+
if (cell.runBudgetSeconds !== undefined)
|
|
73
|
+
return `was killed after exhausting its ${cell.runBudgetSeconds}s run budget (own execution time; host tool calls excluded)`;
|
|
72
74
|
if (cell.state === "completed") return "completed";
|
|
73
75
|
if (cell.state === "cancelled") return "cancelled";
|
|
74
76
|
return "failed";
|
|
@@ -15,6 +15,8 @@ export interface DetachedCellResultSource {
|
|
|
15
15
|
terminalResult: AgentToolResult<EvalToolDetails> | undefined;
|
|
16
16
|
hardLimited?: boolean;
|
|
17
17
|
hardLimitSeconds?: number;
|
|
18
|
+
runBudgetExhausted?: boolean;
|
|
19
|
+
runBudgetSeconds?: number;
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
export function snapshotDetachedCell(cell: DetachedCellResultSource, nowMs: number): EvalDetachedCellSnapshot {
|
|
@@ -31,6 +33,9 @@ export function snapshotDetachedCell(cell: DetachedCellResultSource, nowMs: numb
|
|
|
31
33
|
...(cell.hardLimited === true && cell.hardLimitSeconds !== undefined
|
|
32
34
|
? { hardLimitSeconds: cell.hardLimitSeconds }
|
|
33
35
|
: {}),
|
|
36
|
+
...(cell.runBudgetExhausted === true && cell.runBudgetSeconds !== undefined
|
|
37
|
+
? { runBudgetSeconds: cell.runBudgetSeconds }
|
|
38
|
+
: {}),
|
|
34
39
|
};
|
|
35
40
|
}
|
|
36
41
|
|
|
@@ -20,15 +20,20 @@ import type {
|
|
|
20
20
|
export interface CreateEvalToolOptions {
|
|
21
21
|
readonly enabledLanguages: EnabledEvalLanguages;
|
|
22
22
|
readonly kernelManager: EvalKernelManager;
|
|
23
|
+
/** Idle time an interactive (detach-behavior) call blocks the agent loop before the cell detaches. */
|
|
23
24
|
readonly cellTimeoutSeconds: number;
|
|
24
25
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* Does not affect `on_timeout: "error"` calls or the wall-clock hard limit.
|
|
26
|
+
* Caps `cellTimeoutSeconds` and the bridge-parked grace for interactive calls. Defaults to
|
|
27
|
+
* {@link DEFAULT_FOREGROUND_WINDOW_SECONDS}. Does not affect the kill deadlines.
|
|
28
28
|
*/
|
|
29
29
|
readonly foregroundWindowSeconds?: number;
|
|
30
30
|
/** Wall-clock kill deadline applied to every cell; only used when this factory creates its own manager. */
|
|
31
31
|
readonly hardLimitSeconds?: number;
|
|
32
|
+
/**
|
|
33
|
+
* Kill deadline for a cell's own execution time (host tool calls excluded); a per-call `timeout`
|
|
34
|
+
* replaces it. Rendered into the tool schema and description; also seeds a self-created manager.
|
|
35
|
+
*/
|
|
36
|
+
readonly runBudgetSeconds?: number;
|
|
32
37
|
readonly executeTool: ExecuteTool;
|
|
33
38
|
readonly listTools?: () => readonly EvalSchemaToolInfo[];
|
|
34
39
|
readonly complete?: (request: CompletionRequest, ctx: ExtensionContext) => Promise<CompletionResult>;
|
package/src/tool/eval-tool.ts
CHANGED
|
@@ -1,19 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import type { AgentToolResult, ExtensionContext, ToolDefinition } from "@code-yeongyu/senpi";
|
|
4
|
-
import { DEFAULT_FOREGROUND_WINDOW_SECONDS, defaultCodemodeSettings } from "../config/settings.ts";
|
|
1
|
+
import type { ToolDefinition } from "@code-yeongyu/senpi";
|
|
2
|
+
import { DEFAULT_FOREGROUND_WINDOW_SECONDS } from "../config/settings.ts";
|
|
5
3
|
import { buildEvalPrompt } from "../prompt/eval-prompt.ts";
|
|
6
|
-
import { TIMEOUT_PAUSE_OP, TIMEOUT_RESUME_OP } from "../timeouts/bridge-timeout.ts";
|
|
7
|
-
import { abortError, CellExecution, defaultTimeoutFactory } from "./cell-execution.ts";
|
|
8
|
-
import { CellHandler, type CellState } from "./cell-handler.ts";
|
|
9
4
|
import { EvalDetachedCellManager } from "./detached-cell-manager.ts";
|
|
10
|
-
import { detachedKernelBusyError, executeEvalControl
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import
|
|
14
|
-
import { describeTimeoutState } from "./interrupt-note.ts";
|
|
5
|
+
import { detachedKernelBusyError, executeEvalControl } from "./detached-eval-result.ts";
|
|
6
|
+
import { clampEvalSummary, isEvalControlRequest, parseEvalRequest } from "./eval-request.ts";
|
|
7
|
+
import type { CreateEvalToolOptions } from "./eval-tool-options.ts";
|
|
8
|
+
import { runEvalCell } from "./run-eval-cell.ts";
|
|
15
9
|
import {
|
|
16
10
|
createEvalInputSchema,
|
|
11
|
+
defaultEvalDeadlineSeconds,
|
|
17
12
|
type EvalInputSchema,
|
|
18
13
|
type EvalToolDetails,
|
|
19
14
|
type EvalToolRequest,
|
|
@@ -25,10 +20,18 @@ export type { CreateEvalToolOptions } from "./eval-tool-options.ts";
|
|
|
25
20
|
export type { EnabledEvalLanguages, EvalKernel, EvalKernelManager } from "./types.ts";
|
|
26
21
|
|
|
27
22
|
export function createEvalTool(options: CreateEvalToolOptions): ToolDefinition<EvalInputSchema, EvalToolDetails> {
|
|
28
|
-
const
|
|
23
|
+
const foregroundWindowSeconds = options.foregroundWindowSeconds ?? DEFAULT_FOREGROUND_WINDOW_SECONDS;
|
|
24
|
+
const deadlines = {
|
|
25
|
+
runBudgetSeconds: options.runBudgetSeconds ?? defaultEvalDeadlineSeconds.runBudgetSeconds,
|
|
26
|
+
detachAfterSeconds: Math.min(options.cellTimeoutSeconds, foregroundWindowSeconds),
|
|
27
|
+
foregroundWindowSeconds,
|
|
28
|
+
hardLimitSeconds: options.hardLimitSeconds ?? defaultEvalDeadlineSeconds.hardLimitSeconds,
|
|
29
|
+
};
|
|
30
|
+
const parameters = createEvalInputSchema(options.enabledLanguages, deadlines);
|
|
29
31
|
const prompt = buildEvalPrompt(options.enabledLanguages, {
|
|
30
32
|
spawns: options.spawns ?? false,
|
|
31
33
|
monitor: options.monitor,
|
|
34
|
+
runBudgetSeconds: deadlines.runBudgetSeconds,
|
|
32
35
|
...(options.spawnDefaultAgent === undefined ? {} : { spawnDefaultAgent: options.spawnDefaultAgent }),
|
|
33
36
|
...(options.modelId === undefined ? {} : { modelId: options.modelId }),
|
|
34
37
|
...(options.hostLine === undefined ? {} : { hostLine: options.hostLine }),
|
|
@@ -41,6 +44,7 @@ export function createEvalTool(options: CreateEvalToolOptions): ToolDefinition<E
|
|
|
41
44
|
new EvalDetachedCellManager({
|
|
42
45
|
...(options.artifactsDir === undefined ? {} : { artifactsDir: options.artifactsDir }),
|
|
43
46
|
...(options.hardLimitSeconds === undefined ? {} : { hardLimitSeconds: options.hardLimitSeconds }),
|
|
47
|
+
...(options.runBudgetSeconds === undefined ? {} : { runBudgetSeconds: options.runBudgetSeconds }),
|
|
44
48
|
});
|
|
45
49
|
return {
|
|
46
50
|
name: "eval",
|
|
@@ -94,177 +98,3 @@ export function createEvalTool(options: CreateEvalToolOptions): ToolDefinition<E
|
|
|
94
98
|
},
|
|
95
99
|
};
|
|
96
100
|
}
|
|
97
|
-
|
|
98
|
-
async function runEvalCell(
|
|
99
|
-
options: CreateEvalToolOptions,
|
|
100
|
-
cellManager: EvalDetachedCellManager,
|
|
101
|
-
invocation: EvalCellInvocation,
|
|
102
|
-
): Promise<AgentToolResult<EvalToolDetails>> {
|
|
103
|
-
if (invocation.signal.aborted) throw abortError(invocation.signal.reason);
|
|
104
|
-
const timeoutBehavior = evalTimeoutBehavior(invocation.input, invocation.ctx);
|
|
105
|
-
const requestedTimeoutMs = Math.floor((invocation.input.timeout ?? options.cellTimeoutSeconds) * 1_000);
|
|
106
|
-
// The `timeout` (and its `cellTimeoutSeconds` default) is the detach budget for interactive calls.
|
|
107
|
-
// Cap it at the foreground window so a large `timeout` — whose real purpose is to raise the
|
|
108
|
-
// wall-clock hard limit (see EvalDetachedCellManager) — frees the turn at the window instead of
|
|
109
|
-
// blocking the agent loop for its full duration. `on_timeout: "error"` (and print/json) keep the
|
|
110
|
-
// unclamped deadline, since there the cell is killed rather than detached.
|
|
111
|
-
const foregroundWindowMs = (options.foregroundWindowSeconds ?? DEFAULT_FOREGROUND_WINDOW_SECONDS) * 1_000;
|
|
112
|
-
const timeoutMs =
|
|
113
|
-
timeoutBehavior === "detach" ? Math.min(requestedTimeoutMs, foregroundWindowMs) : requestedTimeoutMs;
|
|
114
|
-
// A cell that pauses its watchdog for a host bridge call would otherwise wait the full pause grace
|
|
115
|
-
// (~10 min) before detaching; cap the grace at the foreground window too so the detach guarantee
|
|
116
|
-
// holds for bridge-parked cells. Error mode keeps the default grace (its timeout is the deadline).
|
|
117
|
-
const bridgeAbortController = new AbortController();
|
|
118
|
-
const cellSignal = AbortSignal.any([invocation.signal, bridgeAbortController.signal]);
|
|
119
|
-
const bridgeContext: ExtensionContext = { ...invocation.ctx, signal: cellSignal };
|
|
120
|
-
const runtime = options.runtimes?.[invocation.input.language];
|
|
121
|
-
const state: CellState = {
|
|
122
|
-
input: invocation.input,
|
|
123
|
-
...(runtime === undefined ? {} : { runtime }),
|
|
124
|
-
startedAt: Date.now(),
|
|
125
|
-
signal: cellSignal,
|
|
126
|
-
onUpdate: invocation.onUpdate,
|
|
127
|
-
toolCalls: [],
|
|
128
|
-
toolCallMetrics: [],
|
|
129
|
-
pendingBridgeCalls: [],
|
|
130
|
-
statusEvents: [],
|
|
131
|
-
active: true,
|
|
132
|
-
output: "",
|
|
133
|
-
phase: undefined,
|
|
134
|
-
error: undefined,
|
|
135
|
-
durationMs: 0,
|
|
136
|
-
status: "pending",
|
|
137
|
-
};
|
|
138
|
-
const cell = cellManager.create(invocation.cellId, invocation.input);
|
|
139
|
-
let detached = false;
|
|
140
|
-
let execution: CellExecution;
|
|
141
|
-
execution = new CellExecution({
|
|
142
|
-
callerSignal: invocation.signal,
|
|
143
|
-
cellId: invocation.cellId,
|
|
144
|
-
timeoutMs,
|
|
145
|
-
...(timeoutBehavior === "detach" ? { maxPauseGraceMs: foregroundWindowMs } : {}),
|
|
146
|
-
timeoutFactory: options.timeoutFactory ?? defaultTimeoutFactory,
|
|
147
|
-
onTimeout: (error) => {
|
|
148
|
-
if (timeoutBehavior === "detach" && cellManager.detach(cell)) {
|
|
149
|
-
detached = true;
|
|
150
|
-
execution.detach();
|
|
151
|
-
return;
|
|
152
|
-
}
|
|
153
|
-
execution.cancel(error);
|
|
154
|
-
},
|
|
155
|
-
onAbort: (error) => {
|
|
156
|
-
state.active = false;
|
|
157
|
-
bridgeAbortController.abort(error);
|
|
158
|
-
},
|
|
159
|
-
});
|
|
160
|
-
const running = executeCell(
|
|
161
|
-
options,
|
|
162
|
-
invocation,
|
|
163
|
-
cellManager,
|
|
164
|
-
cell,
|
|
165
|
-
state,
|
|
166
|
-
execution,
|
|
167
|
-
bridgeContext,
|
|
168
|
-
bridgeAbortController,
|
|
169
|
-
);
|
|
170
|
-
let settleEventEmitted = false;
|
|
171
|
-
const emitSettled = (outcome: EvalExecutionSettleOutcome): void => {
|
|
172
|
-
if (settleEventEmitted) return;
|
|
173
|
-
settleEventEmitted = true;
|
|
174
|
-
options.onCellSettled?.(
|
|
175
|
-
buildEvalExecutionEventPayload({
|
|
176
|
-
cellId: invocation.cellId,
|
|
177
|
-
state,
|
|
178
|
-
outcome,
|
|
179
|
-
completedAt: Date.now(),
|
|
180
|
-
detached,
|
|
181
|
-
}),
|
|
182
|
-
);
|
|
183
|
-
};
|
|
184
|
-
const finalized = running.then(
|
|
185
|
-
(result) => {
|
|
186
|
-
cellManager.complete(cell, result);
|
|
187
|
-
emitSettled({ result });
|
|
188
|
-
return result;
|
|
189
|
-
},
|
|
190
|
-
(error: unknown) => {
|
|
191
|
-
cellManager.fail(cell, error instanceof Error ? error : new Error(String(error)));
|
|
192
|
-
emitSettled({ error });
|
|
193
|
-
throw error;
|
|
194
|
-
},
|
|
195
|
-
);
|
|
196
|
-
const outcome = await Promise.race([
|
|
197
|
-
finalized.then((result) => ({ kind: "result" as const, result })),
|
|
198
|
-
execution.detached.then(() => ({ kind: "detached" as const })),
|
|
199
|
-
]);
|
|
200
|
-
if (outcome.kind === "detached") return resultAfterDetach(cellManager.peek(invocation.cellId), invocation.input);
|
|
201
|
-
return outcome.result;
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
async function executeCell(
|
|
205
|
-
options: CreateEvalToolOptions,
|
|
206
|
-
invocation: EvalCellInvocation,
|
|
207
|
-
cellManager: EvalDetachedCellManager,
|
|
208
|
-
cell: Parameters<EvalDetachedCellManager["markRunning"]>[0],
|
|
209
|
-
state: CellState,
|
|
210
|
-
execution: CellExecution,
|
|
211
|
-
bridgeContext: ExtensionContext,
|
|
212
|
-
bridgeAbortController: AbortController,
|
|
213
|
-
): Promise<AgentToolResult<EvalToolDetails>> {
|
|
214
|
-
let handler: CellHandler | undefined;
|
|
215
|
-
try {
|
|
216
|
-
const kernel = await execution.wait(
|
|
217
|
-
options.kernelManager.getKernel(invocation.input.language, (message) => {
|
|
218
|
-
if (!state.active || handler === undefined) return;
|
|
219
|
-
if (message.type === "status") {
|
|
220
|
-
if (message.event.op === TIMEOUT_PAUSE_OP) {
|
|
221
|
-
execution.pause();
|
|
222
|
-
return;
|
|
223
|
-
}
|
|
224
|
-
if (message.event.op === TIMEOUT_RESUME_OP) {
|
|
225
|
-
execution.resume();
|
|
226
|
-
return;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
const pending = handler.handle(message);
|
|
230
|
-
void pending.catch((error: unknown) => execution.cancel(error));
|
|
231
|
-
}),
|
|
232
|
-
);
|
|
233
|
-
execution.setKernel(kernel);
|
|
234
|
-
const activeHandler = new CellHandler(kernel, state, {
|
|
235
|
-
executeTool: options.executeTool,
|
|
236
|
-
...(options.listTools === undefined ? {} : { listTools: options.listTools }),
|
|
237
|
-
settings: options.settings ?? defaultCodemodeSettings,
|
|
238
|
-
...(options.complete === undefined ? {} : { complete: options.complete }),
|
|
239
|
-
ctx: bridgeContext,
|
|
240
|
-
...(options.artifactsDir === undefined
|
|
241
|
-
? {}
|
|
242
|
-
: { artifactPath: join(options.artifactsDir, `eval-${randomUUID()}.log`) }),
|
|
243
|
-
...(options.imageResizer === undefined ? {} : { imageResizer: options.imageResizer }),
|
|
244
|
-
});
|
|
245
|
-
handler = activeHandler;
|
|
246
|
-
cellManager.markRunning(
|
|
247
|
-
cell,
|
|
248
|
-
kernel,
|
|
249
|
-
() => activeHandler.liveResult(),
|
|
250
|
-
(error) => execution.cancel(error),
|
|
251
|
-
);
|
|
252
|
-
if ("setContext" in options.kernelManager && typeof options.kernelManager.setContext === "function") {
|
|
253
|
-
options.kernelManager.setContext(bridgeContext);
|
|
254
|
-
}
|
|
255
|
-
if (invocation.input.reset) await execution.wait(kernel.reset());
|
|
256
|
-
const result = await execution.wait(kernel.run({ cellId: invocation.cellId, code: invocation.input.code }));
|
|
257
|
-
if (result.ok && state.pendingBridgeCalls.length > 0) await execution.wait(Promise.all(state.pendingBridgeCalls));
|
|
258
|
-
return await handler.finalize(result);
|
|
259
|
-
} catch (error) {
|
|
260
|
-
if (handler && error instanceof Error && error.name === "CodemodeSessionDisposedError")
|
|
261
|
-
return await handler.finalizeCancellation(error);
|
|
262
|
-
if (error instanceof Error && error.name === "TimeoutError") throw await describeTimeoutState(error, execution);
|
|
263
|
-
throw error;
|
|
264
|
-
} finally {
|
|
265
|
-
state.active = false;
|
|
266
|
-
bridgeAbortController.abort();
|
|
267
|
-
execution.finish();
|
|
268
|
-
if (handler) await handler.flushOutput();
|
|
269
|
-
}
|
|
270
|
-
}
|