@code-yeongyu/senpi-codemode 2026.9.9 → 2026.9.10-2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +17 -2
- package/package.json +4 -4
- package/src/config/settings.ts +35 -15
- package/src/index.ts +13 -2
- package/src/prompt/eval-prompt-template.ts +76 -0
- package/src/prompt/eval-prompt.ts +6 -77
- package/src/timeouts/run-budget.ts +107 -0
- package/src/tool/cell-deadlines.ts +76 -0
- package/src/tool/cell-execution.ts +25 -14
- package/src/tool/detached-cell-contract.ts +4 -0
- package/src/tool/detached-cell-manager.ts +47 -40
- package/src/tool/detached-cell-notification.ts +2 -0
- package/src/tool/detached-cell-snapshot.ts +5 -0
- package/src/tool/eval-tool-options.ts +8 -3
- package/src/tool/eval-tool.ts +17 -187
- package/src/tool/run-eval-cell.ts +189 -0
- package/src/tool/types.ts +66 -56
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { AgentToolResult, ExtensionContext } from "@code-yeongyu/senpi";
|
|
4
|
+
import { DEFAULT_FOREGROUND_WINDOW_SECONDS, defaultCodemodeSettings } from "../config/settings.ts";
|
|
5
|
+
import { TIMEOUT_PAUSE_OP, TIMEOUT_RESUME_OP } from "../timeouts/bridge-timeout.ts";
|
|
6
|
+
import { abortError, CellExecution, defaultTimeoutFactory } from "./cell-execution.ts";
|
|
7
|
+
import { CellHandler, type CellState } from "./cell-handler.ts";
|
|
8
|
+
import type { EvalDetachedCellManager } from "./detached-cell-manager.ts";
|
|
9
|
+
import { resultAfterDetach } from "./detached-eval-result.ts";
|
|
10
|
+
import { buildEvalExecutionEventPayload, type EvalExecutionSettleOutcome } from "./eval-execution-event.ts";
|
|
11
|
+
import { evalTimeoutBehavior } from "./eval-request.ts";
|
|
12
|
+
import type { CreateEvalToolOptions, EvalCellInvocation } from "./eval-tool-options.ts";
|
|
13
|
+
import { describeTimeoutState } from "./interrupt-note.ts";
|
|
14
|
+
import type { EvalToolDetails } from "./types.ts";
|
|
15
|
+
|
|
16
|
+
export async function runEvalCell(
|
|
17
|
+
options: CreateEvalToolOptions,
|
|
18
|
+
cellManager: EvalDetachedCellManager,
|
|
19
|
+
invocation: EvalCellInvocation,
|
|
20
|
+
): Promise<AgentToolResult<EvalToolDetails>> {
|
|
21
|
+
if (invocation.signal.aborted) throw abortError(invocation.signal.reason);
|
|
22
|
+
const detaches = evalTimeoutBehavior(invocation.input, invocation.ctx) === "detach";
|
|
23
|
+
// The per-call `timeout` is the cell's run budget (owned by the cell manager's deadlines); how long
|
|
24
|
+
// an interactive call blocks the turn is the idle detach budget, capped at the foreground window
|
|
25
|
+
// — including the grace a bridge-parked cell gets — so `timeout` never delays the detach.
|
|
26
|
+
const foregroundWindowMs = (options.foregroundWindowSeconds ?? DEFAULT_FOREGROUND_WINDOW_SECONDS) * 1_000;
|
|
27
|
+
const detachAfterMs = Math.min(Math.floor(options.cellTimeoutSeconds * 1_000), foregroundWindowMs);
|
|
28
|
+
const bridgeAbortController = new AbortController();
|
|
29
|
+
const cellSignal = AbortSignal.any([invocation.signal, bridgeAbortController.signal]);
|
|
30
|
+
const bridgeContext: ExtensionContext = { ...invocation.ctx, signal: cellSignal };
|
|
31
|
+
const runtime = options.runtimes?.[invocation.input.language];
|
|
32
|
+
const state: CellState = {
|
|
33
|
+
input: invocation.input,
|
|
34
|
+
...(runtime === undefined ? {} : { runtime }),
|
|
35
|
+
startedAt: Date.now(),
|
|
36
|
+
signal: cellSignal,
|
|
37
|
+
onUpdate: invocation.onUpdate,
|
|
38
|
+
toolCalls: [],
|
|
39
|
+
toolCallMetrics: [],
|
|
40
|
+
pendingBridgeCalls: [],
|
|
41
|
+
statusEvents: [],
|
|
42
|
+
active: true,
|
|
43
|
+
output: "",
|
|
44
|
+
phase: undefined,
|
|
45
|
+
error: undefined,
|
|
46
|
+
durationMs: 0,
|
|
47
|
+
status: "pending",
|
|
48
|
+
};
|
|
49
|
+
let detached = false;
|
|
50
|
+
let execution: CellExecution;
|
|
51
|
+
const cell = cellManager.create(invocation.cellId, invocation.input, (error) => execution.cancel(error));
|
|
52
|
+
execution = new CellExecution({
|
|
53
|
+
callerSignal: invocation.signal,
|
|
54
|
+
cellId: invocation.cellId,
|
|
55
|
+
...(detaches
|
|
56
|
+
? {
|
|
57
|
+
idle: {
|
|
58
|
+
timeoutMs: detachAfterMs,
|
|
59
|
+
maxPauseGraceMs: foregroundWindowMs,
|
|
60
|
+
onTimeout: (error: Error) => {
|
|
61
|
+
if (cellManager.detach(cell)) {
|
|
62
|
+
detached = true;
|
|
63
|
+
execution.detach();
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
execution.cancel(error);
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
}
|
|
70
|
+
: {}),
|
|
71
|
+
timeoutFactory: options.timeoutFactory ?? defaultTimeoutFactory,
|
|
72
|
+
onAbort: (error) => {
|
|
73
|
+
state.active = false;
|
|
74
|
+
bridgeAbortController.abort(error);
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
const running = executeCell(
|
|
78
|
+
options,
|
|
79
|
+
invocation,
|
|
80
|
+
cellManager,
|
|
81
|
+
cell,
|
|
82
|
+
state,
|
|
83
|
+
execution,
|
|
84
|
+
bridgeContext,
|
|
85
|
+
bridgeAbortController,
|
|
86
|
+
);
|
|
87
|
+
let settleEventEmitted = false;
|
|
88
|
+
const emitSettled = (outcome: EvalExecutionSettleOutcome): void => {
|
|
89
|
+
if (settleEventEmitted) return;
|
|
90
|
+
settleEventEmitted = true;
|
|
91
|
+
options.onCellSettled?.(
|
|
92
|
+
buildEvalExecutionEventPayload({
|
|
93
|
+
cellId: invocation.cellId,
|
|
94
|
+
state,
|
|
95
|
+
outcome,
|
|
96
|
+
completedAt: Date.now(),
|
|
97
|
+
detached,
|
|
98
|
+
}),
|
|
99
|
+
);
|
|
100
|
+
};
|
|
101
|
+
const finalized = running.then(
|
|
102
|
+
(result) => {
|
|
103
|
+
cellManager.complete(cell, result);
|
|
104
|
+
emitSettled({ result });
|
|
105
|
+
return result;
|
|
106
|
+
},
|
|
107
|
+
(error: unknown) => {
|
|
108
|
+
cellManager.fail(cell, error instanceof Error ? error : new Error(String(error)));
|
|
109
|
+
emitSettled({ error });
|
|
110
|
+
throw error;
|
|
111
|
+
},
|
|
112
|
+
);
|
|
113
|
+
const outcome = await Promise.race([
|
|
114
|
+
finalized.then((result) => ({ kind: "result" as const, result })),
|
|
115
|
+
execution.detached.then(() => ({ kind: "detached" as const })),
|
|
116
|
+
]);
|
|
117
|
+
if (outcome.kind === "detached") return resultAfterDetach(cellManager.peek(invocation.cellId), invocation.input);
|
|
118
|
+
return outcome.result;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function executeCell(
|
|
122
|
+
options: CreateEvalToolOptions,
|
|
123
|
+
invocation: EvalCellInvocation,
|
|
124
|
+
cellManager: EvalDetachedCellManager,
|
|
125
|
+
cell: Parameters<EvalDetachedCellManager["markRunning"]>[0],
|
|
126
|
+
state: CellState,
|
|
127
|
+
execution: CellExecution,
|
|
128
|
+
bridgeContext: ExtensionContext,
|
|
129
|
+
bridgeAbortController: AbortController,
|
|
130
|
+
): Promise<AgentToolResult<EvalToolDetails>> {
|
|
131
|
+
let handler: CellHandler | undefined;
|
|
132
|
+
try {
|
|
133
|
+
const kernel = await execution.wait(
|
|
134
|
+
options.kernelManager.getKernel(invocation.input.language, (message) => {
|
|
135
|
+
if (!state.active || handler === undefined) return;
|
|
136
|
+
if (message.type === "status") {
|
|
137
|
+
if (message.event.op === TIMEOUT_PAUSE_OP) {
|
|
138
|
+
execution.pause();
|
|
139
|
+
cellManager.pause(cell);
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
if (message.event.op === TIMEOUT_RESUME_OP) {
|
|
143
|
+
execution.resume();
|
|
144
|
+
cellManager.resume(cell);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const pending = handler.handle(message);
|
|
149
|
+
void pending.catch((error: unknown) => execution.cancel(error));
|
|
150
|
+
}),
|
|
151
|
+
);
|
|
152
|
+
execution.setKernel(kernel);
|
|
153
|
+
const activeHandler = new CellHandler(kernel, state, {
|
|
154
|
+
executeTool: options.executeTool,
|
|
155
|
+
...(options.listTools === undefined ? {} : { listTools: options.listTools }),
|
|
156
|
+
settings: options.settings ?? defaultCodemodeSettings,
|
|
157
|
+
...(options.complete === undefined ? {} : { complete: options.complete }),
|
|
158
|
+
ctx: bridgeContext,
|
|
159
|
+
...(options.artifactsDir === undefined
|
|
160
|
+
? {}
|
|
161
|
+
: { artifactPath: join(options.artifactsDir, `eval-${randomUUID()}.log`) }),
|
|
162
|
+
...(options.imageResizer === undefined ? {} : { imageResizer: options.imageResizer }),
|
|
163
|
+
});
|
|
164
|
+
handler = activeHandler;
|
|
165
|
+
cellManager.markRunning(
|
|
166
|
+
cell,
|
|
167
|
+
kernel,
|
|
168
|
+
() => activeHandler.liveResult(),
|
|
169
|
+
(error) => execution.cancel(error),
|
|
170
|
+
);
|
|
171
|
+
if ("setContext" in options.kernelManager && typeof options.kernelManager.setContext === "function") {
|
|
172
|
+
options.kernelManager.setContext(bridgeContext);
|
|
173
|
+
}
|
|
174
|
+
if (invocation.input.reset) await execution.wait(kernel.reset());
|
|
175
|
+
const result = await execution.wait(kernel.run({ cellId: invocation.cellId, code: invocation.input.code }));
|
|
176
|
+
if (result.ok && state.pendingBridgeCalls.length > 0) await execution.wait(Promise.all(state.pendingBridgeCalls));
|
|
177
|
+
return await handler.finalize(result);
|
|
178
|
+
} catch (error) {
|
|
179
|
+
if (handler && error instanceof Error && error.name === "CodemodeSessionDisposedError")
|
|
180
|
+
return await handler.finalizeCancellation(error);
|
|
181
|
+
if (error instanceof Error && error.name === "TimeoutError") throw await describeTimeoutState(error, execution);
|
|
182
|
+
throw error;
|
|
183
|
+
} finally {
|
|
184
|
+
state.active = false;
|
|
185
|
+
bridgeAbortController.abort();
|
|
186
|
+
execution.finish();
|
|
187
|
+
if (handler) await handler.flushOutput();
|
|
188
|
+
}
|
|
189
|
+
}
|
package/src/tool/types.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import type { AgentToolResult, AgentToolUpdateCallback } from "@code-yeongyu/senpi";
|
|
2
|
-
import { type TUnsafe, Type } from "typebox";
|
|
2
|
+
import { type TSchema, type TUnsafe, Type } from "typebox";
|
|
3
3
|
import type { HostToKernelMessage, KernelToHostMessage } from "../bridge/protocol.ts";
|
|
4
|
+
import {
|
|
5
|
+
DEFAULT_FOREGROUND_WINDOW_SECONDS,
|
|
6
|
+
DEFAULT_HARD_LIMIT_SECONDS,
|
|
7
|
+
DEFAULT_RUN_BUDGET_SECONDS,
|
|
8
|
+
defaultCodemodeSettings,
|
|
9
|
+
} from "../config/settings.ts";
|
|
4
10
|
import type { TruncationMeta } from "../output/output-meta.ts";
|
|
5
11
|
|
|
6
12
|
export const evalLanguageOrder = ["js", "py", "rb", "jl"] as const;
|
|
@@ -13,11 +19,30 @@ export function enabledLanguageList(enabled: EnabledEvalLanguages): EvalLanguage
|
|
|
13
19
|
|
|
14
20
|
export const EVAL_SUMMARY_MAX_LENGTH = 80;
|
|
15
21
|
|
|
16
|
-
|
|
17
|
-
|
|
22
|
+
/** The deadlines the schema teaches the model; every number comes from the resolved settings. */
|
|
23
|
+
export interface EvalDeadlineSeconds {
|
|
24
|
+
readonly runBudgetSeconds: number;
|
|
25
|
+
/** Effective interactive detach point: `cellTimeoutSeconds` capped by the foreground window. */
|
|
26
|
+
readonly detachAfterSeconds: number;
|
|
27
|
+
/** Longest a host tool call can hold an interactive call before it detaches anyway. */
|
|
28
|
+
readonly foregroundWindowSeconds: number;
|
|
29
|
+
readonly hardLimitSeconds: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export const defaultEvalDeadlineSeconds: EvalDeadlineSeconds = {
|
|
33
|
+
runBudgetSeconds: DEFAULT_RUN_BUDGET_SECONDS,
|
|
34
|
+
detachAfterSeconds: Math.min(defaultCodemodeSettings.cellTimeoutSeconds, DEFAULT_FOREGROUND_WINDOW_SECONDS),
|
|
35
|
+
foregroundWindowSeconds: DEFAULT_FOREGROUND_WINDOW_SECONDS,
|
|
36
|
+
hardLimitSeconds: DEFAULT_HARD_LIMIT_SECONDS,
|
|
37
|
+
};
|
|
18
38
|
|
|
19
|
-
|
|
20
|
-
|
|
39
|
+
function timeoutFieldDescription(deadlines: EvalDeadlineSeconds): string {
|
|
40
|
+
return `Run budget in seconds for this cell's own execution (default ${deadlines.runBudgetSeconds}s); time parked in host tool calls such as agent() or tool.* is not charged. When it runs out the cell is killed, and a js cell that cannot settle (a pending timer or Bun.$ command, a synchronous call) restarts its kernel and loses every global. Raise it only for a declared long run; a value above ${deadlines.hardLimitSeconds}s also raises the wall-clock hard limit. It does not move the detach point.`;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function onTimeoutFieldDescription(deadlines: EvalDeadlineSeconds): string {
|
|
44
|
+
return `'detach' (interactive default): the call returns after ${deadlines.detachAfterSeconds}s of the cell's own work (a host tool call in flight can hold it up to the ${deadlines.foregroundWindowSeconds}s foreground window) while the cell keeps running; completion arrives as a notification. 'error' (print/json default): the call blocks until the cell settles or a deadline kills it.`;
|
|
45
|
+
}
|
|
21
46
|
|
|
22
47
|
export interface EvalToolInput {
|
|
23
48
|
readonly language: EvalLanguage;
|
|
@@ -36,69 +61,54 @@ export interface EvalControlInput {
|
|
|
36
61
|
|
|
37
62
|
export type EvalToolRequest = EvalToolInput | EvalControlInput;
|
|
38
63
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
64
|
+
function evalInputProperties<Language extends TSchema>(languageSchema: Language, deadlines: EvalDeadlineSeconds) {
|
|
65
|
+
return {
|
|
66
|
+
action: Type.Optional(
|
|
67
|
+
Type.Union([Type.Literal("run"), Type.Literal("peek"), Type.Literal("stop")], {
|
|
68
|
+
description: "Defaults to run. peek and stop require cell_id.",
|
|
69
|
+
}),
|
|
70
|
+
),
|
|
71
|
+
language: Type.Optional(languageSchema),
|
|
72
|
+
code: Type.Optional(Type.String({ description: "Cell body, verbatim." })),
|
|
73
|
+
summary: Type.Optional(
|
|
74
|
+
Type.String({
|
|
75
|
+
maxLength: EVAL_SUMMARY_MAX_LENGTH,
|
|
76
|
+
description:
|
|
77
|
+
"REQUIRED for run. ONE line in the USER'S conversational language (Korean conversation -> Korean summary) stating WHAT this cell does and FOR WHAT PURPOSE; shown in the TUI while the cell runs. Longer values are force-truncated to 80 chars.",
|
|
78
|
+
}),
|
|
79
|
+
),
|
|
80
|
+
timeout: Type.Optional(Type.Number({ minimum: 1, description: timeoutFieldDescription(deadlines) })),
|
|
81
|
+
on_timeout: Type.Optional(
|
|
82
|
+
Type.Union([Type.Literal("detach"), Type.Literal("error")], {
|
|
83
|
+
description: onTimeoutFieldDescription(deadlines),
|
|
84
|
+
}),
|
|
85
|
+
),
|
|
86
|
+
reset: Type.Optional(Type.Boolean({ description: "Reset this language kernel before running." })),
|
|
87
|
+
cell_id: Type.Optional(Type.String({ description: "Detached eval cell id for peek or stop." })),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const fullEvalInputSchema = Type.Object(
|
|
92
|
+
evalInputProperties(
|
|
46
93
|
Type.Union([Type.Literal("js"), Type.Literal("py"), Type.Literal("rb"), Type.Literal("jl")]),
|
|
94
|
+
defaultEvalDeadlineSeconds,
|
|
47
95
|
),
|
|
48
|
-
|
|
49
|
-
summary: Type.Optional(
|
|
50
|
-
Type.String({
|
|
51
|
-
maxLength: EVAL_SUMMARY_MAX_LENGTH,
|
|
52
|
-
description:
|
|
53
|
-
"REQUIRED for run. ONE line in the USER'S conversational language (Korean conversation -> Korean summary) stating WHAT this cell does and FOR WHAT PURPOSE; shown in the TUI while the cell runs. Longer values are force-truncated to 80 chars.",
|
|
54
|
-
}),
|
|
55
|
-
),
|
|
56
|
-
timeout: Type.Optional(Type.Number({ minimum: 1, description: TIMEOUT_FIELD_DESCRIPTION })),
|
|
57
|
-
on_timeout: Type.Optional(
|
|
58
|
-
Type.Union([Type.Literal("detach"), Type.Literal("error")], {
|
|
59
|
-
description: ON_TIMEOUT_FIELD_DESCRIPTION,
|
|
60
|
-
}),
|
|
61
|
-
),
|
|
62
|
-
reset: Type.Optional(Type.Boolean({ description: "Reset this language kernel before running." })),
|
|
63
|
-
cell_id: Type.Optional(Type.String({ description: "Detached eval cell id for peek or stop." })),
|
|
64
|
-
});
|
|
96
|
+
);
|
|
65
97
|
|
|
66
98
|
/** Runtime accepts a discriminated run/control union. */
|
|
67
99
|
export type EvalInputSchema = TUnsafe<EvalToolRequest> & Pick<typeof fullEvalInputSchema, "properties">;
|
|
68
100
|
|
|
69
|
-
export function createEvalInputSchema(
|
|
101
|
+
export function createEvalInputSchema(
|
|
102
|
+
enabled: EnabledEvalLanguages,
|
|
103
|
+
deadlines: EvalDeadlineSeconds = defaultEvalDeadlineSeconds,
|
|
104
|
+
): EvalInputSchema {
|
|
70
105
|
const languages = enabledLanguageList(enabled);
|
|
71
106
|
if (languages.length === 0) throw new Error("eval requires at least one enabled language");
|
|
72
107
|
const languageSchema =
|
|
73
108
|
languages.length === 1
|
|
74
109
|
? Type.Union([Type.Literal(languages[0])])
|
|
75
110
|
: Type.Union(languages.map((item) => Type.Literal(item)));
|
|
76
|
-
return Type.Unsafe<EvalToolRequest>(
|
|
77
|
-
Type.Object({
|
|
78
|
-
action: Type.Optional(
|
|
79
|
-
Type.Union([Type.Literal("run"), Type.Literal("peek"), Type.Literal("stop")], {
|
|
80
|
-
description: "Defaults to run. peek and stop require cell_id.",
|
|
81
|
-
}),
|
|
82
|
-
),
|
|
83
|
-
language: Type.Optional(languageSchema),
|
|
84
|
-
code: Type.Optional(Type.String({ description: "Cell body, verbatim." })),
|
|
85
|
-
summary: Type.Optional(
|
|
86
|
-
Type.String({
|
|
87
|
-
maxLength: EVAL_SUMMARY_MAX_LENGTH,
|
|
88
|
-
description:
|
|
89
|
-
"REQUIRED for run. ONE line in the USER'S conversational language (Korean conversation -> Korean summary) stating WHAT this cell does and FOR WHAT PURPOSE; shown in the TUI while the cell runs. Longer values are force-truncated to 80 chars.",
|
|
90
|
-
}),
|
|
91
|
-
),
|
|
92
|
-
timeout: Type.Optional(Type.Number({ minimum: 1, description: TIMEOUT_FIELD_DESCRIPTION })),
|
|
93
|
-
on_timeout: Type.Optional(
|
|
94
|
-
Type.Union([Type.Literal("detach"), Type.Literal("error")], {
|
|
95
|
-
description: ON_TIMEOUT_FIELD_DESCRIPTION,
|
|
96
|
-
}),
|
|
97
|
-
),
|
|
98
|
-
reset: Type.Optional(Type.Boolean({ description: "Reset this language kernel before running." })),
|
|
99
|
-
cell_id: Type.Optional(Type.String({ description: "Detached eval cell id for peek or stop." })),
|
|
100
|
-
}),
|
|
101
|
-
) as EvalInputSchema;
|
|
111
|
+
return Type.Unsafe<EvalToolRequest>(Type.Object(evalInputProperties(languageSchema, deadlines))) as EvalInputSchema;
|
|
102
112
|
}
|
|
103
113
|
export type EvalKernelResult = Extract<KernelToHostMessage, { type: "result" }>;
|
|
104
114
|
export type EvalToolCallMessage = Extract<KernelToHostMessage, { type: "tool-call" }>;
|