@code-yeongyu/senpi-codemode 2026.10.8 → 2026.10.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +51 -3
- package/package.json +4 -4
- package/src/bridge/http-server.ts +9 -1
- package/src/bridge/protocol.ts +4 -0
- package/src/bridges/eval-virtual-schemas.ts +1 -0
- package/src/environments/py-installer.ts +64 -4
- package/src/environments/python-environments.ts +85 -0
- package/src/environments/revision-store.ts +18 -3
- package/src/extension/bridge-tool-call.ts +2 -0
- package/src/extension/cell-kernel-tools.ts +28 -0
- package/src/extension/kernel-replacement.ts +20 -0
- package/src/extension/runtime-factory.ts +12 -0
- package/src/extension/session-manager-proxy.ts +6 -0
- package/src/extension/session-manager.ts +10 -2
- package/src/index.ts +1 -0
- package/src/kernels/jl/prelude.jl +6 -1
- package/src/kernels/js/context-manager.ts +15 -1
- package/src/kernels/js/local-module-loader.ts +9 -4
- package/src/kernels/js/worker-runtime.js +3 -1
- package/src/kernels/js/workpool.js +5 -2
- package/src/kernels/py/kernel.ts +8 -2
- package/src/kernels/py/prelude.py +112 -9
- package/src/kernels/py/transport.ts +13 -1
- package/src/kernels/rb/workpool.rb +7 -1
- package/src/kernels/shared/cell-source-at-start.ts +18 -0
- package/src/tool/cell-handler.ts +1 -1
- package/src/tool/eval-tool-options.ts +3 -0
- package/src/tool/load-cell.ts +73 -0
- package/src/tool/magic-cell-host.ts +65 -0
- package/src/tool/magic-cells.ts +65 -0
- package/src/tool/run-eval-cell.ts +30 -0
- package/src/tool/types.ts +27 -0
|
@@ -4,8 +4,11 @@ import { kernelToolError } from "./kernel-tools-errors.js";
|
|
|
4
4
|
// Only host-tool sugar: no admission, worker, or queue state belongs to the kernel.
|
|
5
5
|
export async function createWorkpool(callTool, agent, name, options = {}) {
|
|
6
6
|
if (inKernelToolInvoke()) throw kernelToolError("kernel_tool_recursion", "kernel tools may not invoke workpool()");
|
|
7
|
-
if (options === null || typeof options !== "object" || Array.isArray(options) || Object.keys(options).some(key => key !== "mode")) {
|
|
8
|
-
throw new TypeError("workpool() options only accept mode");
|
|
7
|
+
if (options === null || typeof options !== "object" || Array.isArray(options) || Object.keys(options).some(key => key !== "mode" && key !== "tools")) {
|
|
8
|
+
throw new TypeError("workpool() options only accept mode and tools");
|
|
9
|
+
}
|
|
10
|
+
if (options.tools !== undefined && (!Array.isArray(options.tools) || !options.tools.every(name => typeof name === "string"))) {
|
|
11
|
+
throw workpoolError("invalid_tools", `workpool() tools takes an array of tool names; got ${Array.isArray(options.tools) ? "an array with a non-string" : typeof options.tools}`);
|
|
9
12
|
}
|
|
10
13
|
const call = async args => {
|
|
11
14
|
try {
|
package/src/kernels/py/kernel.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
KernelToolsInvokeOptions,
|
|
6
6
|
KernelToolsInvokeRequest,
|
|
7
7
|
} from "../js/kernel-tools-types.ts";
|
|
8
|
+
import { inputAtStart } from "../shared/cell-source-at-start.ts";
|
|
8
9
|
import { describeExit } from "../shared/kernel-death.ts";
|
|
9
10
|
import { KernelMemoryHost } from "../shared/kernel-memory-host.ts";
|
|
10
11
|
import { KernelPreludeTracker } from "../shared/kernel-prelude-plan.ts";
|
|
@@ -243,10 +244,15 @@ export class PythonKernel {
|
|
|
243
244
|
});
|
|
244
245
|
return;
|
|
245
246
|
}
|
|
247
|
+
const input = inputAtStart(pending.input);
|
|
248
|
+
if ("refused" in input) {
|
|
249
|
+
this.#settleRun(pending, failedPythonResult(pending.input.cellId, input.refused));
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
246
252
|
try {
|
|
247
253
|
this.#transport?.run({
|
|
248
|
-
...
|
|
249
|
-
preludePlan: this.#preludes.plan(
|
|
254
|
+
...input,
|
|
255
|
+
preludePlan: this.#preludes.plan(input.kernelPreludes ?? []),
|
|
250
256
|
});
|
|
251
257
|
} catch (error) {
|
|
252
258
|
const failure = error instanceof Error ? error : new Error(String(error));
|
|
@@ -13,6 +13,7 @@ import asyncio # noqa: ANYIO_OK — stdlib-only embedded kernel runner.
|
|
|
13
13
|
import base64
|
|
14
14
|
import codecs
|
|
15
15
|
import contextlib
|
|
16
|
+
import importlib
|
|
16
17
|
import gc
|
|
17
18
|
import inspect
|
|
18
19
|
import io
|
|
@@ -374,7 +375,15 @@ _BRIDGE_SOCKET_TIMEOUT_SECONDS = 60
|
|
|
374
375
|
_WAIT_SOCKET_GRACE_SECONDS = 30
|
|
375
376
|
|
|
376
377
|
|
|
378
|
+
# The host's secret for the run in this context. Every host call carries it, so the host gives the call that run's
|
|
379
|
+
# kernel tools; copied contexts (parallel workers) keep it, plain threads do not, and the host forgets it at settle.
|
|
380
|
+
CURRENT_CELL_TOKEN: contextvars.ContextVar[str | None] = contextvars.ContextVar("senpi_cell_token", default=None)
|
|
381
|
+
|
|
382
|
+
|
|
377
383
|
def bridge_post(path: str, payload: dict[str, Any], *, socket_timeout: float | None = _BRIDGE_SOCKET_TIMEOUT_SECONDS) -> Any:
|
|
384
|
+
cell_token = CURRENT_CELL_TOKEN.get()
|
|
385
|
+
if path == "/call" and cell_token is not None and "cellToken" not in payload:
|
|
386
|
+
payload = {**payload, "cellToken": cell_token}
|
|
378
387
|
port = CONNECTION.get("port")
|
|
379
388
|
token = CONNECTION.get("token")
|
|
380
389
|
if not isinstance(port, int) or not isinstance(token, str):
|
|
@@ -1088,10 +1097,18 @@ class Workpool:
|
|
|
1088
1097
|
return _workpool_call({"op": "cancel", "pool_id": self.pool_id})
|
|
1089
1098
|
|
|
1090
1099
|
|
|
1091
|
-
def workpool(
|
|
1100
|
+
def workpool(
|
|
1101
|
+
agent: dict[str, JsonValue], name: str, *, mode: str | None = None, tools: list[str] | None = None
|
|
1102
|
+
) -> Workpool:
|
|
1092
1103
|
args: dict[str, JsonValue] = {"op": "create", "agent": agent, "name": name}
|
|
1093
1104
|
if mode is not None:
|
|
1094
1105
|
args["mode"] = mode
|
|
1106
|
+
if tools is not None:
|
|
1107
|
+
if not isinstance(tools, (list, tuple)) or not all(isinstance(name, str) for name in tools):
|
|
1108
|
+
raise PreludeRuntimeError(
|
|
1109
|
+
f"workpool(tools=...) takes a list of tool names; got {type(tools).__name__}", "invalid_tools"
|
|
1110
|
+
)
|
|
1111
|
+
args["tools"] = list(tools)
|
|
1095
1112
|
result = _workpool_call(args)
|
|
1096
1113
|
details = result.get("details")
|
|
1097
1114
|
if isinstance(details, dict):
|
|
@@ -1169,9 +1186,12 @@ def agent(
|
|
|
1169
1186
|
apply: bool | None = None,
|
|
1170
1187
|
merge: bool | str | None = None,
|
|
1171
1188
|
handle: bool = False,
|
|
1189
|
+
tools: list[str] | None = None,
|
|
1172
1190
|
) -> Any:
|
|
1173
1191
|
"""Delegate work; isolated/apply/merge need a host that supports isolation, otherwise a warning.
|
|
1174
1192
|
|
|
1193
|
+
tools grants the child this kernel's @tool functions by name, as JavaScript's agent(prompt, { tools }) does.
|
|
1194
|
+
|
|
1175
1195
|
merge accepts "patch"/"branch" or False/True respectively. Unapplied foreground
|
|
1176
1196
|
changes raise an error with recovery instructions. A handle returns immediately;
|
|
1177
1197
|
await the completion notification or read task_output for the isolation result.
|
|
@@ -1193,6 +1213,10 @@ def agent(
|
|
|
1193
1213
|
args["merge"] = merge
|
|
1194
1214
|
if handle:
|
|
1195
1215
|
args["handle"] = True
|
|
1216
|
+
if tools is not None:
|
|
1217
|
+
if not isinstance(tools, (list, tuple)) or not all(isinstance(name, str) for name in tools):
|
|
1218
|
+
raise PreludeRuntimeError(f"agent(tools=...) takes a list of tool names; got {type(tools).__name__}", "invalid_tools")
|
|
1219
|
+
args["tools"] = list(tools)
|
|
1196
1220
|
|
|
1197
1221
|
response = bridge_post(
|
|
1198
1222
|
"/call",
|
|
@@ -1577,8 +1601,14 @@ _LINE_MAGICS: dict[str, Callable[[str], Any]] = {
|
|
|
1577
1601
|
}
|
|
1578
1602
|
|
|
1579
1603
|
|
|
1604
|
+
_HOST_MAGICS = frozenset({"pip", "environment"})
|
|
1605
|
+
|
|
1606
|
+
|
|
1580
1607
|
def _magic(name: str, args: str) -> Any:
|
|
1581
1608
|
handler = _LINE_MAGICS.get(name)
|
|
1609
|
+
if handler is None and name in _HOST_MAGICS:
|
|
1610
|
+
# The host runs these only as a cell's first code line; anywhere else they reach here.
|
|
1611
|
+
raise PreludeRuntimeError(f"put %{name} on its own cell, then run the code that uses it in the next cell")
|
|
1582
1612
|
if handler is None:
|
|
1583
1613
|
raise PreludeRuntimeError(f"Unsupported line magic: %{name}")
|
|
1584
1614
|
return handler(args)
|
|
@@ -2069,8 +2099,8 @@ KERNEL_MEMORY = _KernelMemory()
|
|
|
2069
2099
|
TLA_FLAG = getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x2000)
|
|
2070
2100
|
|
|
2071
2101
|
|
|
2072
|
-
def compile_cell(source: str) -> tuple[Any | None, Any | None]:
|
|
2073
|
-
module = ast.parse(transform_cell(source), mode="exec")
|
|
2102
|
+
def compile_cell(source: str, filename: str = "<cell>") -> tuple[Any | None, Any | None]:
|
|
2103
|
+
module = ast.parse(transform_cell(source), filename=filename, mode="exec")
|
|
2074
2104
|
if not module.body:
|
|
2075
2105
|
return None, None
|
|
2076
2106
|
last = module.body[-1]
|
|
@@ -2078,13 +2108,13 @@ def compile_cell(source: str) -> tuple[Any | None, Any | None]:
|
|
|
2078
2108
|
body = ast.Module(body=module.body[:-1], type_ignores=[])
|
|
2079
2109
|
expression = ast.Expression(body=last.value)
|
|
2080
2110
|
ast.copy_location(expression, last)
|
|
2081
|
-
return compile(body,
|
|
2111
|
+
return compile(body, filename, "exec", flags=TLA_FLAG), compile(
|
|
2082
2112
|
expression,
|
|
2083
|
-
|
|
2113
|
+
filename,
|
|
2084
2114
|
"eval",
|
|
2085
2115
|
flags=TLA_FLAG,
|
|
2086
2116
|
)
|
|
2087
|
-
return compile(module,
|
|
2117
|
+
return compile(module, filename, "exec", flags=TLA_FLAG), None
|
|
2088
2118
|
|
|
2089
2119
|
|
|
2090
2120
|
async def run_code(code: Any, want_value: bool) -> Any:
|
|
@@ -2111,7 +2141,41 @@ def apply_preludes(preludes: Any) -> None:
|
|
|
2111
2141
|
exec(compile(contribution.get("python", ""), "<kernel-prelude>", "exec"), USER_NS)
|
|
2112
2142
|
|
|
2113
2143
|
|
|
2114
|
-
def
|
|
2144
|
+
def _enter_source_file(source_file: str | None) -> Callable[[], None]:
|
|
2145
|
+
# A %load cell runs as its file: __file__ names it and its directory comes first on the import path,
|
|
2146
|
+
# so `from sibling import x` resolves next to the file the way it does when the file runs as a script.
|
|
2147
|
+
# The returned step undoes both when the cell ends, so a later cell never imports from that directory
|
|
2148
|
+
# ahead of the session environment; the file's own sys.path edits stay.
|
|
2149
|
+
if source_file is None:
|
|
2150
|
+
return lambda: None
|
|
2151
|
+
missing = object()
|
|
2152
|
+
previous_file = USER_NS.get("__file__", missing)
|
|
2153
|
+
directory = os.path.dirname(source_file)
|
|
2154
|
+
previous_index = sys.path.index(directory) if directory in sys.path else None
|
|
2155
|
+
with contextlib.suppress(ValueError):
|
|
2156
|
+
sys.path.remove(directory)
|
|
2157
|
+
sys.path.insert(0, directory)
|
|
2158
|
+
importlib.invalidate_caches()
|
|
2159
|
+
USER_NS["__file__"] = source_file
|
|
2160
|
+
|
|
2161
|
+
def restore() -> None:
|
|
2162
|
+
with contextlib.suppress(ValueError):
|
|
2163
|
+
sys.path.remove(directory)
|
|
2164
|
+
if previous_index is not None:
|
|
2165
|
+
sys.path.insert(min(previous_index, len(sys.path)), directory)
|
|
2166
|
+
if previous_file is missing:
|
|
2167
|
+
USER_NS.pop("__file__", None)
|
|
2168
|
+
else:
|
|
2169
|
+
USER_NS["__file__"] = previous_file
|
|
2170
|
+
# Last: it calls every sys.meta_path finder, and user code may have installed one that raises.
|
|
2171
|
+
importlib.invalidate_caches()
|
|
2172
|
+
|
|
2173
|
+
return restore
|
|
2174
|
+
|
|
2175
|
+
|
|
2176
|
+
def run_cell(
|
|
2177
|
+
cell_id: str, code: str, preludes: Any = None, source_file: str | None = None, cell_token: str | None = None
|
|
2178
|
+
) -> None:
|
|
2115
2179
|
start = time.monotonic()
|
|
2116
2180
|
stdout = io.StringIO()
|
|
2117
2181
|
stderr = io.StringIO()
|
|
@@ -2119,12 +2183,15 @@ def run_cell(cell_id: str, code: str, preludes: Any = None) -> None:
|
|
|
2119
2183
|
# cells) ignores it so a late signal cannot kill the stdin-read loop.
|
|
2120
2184
|
signal.signal(signal.SIGINT, signal.default_int_handler)
|
|
2121
2185
|
result: dict[str, Any]
|
|
2186
|
+
leave_source_file: Callable[[], None] = lambda: None
|
|
2187
|
+
cell_scope = CURRENT_CELL_TOKEN.set(cell_token)
|
|
2122
2188
|
try:
|
|
2123
2189
|
KERNEL_TOOL_TOKEN.acquire()
|
|
2124
2190
|
cell_stdout, cell_stderr = KERNEL_TOOL_STREAMS
|
|
2125
2191
|
with cell_stdout.capture(stdout, cell=True), cell_stderr.capture(stderr, cell=True):
|
|
2126
2192
|
apply_preludes(preludes)
|
|
2127
|
-
|
|
2193
|
+
leave_source_file = _enter_source_file(source_file)
|
|
2194
|
+
body, expression = compile_cell(code, source_file or "<cell>")
|
|
2128
2195
|
LOOP.run_until_complete(run_code(body, False))
|
|
2129
2196
|
value = LOOP.run_until_complete(run_code(expression, True))
|
|
2130
2197
|
text("stdout", stdout.getvalue())
|
|
@@ -2148,7 +2215,13 @@ def run_cell(cell_id: str, code: str, preludes: Any = None) -> None:
|
|
|
2148
2215
|
"durationMs": elapsed(start),
|
|
2149
2216
|
}
|
|
2150
2217
|
finally:
|
|
2218
|
+
# Liveness and revocation first: nothing the cell installed (an import finder, a path hook) can skip them.
|
|
2151
2219
|
KERNEL_TOOL_TOKEN.release()
|
|
2220
|
+
CURRENT_CELL_TOKEN.reset(cell_scope)
|
|
2221
|
+
try:
|
|
2222
|
+
leave_source_file()
|
|
2223
|
+
except BaseException as exc: # noqa: BROAD_EXCEPT_OK — user code can break the restore; it must not wedge the kernel.
|
|
2224
|
+
text("stderr", f"[senpi] %load could not fully restore the import path: {exc}\n")
|
|
2152
2225
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
2153
2226
|
try:
|
|
2154
2227
|
memory = KERNEL_MEMORY.after_cell()
|
|
@@ -2180,7 +2253,18 @@ def _handle_message(message: dict[str, Any]) -> bool:
|
|
|
2180
2253
|
emit({"type": "ready"})
|
|
2181
2254
|
return True
|
|
2182
2255
|
if message_type == "run":
|
|
2183
|
-
|
|
2256
|
+
env_root = message.get("envRoot")
|
|
2257
|
+
if isinstance(env_root, str):
|
|
2258
|
+
_activate_env_root(env_root)
|
|
2259
|
+
source_file = message.get("sourceFile")
|
|
2260
|
+
token = message.get("bridgeCellToken")
|
|
2261
|
+
run_cell(
|
|
2262
|
+
str(message.get("cellId", "")),
|
|
2263
|
+
str(message.get("code", "")),
|
|
2264
|
+
message.get("preludes"),
|
|
2265
|
+
source_file if isinstance(source_file, str) and source_file else None,
|
|
2266
|
+
token if isinstance(token, str) else None,
|
|
2267
|
+
)
|
|
2184
2268
|
return True
|
|
2185
2269
|
if message_type == "close":
|
|
2186
2270
|
KERNEL_TOOL_RUNNER.close()
|
|
@@ -2189,6 +2273,25 @@ def _handle_message(message: dict[str, Any]) -> bool:
|
|
|
2189
2273
|
return True
|
|
2190
2274
|
|
|
2191
2275
|
|
|
2276
|
+
_ACTIVE_ENV_ROOT: list[str] = []
|
|
2277
|
+
|
|
2278
|
+
|
|
2279
|
+
def _activate_env_root(root: str) -> None:
|
|
2280
|
+
# One environment revision is on the import path at a time: a newer revision already holds every
|
|
2281
|
+
# package of the previous one, so it replaces that entry instead of stacking another.
|
|
2282
|
+
# An empty root means the session's current mode has nothing installed: the previous revision leaves too.
|
|
2283
|
+
wanted = [root] if root else []
|
|
2284
|
+
if _ACTIVE_ENV_ROOT == wanted:
|
|
2285
|
+
return
|
|
2286
|
+
for previous in _ACTIVE_ENV_ROOT:
|
|
2287
|
+
with contextlib.suppress(ValueError):
|
|
2288
|
+
sys.path.remove(previous)
|
|
2289
|
+
_ACTIVE_ENV_ROOT[:] = wanted
|
|
2290
|
+
if root:
|
|
2291
|
+
sys.path.insert(0, root)
|
|
2292
|
+
importlib.invalidate_caches()
|
|
2293
|
+
|
|
2294
|
+
|
|
2192
2295
|
def _terminate_process_group() -> None:
|
|
2193
2296
|
# The kernel is spawned into its own session (setsid), so its pid is its process
|
|
2194
2297
|
# group id and a cell's subprocesses inherit that group. Killing the group takes
|
|
@@ -35,6 +35,9 @@ export interface PythonTransportRunInput {
|
|
|
35
35
|
readonly code: string;
|
|
36
36
|
readonly timeoutMs?: number;
|
|
37
37
|
readonly preludePlan?: KernelPreludePlan;
|
|
38
|
+
readonly envRoot?: () => string;
|
|
39
|
+
readonly sourceFile?: string;
|
|
40
|
+
readonly bridgeCellToken?: string;
|
|
38
41
|
}
|
|
39
42
|
|
|
40
43
|
export interface PythonTransportOptions {
|
|
@@ -157,7 +160,16 @@ export class PythonKernelTransport {
|
|
|
157
160
|
install: input.preludePlan.install.map(({ exports, python }) => ({ exports: [...exports], python })),
|
|
158
161
|
remove: [...input.preludePlan.remove],
|
|
159
162
|
};
|
|
160
|
-
this.#write({
|
|
163
|
+
this.#write({
|
|
164
|
+
type: "run",
|
|
165
|
+
cellId: input.cellId,
|
|
166
|
+
code: input.code,
|
|
167
|
+
timeoutMs: input.timeoutMs,
|
|
168
|
+
preludes,
|
|
169
|
+
...(input.envRoot === undefined ? {} : { envRoot: input.envRoot() }),
|
|
170
|
+
...(input.sourceFile === undefined ? {} : { sourceFile: input.sourceFile }),
|
|
171
|
+
...(input.bridgeCellToken === undefined ? {} : { bridgeCellToken: input.bridgeCellToken }),
|
|
172
|
+
});
|
|
161
173
|
}
|
|
162
174
|
|
|
163
175
|
interrupt(reason: string): void {
|
|
@@ -23,9 +23,15 @@ class SenpiWorkpool
|
|
|
23
23
|
end
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
def workpool(agent, name, mode: nil)
|
|
26
|
+
def workpool(agent, name, mode: nil, tools: nil)
|
|
27
27
|
args = { "op" => "create", "agent" => agent, "name" => name }
|
|
28
28
|
args["mode"] = mode unless mode.nil?
|
|
29
|
+
unless tools.nil?
|
|
30
|
+
unless tools.is_a?(Array) && tools.all? { |name| name.is_a?(String) }
|
|
31
|
+
raise SenpiBridgeError.new("workpool(tools: ...) takes an array of tool names; got #{tools.class}", "invalid_tools")
|
|
32
|
+
end
|
|
33
|
+
args["tools"] = tools.dup
|
|
34
|
+
end
|
|
29
35
|
result = __senpi_workpool_call(args)
|
|
30
36
|
details = result["details"]
|
|
31
37
|
if details.is_a?(Hash)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { CellSourceAtStart, EvalKernelRunInput } from "../../tool/types.ts";
|
|
2
|
+
|
|
3
|
+
/** The run input with its turn-time source applied, or the refusal to settle the cell with. */
|
|
4
|
+
export function inputAtStart<T extends EvalKernelRunInput>(input: T): T | { readonly refused: string } {
|
|
5
|
+
if (input.resolveAtStart === undefined) return input;
|
|
6
|
+
let source: CellSourceAtStart;
|
|
7
|
+
try {
|
|
8
|
+
source = input.resolveAtStart();
|
|
9
|
+
} catch (error) {
|
|
10
|
+
return { refused: error instanceof Error ? error.message : String(error) };
|
|
11
|
+
}
|
|
12
|
+
if (!source.ok) return { refused: source.message };
|
|
13
|
+
return {
|
|
14
|
+
...input,
|
|
15
|
+
code: source.code,
|
|
16
|
+
...(source.sourceFile === undefined ? {} : { sourceFile: source.sourceFile }),
|
|
17
|
+
};
|
|
18
|
+
}
|
package/src/tool/cell-handler.ts
CHANGED
|
@@ -48,7 +48,7 @@ export interface CellBridgeRuntime {
|
|
|
48
48
|
readonly ctx: ExtensionContext;
|
|
49
49
|
readonly artifactPath?: string;
|
|
50
50
|
readonly imageResizer?: EvalImageResizer;
|
|
51
|
-
/** This cell's live kernel-tool capability;
|
|
51
|
+
/** This cell's live kernel-tool capability; JS and Python kernels have one (#1754, #2731). */
|
|
52
52
|
readonly kernelTools?: KernelToolsCapability;
|
|
53
53
|
/** The session generation's handle registry behind `wait()` / `handle()` / completion handles. */
|
|
54
54
|
readonly handles?: HandleRegistry;
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
import type { EvalSchemaToolInfo } from "../bridges/schema-bridge.ts";
|
|
9
9
|
import type { CompletionRequest, CompletionResult } from "../completion/handler.ts";
|
|
10
10
|
import type { ResolvedCodemodeSettings } from "../config/settings.ts";
|
|
11
|
+
import type { PythonEnvironments } from "../environments/python-environments.ts";
|
|
11
12
|
import type { EvalExecutionTracker } from "../extension/session-manager.ts";
|
|
12
13
|
import type { HandleRegistry } from "../handles/handle-registry.ts";
|
|
13
14
|
import type { EvalTimeoutFactory } from "./cell-execution.ts";
|
|
@@ -69,6 +70,8 @@ export interface CreateEvalToolOptions {
|
|
|
69
70
|
readonly bunSkillPath?: string;
|
|
70
71
|
/** Kernel globals of the tools active when a cell is submitted; read once per cell. */
|
|
71
72
|
readonly kernelPreludes?: () => readonly KernelPreludeContribution[];
|
|
73
|
+
/** The session's Python environments: `%pip` / `%environment` cells and the import root of every Python cell. */
|
|
74
|
+
readonly pythonEnvironments?: PythonEnvironments;
|
|
72
75
|
/** Contributions whose documentation lines the description lists; snapshot taken when the tool is (re)registered. */
|
|
73
76
|
readonly promptKernelPreludes?: readonly KernelPreludeContribution[];
|
|
74
77
|
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { closeSync, constants, fstatSync, openSync, readFileSync } from "node:fs";
|
|
2
|
+
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
|
|
5
|
+
export type LoadedCell =
|
|
6
|
+
| { readonly ok: true; readonly code: string; readonly sourceFile: string }
|
|
7
|
+
| { readonly ok: false; readonly message: string };
|
|
8
|
+
|
|
9
|
+
export interface LoadCellOptions {
|
|
10
|
+
readonly cwd: string;
|
|
11
|
+
readonly artifactsDir: string | undefined;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const MAX_LOAD_BYTES = 8 * 1024 * 1024;
|
|
15
|
+
const URL_SCHEME = /^([a-z][a-z0-9+.-]*):\/\//i;
|
|
16
|
+
|
|
17
|
+
function errorCode(error: unknown): string | undefined {
|
|
18
|
+
return typeof error === "object" && error !== null && "code" in error && typeof error.code === "string"
|
|
19
|
+
? error.code
|
|
20
|
+
: undefined;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function resolveTarget(target: string, options: LoadCellOptions): { path: string } | { message: string } {
|
|
24
|
+
const scheme = URL_SCHEME.exec(target)?.[1]?.toLowerCase();
|
|
25
|
+
try {
|
|
26
|
+
if (scheme === "file") return { path: fileURLToPath(target) };
|
|
27
|
+
if (scheme === "local") {
|
|
28
|
+
if (options.artifactsDir === undefined)
|
|
29
|
+
return { message: "%load local:// needs a session artifacts directory" };
|
|
30
|
+
const root = join(options.artifactsDir, "local");
|
|
31
|
+
const path = resolve(root, decodeURIComponent(target.slice("local://".length)));
|
|
32
|
+
const inside = relative(root, path);
|
|
33
|
+
if (inside.startsWith("..") || isAbsolute(inside))
|
|
34
|
+
return { message: `%load path escapes local://: ${target}` };
|
|
35
|
+
return { path };
|
|
36
|
+
}
|
|
37
|
+
} catch {
|
|
38
|
+
// fileURLToPath and decodeURIComponent throw on a malformed URL; the message names the user's own text.
|
|
39
|
+
return { message: `%load could not read the path in ${target}` };
|
|
40
|
+
}
|
|
41
|
+
if (scheme !== undefined) {
|
|
42
|
+
return {
|
|
43
|
+
message: `%load reads local files only (a path, local:// or file://); it does not fetch ${scheme}:// URLs`,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
return { path: resolve(options.cwd, target) };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Reads a `%load` target when its cell's turn comes in the kernel's queue, so a file the previous cell wrote is
|
|
51
|
+
* there. Every failure is a refusal that names the target as the user wrote it, never an absolute path.
|
|
52
|
+
*/
|
|
53
|
+
export function loadCell(target: string, options: LoadCellOptions): LoadedCell {
|
|
54
|
+
const resolved = resolveTarget(target, options);
|
|
55
|
+
if ("message" in resolved) return { ok: false, message: resolved.message };
|
|
56
|
+
let fd: number | undefined;
|
|
57
|
+
try {
|
|
58
|
+
// One non-blocking open, then checks on that same handle: a FIFO or device swapped in after a check can
|
|
59
|
+
// neither block the host nor be read, and the size cap keeps a huge file from stalling the event loop.
|
|
60
|
+
fd = openSync(resolved.path, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
61
|
+
const info = fstatSync(fd);
|
|
62
|
+
if (!info.isFile()) return { ok: false, message: `not a file: ${target}` };
|
|
63
|
+
if (info.size > MAX_LOAD_BYTES) return { ok: false, message: `%load reads files up to 8 MiB: ${target}` };
|
|
64
|
+
return { ok: true, code: readFileSync(fd, "utf8"), sourceFile: resolved.path };
|
|
65
|
+
} catch (error) {
|
|
66
|
+
const code = errorCode(error);
|
|
67
|
+
if (code === "ENOENT" || code === "ENOTDIR") return { ok: false, message: `file not found: ${target}` };
|
|
68
|
+
if (code === "EACCES" || code === "EPERM") return { ok: false, message: `permission denied: ${target}` };
|
|
69
|
+
return { ok: false, message: `%load could not read ${target}${code === undefined ? "" : ` (${code})`}` };
|
|
70
|
+
} finally {
|
|
71
|
+
if (fd !== undefined) closeSync(fd);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { PythonEnvironments } from "../environments/python-environments.ts";
|
|
2
|
+
import { TIMEOUT_PAUSE_OP, TIMEOUT_RESUME_OP } from "../timeouts/bridge-timeout.ts";
|
|
3
|
+
import { MagicCellError, parseMagicCell } from "./magic-cells.ts";
|
|
4
|
+
import type { EvalLanguage, HostCellExecutor } from "./types.ts";
|
|
5
|
+
|
|
6
|
+
export type MagicCellPlan =
|
|
7
|
+
| { readonly kind: "ordinary" }
|
|
8
|
+
| { readonly kind: "host"; readonly executor: HostCellExecutor }
|
|
9
|
+
| { readonly kind: "refused"; readonly message: string }
|
|
10
|
+
| { readonly kind: "load"; readonly target: string };
|
|
11
|
+
|
|
12
|
+
export function planMagicCell(
|
|
13
|
+
language: EvalLanguage,
|
|
14
|
+
code: string,
|
|
15
|
+
environments: PythonEnvironments | undefined,
|
|
16
|
+
): MagicCellPlan {
|
|
17
|
+
let magic: ReturnType<typeof parseMagicCell>;
|
|
18
|
+
try {
|
|
19
|
+
magic = parseMagicCell(language, code);
|
|
20
|
+
} catch (error) {
|
|
21
|
+
if (error instanceof MagicCellError) return { kind: "refused", message: error.message };
|
|
22
|
+
throw error;
|
|
23
|
+
}
|
|
24
|
+
if (magic === undefined) return { kind: "ordinary" };
|
|
25
|
+
if (magic.kind === "load") return { kind: "load", target: magic.target };
|
|
26
|
+
if (environments === undefined) {
|
|
27
|
+
return {
|
|
28
|
+
kind: "refused",
|
|
29
|
+
message: "environment_installer_unavailable: this session has no Python interpreter to install packages for",
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
if (magic.kind === "environment") {
|
|
33
|
+
const mode = magic.mode;
|
|
34
|
+
return {
|
|
35
|
+
kind: "host",
|
|
36
|
+
executor: async () => {
|
|
37
|
+
const root = await environments.setMode(mode);
|
|
38
|
+
return { ok: true, valueRepr: `environment: ${mode} (${root})` };
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
const requirements = magic.args;
|
|
43
|
+
return {
|
|
44
|
+
kind: "host",
|
|
45
|
+
// Install time is parked time for the run budget, the same accounting as a bridge call.
|
|
46
|
+
executor: async ({ signal, emit }) => {
|
|
47
|
+
emit({ type: "status", event: { op: TIMEOUT_PAUSE_OP } });
|
|
48
|
+
try {
|
|
49
|
+
const receipt = await environments.install(requirements, signal, (stream, data) =>
|
|
50
|
+
emit({ type: "text", stream, data }),
|
|
51
|
+
);
|
|
52
|
+
const packages = receipt.resolved.length > 0 ? receipt.resolved.join(", ") : "nothing new";
|
|
53
|
+
return {
|
|
54
|
+
ok: true,
|
|
55
|
+
valueRepr: `installed ${packages} into ${receipt.mode} (revision ${receipt.revision}); already-imported modules stay cached until reset`,
|
|
56
|
+
};
|
|
57
|
+
} catch (error) {
|
|
58
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
59
|
+
return { ok: false, error: { message } };
|
|
60
|
+
} finally {
|
|
61
|
+
emit({ type: "status", event: { op: TIMEOUT_RESUME_OP } });
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { EvalLanguage } from "./types.ts";
|
|
2
|
+
|
|
3
|
+
export type MagicCell =
|
|
4
|
+
| { readonly kind: "pip"; readonly args: string }
|
|
5
|
+
| { readonly kind: "environment"; readonly mode: "managed" | "project" }
|
|
6
|
+
| { readonly kind: "load"; readonly target: string };
|
|
7
|
+
|
|
8
|
+
const HOST_MAGICS = ["pip", "environment", "load"] as const;
|
|
9
|
+
const LOAD_LANGUAGES: ReadonlySet<EvalLanguage> = new Set(["py", "js"]);
|
|
10
|
+
const COMMENT_PREFIX: Partial<Record<EvalLanguage, string>> = { py: "#", js: "//" };
|
|
11
|
+
type HostMagic = (typeof HOST_MAGICS)[number];
|
|
12
|
+
|
|
13
|
+
export class MagicCellError extends Error {
|
|
14
|
+
readonly name = "MagicCellError";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function joinContinuations(lines: readonly string[]): string[] {
|
|
18
|
+
const joined: string[] = [];
|
|
19
|
+
let pending: string | undefined;
|
|
20
|
+
for (const line of lines) {
|
|
21
|
+
const current = pending === undefined ? line : `${pending} ${line.trim()}`;
|
|
22
|
+
if (current.trimEnd().endsWith("\\")) pending = current.trimEnd().slice(0, -1).trimEnd();
|
|
23
|
+
else {
|
|
24
|
+
joined.push(current);
|
|
25
|
+
pending = undefined;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
if (pending !== undefined) joined.push(pending);
|
|
29
|
+
return joined;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function hostMagicOf(language: EvalLanguage, line: string): HostMagic | undefined {
|
|
33
|
+
const match = /^%([A-Za-z]+)(?:\s|$)/.exec(line.trim());
|
|
34
|
+
const name = match?.[1];
|
|
35
|
+
const magic = HOST_MAGICS.find((candidate) => candidate === name);
|
|
36
|
+
if (magic === "load") return LOAD_LANGUAGES.has(language) ? magic : undefined;
|
|
37
|
+
return language === "py" ? magic : undefined;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* A cell whose first code line (blank and comment lines skipped) is a host magic runs on the host instead of
|
|
42
|
+
* the interpreter: `%pip` and `%environment` in Python, `%load` in Python and JavaScript. In Python a trailing
|
|
43
|
+
* backslash continues the line. Any other cell is ordinary code, so a magic-looking line later in the cell
|
|
44
|
+
* (say, inside a string) is left alone. A magic followed by more code is refused, because the host step must
|
|
45
|
+
* finish before the code that depends on it runs.
|
|
46
|
+
*/
|
|
47
|
+
export function parseMagicCell(language: EvalLanguage, code: string): MagicCell | undefined {
|
|
48
|
+
const comment = COMMENT_PREFIX[language];
|
|
49
|
+
// Comment lines go before Python's backslash continuations are joined: Python never continues a comment line.
|
|
50
|
+
const raw = code.split("\n").filter((line) => comment === undefined || !line.trim().startsWith(comment));
|
|
51
|
+
const lines = (language === "py" ? joinContinuations(raw) : raw).filter((line) => line.trim() !== "");
|
|
52
|
+
const first = lines[0] ?? "";
|
|
53
|
+
const magic = hostMagicOf(language, first);
|
|
54
|
+
if (magic === undefined) return undefined;
|
|
55
|
+
if (lines.length > 1)
|
|
56
|
+
throw new MagicCellError(`put %${magic} on its own cell, then run the code that uses it in the next cell`);
|
|
57
|
+
const args = first.trim().slice(`%${magic}`.length).trim();
|
|
58
|
+
if (magic === "pip") return { kind: "pip", args };
|
|
59
|
+
if (magic === "load") {
|
|
60
|
+
if (args === "") throw new MagicCellError("%load takes one argument: the path of the file to run");
|
|
61
|
+
return { kind: "load", target: args };
|
|
62
|
+
}
|
|
63
|
+
if (args === "managed" || args === "project") return { kind: "environment", mode: args };
|
|
64
|
+
throw new MagicCellError("%environment takes one argument: managed or project");
|
|
65
|
+
}
|
|
@@ -23,6 +23,8 @@ import { EvalKernelResetRefusedError } from "./eval-kernel-reset-refused-error.t
|
|
|
23
23
|
import { evalTimeoutBehavior } from "./eval-request.ts";
|
|
24
24
|
import type { CreateEvalToolOptions, EvalCellInvocation } from "./eval-tool-options.ts";
|
|
25
25
|
import { describeTimeoutState } from "./interrupt-note.ts";
|
|
26
|
+
import { loadCell } from "./load-cell.ts";
|
|
27
|
+
import { planMagicCell } from "./magic-cell-host.ts";
|
|
26
28
|
import type { EvalKernel, EvalToolDetails } from "./types.ts";
|
|
27
29
|
|
|
28
30
|
export async function runEvalCell(
|
|
@@ -205,11 +207,21 @@ async function executeCell(
|
|
|
205
207
|
const pending = handler.handle(message);
|
|
206
208
|
void pending.catch((error: unknown) => execution.cancel(error));
|
|
207
209
|
};
|
|
210
|
+
let releaseCellKernelTools: (() => void) | undefined;
|
|
208
211
|
try {
|
|
209
212
|
const kernel = await execution.wait(options.kernelManager.getKernel(invocation.input.language, onMessage));
|
|
210
213
|
// Computed before the handler so its construction-time snapshot captures this cell's capability.
|
|
211
214
|
// Worker messages later restore that snapshot before calling host tools (#1754, #2512).
|
|
212
215
|
const kernelTools = kernelToolsFor(kernel);
|
|
216
|
+
// Subprocess kernels call the host over the bridge, outside this async context. Their calls carry a fresh secret
|
|
217
|
+
// minted for this run and sent only to this cell's kernel, never the model-visible cell id, so code in another
|
|
218
|
+
// kernel cannot name this cell into its tools. JS calls already run inside this cell's scope (#1754).
|
|
219
|
+
const bridgeCellToken =
|
|
220
|
+
kernelTools === undefined || invocation.input.language === "js" ? undefined : randomUUID();
|
|
221
|
+
releaseCellKernelTools =
|
|
222
|
+
kernelTools === undefined || bridgeCellToken === undefined
|
|
223
|
+
? undefined
|
|
224
|
+
: options.kernelManager.bindCellKernelTools?.(bridgeCellToken, kernelTools);
|
|
213
225
|
const runBound = async (): Promise<AgentToolResult<EvalToolDetails>> => {
|
|
214
226
|
const queue = kernel.queueSnapshot();
|
|
215
227
|
state.queuedBehind = [...(queue.activeCellId === null ? [] : [queue.activeCellId]), ...queue.queuedCellIds];
|
|
@@ -249,10 +261,26 @@ async function executeCell(
|
|
|
249
261
|
}
|
|
250
262
|
if (invocation.input.reset) await execution.wait(kernel.reset());
|
|
251
263
|
execution.setKernel(kernel);
|
|
264
|
+
const magic = planMagicCell(invocation.input.language, invocation.input.code, options.pythonEnvironments);
|
|
265
|
+
// Resolved when the cell's turn comes in the kernel's queue: a %load reads the file the cells ahead of it
|
|
266
|
+
// wrote, and a refusal settles in queue order like any cell.
|
|
267
|
+
const loadOptions = { cwd: invocation.ctx.cwd, artifactsDir: options.artifactsDir };
|
|
268
|
+
const resolveAtStart =
|
|
269
|
+
magic.kind === "load"
|
|
270
|
+
? () => loadCell(magic.target, loadOptions)
|
|
271
|
+
: magic.kind === "refused"
|
|
272
|
+
? () => ({ ok: false as const, message: magic.message })
|
|
273
|
+
: undefined;
|
|
274
|
+
const environments = invocation.input.language === "py" ? options.pythonEnvironments : undefined;
|
|
275
|
+
const envRoot = environments === undefined ? undefined : () => environments.activeRoot ?? "";
|
|
252
276
|
const result = await execution.wait(
|
|
253
277
|
kernel.run({
|
|
254
278
|
cellId: invocation.cellId,
|
|
255
279
|
code: invocation.input.code,
|
|
280
|
+
...(resolveAtStart === undefined ? {} : { resolveAtStart }),
|
|
281
|
+
...(magic.kind === "host" ? { host: magic.executor } : {}),
|
|
282
|
+
...(envRoot === undefined ? {} : { envRoot }),
|
|
283
|
+
...(bridgeCellToken === undefined ? {} : { bridgeCellToken }),
|
|
256
284
|
kernelPreludes: options.kernelPreludes?.(),
|
|
257
285
|
onMessage,
|
|
258
286
|
onStarted: () => {
|
|
@@ -285,6 +313,8 @@ async function executeCell(
|
|
|
285
313
|
if (error instanceof Error && error.name === "TimeoutError") throw await describeTimeoutState(error, execution);
|
|
286
314
|
throw error;
|
|
287
315
|
} finally {
|
|
316
|
+
// First, so nothing below can skip it: from here on a host call carrying this run's secret gets no kernel tools.
|
|
317
|
+
releaseCellKernelTools?.();
|
|
288
318
|
state.active = false;
|
|
289
319
|
bridgeAbortController.abort();
|
|
290
320
|
execution.finish();
|