@code-yeongyu/senpi-codemode 2026.10.8 → 2026.10.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +100 -4
- package/package.json +4 -4
- package/src/bridge/http-server.ts +9 -1
- package/src/bridge/protocol.ts +4 -0
- package/src/bridges/eval-virtual-schemas.ts +1 -0
- package/src/environments/js-environments.ts +208 -0
- package/src/environments/js-installer.ts +268 -0
- package/src/environments/js-revision-files.ts +91 -0
- package/src/environments/no-links.ts +132 -0
- package/src/environments/project-resolves.ts +54 -0
- package/src/environments/py-installer.ts +64 -4
- package/src/environments/python-environments.ts +85 -0
- package/src/environments/revision-store.ts +38 -7
- package/src/extension/bridge-tool-call.ts +2 -0
- package/src/extension/cell-kernel-tools.ts +28 -0
- package/src/extension/kernel-replacement.ts +20 -0
- package/src/extension/runtime-factory.ts +21 -0
- package/src/extension/session-manager-proxy.ts +6 -0
- package/src/extension/session-manager.ts +10 -2
- package/src/index.ts +2 -0
- package/src/kernels/jl/prelude.jl +6 -1
- package/src/kernels/js/cell-dispatch.ts +44 -0
- package/src/kernels/js/context-manager.ts +25 -15
- package/src/kernels/js/host-entries.ts +62 -0
- package/src/kernels/js/local-module-loader.ts +21 -5
- package/src/kernels/js/worker-core.js +2 -0
- package/src/kernels/js/worker-package-resolve.js +153 -0
- package/src/kernels/js/worker-runtime.js +3 -1
- package/src/kernels/js/workpool.js +5 -2
- package/src/kernels/py/host-cell.ts +8 -31
- package/src/kernels/py/kernel.ts +8 -2
- package/src/kernels/py/prelude.py +112 -9
- package/src/kernels/py/transport.ts +13 -1
- package/src/kernels/rb/workpool.rb +7 -1
- package/src/kernels/sandbox/sandbox-cell.ts +116 -0
- package/src/kernels/sandbox/vendor/pi-codemode/VENDORED.md +1 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/prelude-source.ts +14 -1
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/worker.ts +4 -1
- package/src/kernels/sandbox/vendor/pi-codemode/types.ts +2 -0
- package/src/kernels/shared/cell-source-at-start.ts +18 -0
- package/src/kernels/shared/host-cell.ts +57 -0
- package/src/tool/cell-handler.ts +10 -12
- package/src/tool/eval-request.ts +23 -1
- package/src/tool/eval-tool-options.ts +6 -0
- package/src/tool/eval-tool.ts +16 -4
- package/src/tool/load-cell.ts +88 -0
- package/src/tool/magic-cell-host.ts +123 -0
- package/src/tool/magic-cells.ts +75 -0
- package/src/tool/run-eval-cell.ts +57 -0
- package/src/tool/types.ts +48 -1
|
@@ -46,6 +46,10 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|
|
46
46
|
const options = optionsJson === undefined ? {} : JSON.parse(optionsJson);
|
|
47
47
|
const streamOutput = options.streamOutput === true;
|
|
48
48
|
const rejectStore = options.store === "reject";
|
|
49
|
+
// The engine's own InternalError, captured before user code runs: a failure is tagged by identity, never by its
|
|
50
|
+
// text, so a cell's own error that merely mentions memory keeps its own code. A script can still construct this
|
|
51
|
+
// constructor itself and so label its own failure as the memory limit; that mislabels only its own cell.
|
|
52
|
+
const EngineInternalError = typeof InternalError === "function" ? InternalError : undefined;
|
|
49
53
|
// senpi-change end
|
|
50
54
|
"use strict";
|
|
51
55
|
const stringify = JSON.stringify;
|
|
@@ -94,7 +98,13 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|
|
94
98
|
|
|
95
99
|
function describeError(error) {
|
|
96
100
|
if (error instanceof ErrorCtor) {
|
|
97
|
-
|
|
101
|
+
// senpi-change begin: the engine's out-of-memory error is reported by identity, not by matching its message
|
|
102
|
+
const reason =
|
|
103
|
+
EngineInternalError !== undefined && error instanceof EngineInternalError && error.message === "out of memory"
|
|
104
|
+
? "memory"
|
|
105
|
+
: undefined;
|
|
106
|
+
return stringify({ name: error.name, message: error.message, stack: errorText(error), reason });
|
|
107
|
+
// senpi-change end
|
|
98
108
|
}
|
|
99
109
|
return stringify({ message: format(error) });
|
|
100
110
|
}
|
|
@@ -285,6 +295,8 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|
|
285
295
|
try {
|
|
286
296
|
rendered = outputText(value);
|
|
287
297
|
} catch (error) {
|
|
298
|
+
// The engine's own out-of-memory error keeps its identity, so it is still reported as the memory limit.
|
|
299
|
+
if (EngineInternalError !== undefined && error instanceof EngineInternalError) throw error; // senpi-change
|
|
288
300
|
throw new TypeErrorCtor(error instanceof ErrorCtor ? error.message : String(error));
|
|
289
301
|
}
|
|
290
302
|
output("text", rendered);
|
|
@@ -423,6 +435,7 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|
|
423
435
|
name: "Error",
|
|
424
436
|
message:
|
|
425
437
|
"The script is waiting on a promise that can never settle: no tool call is pending, and timers do not exist here.",
|
|
438
|
+
reason: "unresolved", // senpi-change: tagged by the runtime, so a script's own error text never matches it
|
|
426
439
|
}),
|
|
427
440
|
);
|
|
428
441
|
return true;
|
|
@@ -48,7 +48,10 @@ function discardOutput(memory: { readonly buffer: ArrayBufferLike }) {
|
|
|
48
48
|
function describeException(error: JSException): string {
|
|
49
49
|
const head = error.message ? `${error.name}: ${error.message}` : error.name;
|
|
50
50
|
const stack = error.stack?.trimEnd();
|
|
51
|
-
|
|
51
|
+
// senpi-change begin: evalCode fails before any script runs, so an InternalError here is the engine's own
|
|
52
|
+
const reason = error.name === "InternalError" && error.message === "out of memory" ? "memory" : undefined;
|
|
53
|
+
return JSON.stringify({ name: error.name, message: error.message, stack: stack ? `${head}\n${stack}` : head, reason });
|
|
54
|
+
// senpi-change end
|
|
52
55
|
}
|
|
53
56
|
// senpi-change begin: output streaming
|
|
54
57
|
/**
|
|
@@ -71,6 +71,8 @@ export interface CodemodeError {
|
|
|
71
71
|
name?: string;
|
|
72
72
|
message: string;
|
|
73
73
|
stack?: string;
|
|
74
|
+
/** senpi-change: set by the runtime itself (never from script text) for its own memory-limit and stalled failures. */
|
|
75
|
+
reason?: "memory" | "unresolved";
|
|
74
76
|
}
|
|
75
77
|
|
|
76
78
|
/** Keys the script changed with `store()`. Only successful executions report writes. */
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { CellSourceAtStart, EvalKernelRunInput } from "../../tool/types.ts";
|
|
2
|
+
|
|
3
|
+
/** The run input with its turn-time source applied, or the refusal to settle the cell with. */
|
|
4
|
+
export function inputAtStart<T extends EvalKernelRunInput>(input: T): T | { readonly refused: string } {
|
|
5
|
+
if (input.resolveAtStart === undefined) return input;
|
|
6
|
+
let source: CellSourceAtStart;
|
|
7
|
+
try {
|
|
8
|
+
source = input.resolveAtStart();
|
|
9
|
+
} catch (error) {
|
|
10
|
+
return { refused: error instanceof Error ? error.message : String(error) };
|
|
11
|
+
}
|
|
12
|
+
if (!source.ok) return { refused: source.message };
|
|
13
|
+
return {
|
|
14
|
+
...input,
|
|
15
|
+
code: source.code,
|
|
16
|
+
...(source.sourceFile === undefined ? {} : { sourceFile: source.sourceFile }),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { KernelToHostMessage } from "../../bridge/protocol.ts";
|
|
2
|
+
import type { HostCellExecutor } from "../../tool/types.ts";
|
|
3
|
+
|
|
4
|
+
type HostCellOutcome = Awaited<ReturnType<HostCellExecutor>>;
|
|
5
|
+
type HostResult = Extract<KernelToHostMessage, { type: "result" }>;
|
|
6
|
+
|
|
7
|
+
export interface HostCellRun {
|
|
8
|
+
readonly cellId: string;
|
|
9
|
+
readonly abort: AbortController;
|
|
10
|
+
readonly done: Promise<HostResult>;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Runs a host entry (a cell the host executes, such as an install) for one queue slot: output after the outcome or after
|
|
15
|
+
* an abort belongs to no cell, a synchronous throw stays in this cell, and `settle` receives the cell's result.
|
|
16
|
+
* `settleAfterAbort` says whether an aborted entry still settles here or whoever aborted it settles the cell.
|
|
17
|
+
*/
|
|
18
|
+
export function runHostCell(
|
|
19
|
+
cellId: string,
|
|
20
|
+
host: HostCellExecutor,
|
|
21
|
+
io: {
|
|
22
|
+
readonly emit: (message: KernelToHostMessage) => void;
|
|
23
|
+
readonly settle: (result: HostResult) => void;
|
|
24
|
+
readonly durationMs: () => number;
|
|
25
|
+
readonly settleAfterAbort: boolean;
|
|
26
|
+
},
|
|
27
|
+
): HostCellRun {
|
|
28
|
+
const abort = new AbortController();
|
|
29
|
+
let finished = false;
|
|
30
|
+
const emit = (message: KernelToHostMessage): void => {
|
|
31
|
+
if (!finished && !abort.signal.aborted) io.emit(message);
|
|
32
|
+
};
|
|
33
|
+
const done = new Promise<HostCellOutcome>((resolve) => resolve(host({ signal: abort.signal, emit }))).then(
|
|
34
|
+
(outcome): HostResult =>
|
|
35
|
+
outcome.ok
|
|
36
|
+
? {
|
|
37
|
+
type: "result",
|
|
38
|
+
cellId,
|
|
39
|
+
ok: true,
|
|
40
|
+
durationMs: io.durationMs(),
|
|
41
|
+
...(outcome.valueRepr === undefined ? {} : { valueRepr: outcome.valueRepr }),
|
|
42
|
+
}
|
|
43
|
+
: { type: "result", cellId, ok: false, error: outcome.error, durationMs: 0 },
|
|
44
|
+
(error: unknown): HostResult => ({
|
|
45
|
+
type: "result",
|
|
46
|
+
cellId,
|
|
47
|
+
ok: false,
|
|
48
|
+
error: { message: error instanceof Error ? error.message : String(error) },
|
|
49
|
+
durationMs: 0,
|
|
50
|
+
}),
|
|
51
|
+
);
|
|
52
|
+
void done.then((result) => {
|
|
53
|
+
finished = true;
|
|
54
|
+
if (io.settleAfterAbort || !abort.signal.aborted) io.settle(result);
|
|
55
|
+
});
|
|
56
|
+
return { cellId, abort, done };
|
|
57
|
+
}
|
package/src/tool/cell-handler.ts
CHANGED
|
@@ -48,7 +48,7 @@ export interface CellBridgeRuntime {
|
|
|
48
48
|
readonly ctx: ExtensionContext;
|
|
49
49
|
readonly artifactPath?: string;
|
|
50
50
|
readonly imageResizer?: EvalImageResizer;
|
|
51
|
-
/** This cell's live kernel-tool capability;
|
|
51
|
+
/** This cell's live kernel-tool capability; JS and Python kernels have one (#1754, #2731). */
|
|
52
52
|
readonly kernelTools?: KernelToolsCapability;
|
|
53
53
|
/** The session generation's handle registry behind `wait()` / `handle()` / completion handles. */
|
|
54
54
|
readonly handles?: HandleRegistry;
|
|
@@ -235,22 +235,20 @@ export class CellHandler {
|
|
|
235
235
|
this.#kernel.deliverToolReply({ type: "tool-reply", callId: message.callId, ok: true, value: reply.value });
|
|
236
236
|
} catch (error) {
|
|
237
237
|
if (!this.#state.active) return;
|
|
238
|
-
const
|
|
239
|
-
error instanceof Error ? error.
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
238
|
+
const code =
|
|
239
|
+
error instanceof Error && "code" in error && typeof error.code === "string" ? error.code : undefined;
|
|
240
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
241
|
+
// A blocked call (a permission denial or another hook's veto) is not an argument problem.
|
|
242
|
+
const text =
|
|
243
|
+
code === "blocked"
|
|
244
|
+
? errorMessage
|
|
245
|
+
: appendSchemaHint(errorMessage, message.toolName, this.#toolParameters(message.toolName));
|
|
243
246
|
recordToolCall(this.#state.toolCalls, false, capture, undefined, text);
|
|
244
247
|
this.#kernel.deliverToolReply({
|
|
245
248
|
type: "tool-reply",
|
|
246
249
|
callId: message.callId,
|
|
247
250
|
ok: false,
|
|
248
|
-
error: {
|
|
249
|
-
message: text,
|
|
250
|
-
...(error instanceof Error && "code" in error && typeof error.code === "string"
|
|
251
|
-
? { code: error.code }
|
|
252
|
-
: {}),
|
|
253
|
-
},
|
|
251
|
+
error: { message: text, ...(code === undefined ? {} : { code }) },
|
|
254
252
|
});
|
|
255
253
|
}
|
|
256
254
|
this.#resultBuilder.emitUpdate(false);
|
package/src/tool/eval-request.ts
CHANGED
|
@@ -17,9 +17,19 @@ export function normalizeEvalSummary(value: unknown): string | undefined {
|
|
|
17
17
|
return normalized.length === 0 ? undefined : normalized;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
+
export class EvalIsolateInvalidError extends TypeError {
|
|
21
|
+
readonly name = "EvalIsolateInvalidError";
|
|
22
|
+
readonly code = "eval_isolate_invalid";
|
|
23
|
+
|
|
24
|
+
constructor(reason: string) {
|
|
25
|
+
super(`eval_isolate_invalid: ${reason}`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
20
29
|
export function parseEvalRequest(
|
|
21
30
|
params: unknown,
|
|
22
31
|
enabledLanguages: readonly EvalLanguage[] = evalLanguageOrder,
|
|
32
|
+
options: { readonly sandbox?: boolean } = {},
|
|
23
33
|
): EvalToolRequest {
|
|
24
34
|
if (!isRecord(params)) throw new TypeError("eval parameters must be an object");
|
|
25
35
|
if (params.action === "list") return { action: "list" };
|
|
@@ -30,7 +40,9 @@ export function parseEvalRequest(
|
|
|
30
40
|
}
|
|
31
41
|
if (params.action !== undefined && params.action !== "run")
|
|
32
42
|
throw new TypeError(`Unknown eval action "${String(params.action)}"`);
|
|
33
|
-
if (
|
|
43
|
+
if (params.language === undefined) throw new TypeError(evalRunRequiresLanguageMessage(enabledLanguages));
|
|
44
|
+
if (!isEvalLanguage(params.language))
|
|
45
|
+
throw new TypeError(`eval run language must be one of: ${enabledLanguages.join(", ")}`);
|
|
34
46
|
if (typeof params.code !== "string")
|
|
35
47
|
throw new TypeError("eval run requires code — the cell body to execute, verbatim");
|
|
36
48
|
const summary = normalizeEvalSummary(params.summary);
|
|
@@ -40,6 +52,15 @@ export function parseEvalRequest(
|
|
|
40
52
|
);
|
|
41
53
|
if (params.on_timeout !== undefined && params.on_timeout !== "detach" && params.on_timeout !== "error")
|
|
42
54
|
throw new TypeError(`Unknown eval on_timeout value "${String(params.on_timeout)}"`);
|
|
55
|
+
if (params.isolate !== undefined && params.isolate !== false) {
|
|
56
|
+
if (params.isolate !== true) throw new EvalIsolateInvalidError("isolate must be true or false");
|
|
57
|
+
if (options.sandbox !== true) {
|
|
58
|
+
throw new EvalIsolateInvalidError("sandbox cells are turned off (set sandbox.enabled in codemode settings)");
|
|
59
|
+
}
|
|
60
|
+
if (params.language !== "js")
|
|
61
|
+
throw new EvalIsolateInvalidError('sandbox cells run JavaScript only (language: "js")');
|
|
62
|
+
if (params.reset === true) throw new EvalIsolateInvalidError("a sandbox cell starts fresh; reset does not apply");
|
|
63
|
+
}
|
|
43
64
|
return {
|
|
44
65
|
language: params.language,
|
|
45
66
|
code: params.code,
|
|
@@ -48,6 +69,7 @@ export function parseEvalRequest(
|
|
|
48
69
|
...(typeof params.timeout === "number" ? { timeout: params.timeout } : {}),
|
|
49
70
|
...(params.on_timeout === "detach" || params.on_timeout === "error" ? { on_timeout: params.on_timeout } : {}),
|
|
50
71
|
...(typeof params.reset === "boolean" ? { reset: params.reset } : {}),
|
|
72
|
+
...(params.isolate === true ? { isolate: true } : {}),
|
|
51
73
|
};
|
|
52
74
|
}
|
|
53
75
|
|
|
@@ -8,6 +8,8 @@ import type {
|
|
|
8
8
|
import type { EvalSchemaToolInfo } from "../bridges/schema-bridge.ts";
|
|
9
9
|
import type { CompletionRequest, CompletionResult } from "../completion/handler.ts";
|
|
10
10
|
import type { ResolvedCodemodeSettings } from "../config/settings.ts";
|
|
11
|
+
import type { JsEnvironments } from "../environments/js-environments.ts";
|
|
12
|
+
import type { PythonEnvironments } from "../environments/python-environments.ts";
|
|
11
13
|
import type { EvalExecutionTracker } from "../extension/session-manager.ts";
|
|
12
14
|
import type { HandleRegistry } from "../handles/handle-registry.ts";
|
|
13
15
|
import type { EvalTimeoutFactory } from "./cell-execution.ts";
|
|
@@ -69,6 +71,10 @@ export interface CreateEvalToolOptions {
|
|
|
69
71
|
readonly bunSkillPath?: string;
|
|
70
72
|
/** Kernel globals of the tools active when a cell is submitted; read once per cell. */
|
|
71
73
|
readonly kernelPreludes?: () => readonly KernelPreludeContribution[];
|
|
74
|
+
/** The session's Python environments: `%pip` / `%environment` cells and the import root of every Python cell. */
|
|
75
|
+
readonly pythonEnvironments?: PythonEnvironments;
|
|
76
|
+
/** The session's managed JavaScript packages: `%bun add` / `%npm add` cells and the bare-import fallback root. */
|
|
77
|
+
readonly jsEnvironments?: JsEnvironments;
|
|
72
78
|
/** Contributions whose documentation lines the description lists; snapshot taken when the tool is (re)registered. */
|
|
73
79
|
readonly promptKernelPreludes?: readonly KernelPreludeContribution[];
|
|
74
80
|
}
|
package/src/tool/eval-tool.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AgentToolResult, ToolDefinition } from "@code-yeongyu/senpi";
|
|
2
2
|
import type { TUnsafe } from "typebox";
|
|
3
|
+
import { resolveSandbox } from "../config/feature-settings.ts";
|
|
3
4
|
import {
|
|
4
5
|
DEFAULT_FOREGROUND_WINDOW_SECONDS,
|
|
5
6
|
defaultCodemodeSettings,
|
|
@@ -8,7 +9,12 @@ import {
|
|
|
8
9
|
import { buildEvalPrompt } from "../prompt/eval-prompt.ts";
|
|
9
10
|
import { EvalDetachedCellManager } from "./detached-cell-manager.ts";
|
|
10
11
|
import { executeEvalControl } from "./detached-eval-result.ts";
|
|
11
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
EvalIsolateInvalidError,
|
|
14
|
+
isEvalControlRequest,
|
|
15
|
+
normalizeEvalSummary,
|
|
16
|
+
parseEvalRequest,
|
|
17
|
+
} from "./eval-request.ts";
|
|
12
18
|
import type { CreateEvalToolOptions } from "./eval-tool-options.ts";
|
|
13
19
|
import { runEvalCell } from "./run-eval-cell.ts";
|
|
14
20
|
import {
|
|
@@ -43,7 +49,8 @@ export function createEvalTool(options: CreateEvalToolOptions) {
|
|
|
43
49
|
foregroundWindowSeconds,
|
|
44
50
|
hardLimitSeconds: options.hardLimitSeconds ?? defaultEvalDeadlineSeconds.hardLimitSeconds,
|
|
45
51
|
};
|
|
46
|
-
const
|
|
52
|
+
const sandbox = resolveSandbox(options.settings ?? defaultCodemodeSettings).enabled;
|
|
53
|
+
const parameters = createEvalInputSchema(options.enabledLanguages, deadlines, { sandbox });
|
|
47
54
|
const prompt = buildEvalPrompt(options.enabledLanguages, {
|
|
48
55
|
spawns: options.spawns ?? false,
|
|
49
56
|
monitor: options.monitor,
|
|
@@ -74,9 +81,14 @@ export function createEvalTool(options: CreateEvalToolOptions) {
|
|
|
74
81
|
async function execute(
|
|
75
82
|
...[toolCallId, params, signal, onUpdate, ctx]: EvalExecuteArgs<EvalToolRequest>
|
|
76
83
|
): Promise<AgentToolResult<EvalResultDetails>> {
|
|
77
|
-
const request = parseEvalRequest(params, languages);
|
|
84
|
+
const request = parseEvalRequest(params, languages, { sandbox });
|
|
78
85
|
if (isEvalControlRequest(request)) return await executeEvalControl(cellManager, request);
|
|
79
|
-
if (options.proxyExecutor)
|
|
86
|
+
if (options.proxyExecutor) {
|
|
87
|
+
// A proxy runs cells elsewhere and knows nothing of sandbox cells: an isolated cell must never run unisolated.
|
|
88
|
+
if ("isolate" in request && request.isolate === true)
|
|
89
|
+
throw new EvalIsolateInvalidError("isolate: true is not available through this eval proxy");
|
|
90
|
+
return await options.proxyExecutor(request, signal);
|
|
91
|
+
}
|
|
80
92
|
if (!languages.includes(request.language))
|
|
81
93
|
throw new RangeError(
|
|
82
94
|
`Unsupported eval language "${request.language}". Enabled languages: ${languages.join(", ")}`,
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
|
|
2
|
+
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
|
|
5
|
+
export type LoadedCell =
|
|
6
|
+
| { readonly ok: true; readonly code: string; readonly sourceFile: string }
|
|
7
|
+
| { readonly ok: false; readonly message: string };
|
|
8
|
+
|
|
9
|
+
export interface LoadCellOptions {
|
|
10
|
+
readonly cwd: string;
|
|
11
|
+
readonly artifactsDir: string | undefined;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const MAX_LOAD_BYTES = 8 * 1024 * 1024;
|
|
15
|
+
const URL_SCHEME = /^([a-z][a-z0-9+.-]*):\/\//i;
|
|
16
|
+
|
|
17
|
+
function errorCode(error: unknown): string | undefined {
|
|
18
|
+
return typeof error === "object" && error !== null && "code" in error && typeof error.code === "string"
|
|
19
|
+
? error.code
|
|
20
|
+
: undefined;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function resolveTarget(target: string, options: LoadCellOptions): { path: string } | { message: string } {
|
|
24
|
+
const scheme = URL_SCHEME.exec(target)?.[1]?.toLowerCase();
|
|
25
|
+
try {
|
|
26
|
+
if (scheme === "file") return { path: fileURLToPath(target) };
|
|
27
|
+
if (scheme === "local") {
|
|
28
|
+
if (options.artifactsDir === undefined)
|
|
29
|
+
return { message: "%load local:// needs a session artifacts directory" };
|
|
30
|
+
const root = join(options.artifactsDir, "local");
|
|
31
|
+
const path = resolve(root, decodeURIComponent(target.slice("local://".length)));
|
|
32
|
+
const inside = relative(root, path);
|
|
33
|
+
if (inside.startsWith("..") || isAbsolute(inside))
|
|
34
|
+
return { message: `%load path escapes local://: ${target}` };
|
|
35
|
+
return { path };
|
|
36
|
+
}
|
|
37
|
+
} catch {
|
|
38
|
+
// fileURLToPath and decodeURIComponent throw on a malformed URL; the message names the user's own text.
|
|
39
|
+
return { message: `%load could not read the path in ${target}` };
|
|
40
|
+
}
|
|
41
|
+
if (scheme !== undefined) {
|
|
42
|
+
return {
|
|
43
|
+
message: `%load reads local files only (a path, local:// or file://); it does not fetch ${scheme}:// URLs`,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
return { path: resolve(options.cwd, target) };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Reads a `%load` target when its cell's turn comes in the kernel's queue, so a file the previous cell wrote is
|
|
51
|
+
* there. Every failure is a refusal that names the target as the user wrote it, never an absolute path.
|
|
52
|
+
*/
|
|
53
|
+
/** Reads from `fd` until EOF or `limit` bytes, whichever comes first; a file still growing cannot exceed it. */
|
|
54
|
+
export function readAtMost(fd: number, limit: number): Buffer {
|
|
55
|
+
const buffer = Buffer.alloc(limit);
|
|
56
|
+
let filled = 0;
|
|
57
|
+
while (filled < limit) {
|
|
58
|
+
const read = readSync(fd, buffer, filled, limit - filled, null);
|
|
59
|
+
if (read === 0) break;
|
|
60
|
+
filled += read;
|
|
61
|
+
}
|
|
62
|
+
return buffer.subarray(0, filled);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function loadCell(target: string, options: LoadCellOptions): LoadedCell {
|
|
66
|
+
const resolved = resolveTarget(target, options);
|
|
67
|
+
if ("message" in resolved) return { ok: false, message: resolved.message };
|
|
68
|
+
let fd: number | undefined;
|
|
69
|
+
try {
|
|
70
|
+
// One non-blocking open, then checks on that same handle: a FIFO or device swapped in after a check can
|
|
71
|
+
// neither block the host nor be read, and the size cap keeps a huge file from stalling the event loop.
|
|
72
|
+
fd = openSync(resolved.path, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
73
|
+
const info = fstatSync(fd);
|
|
74
|
+
if (!info.isFile()) return { ok: false, message: `not a file: ${target}` };
|
|
75
|
+
const tooLarge = { ok: false as const, message: `%load reads files up to 8 MiB: ${target}` };
|
|
76
|
+
if (info.size > MAX_LOAD_BYTES) return tooLarge;
|
|
77
|
+
const bytes = readAtMost(fd, MAX_LOAD_BYTES + 1);
|
|
78
|
+
if (bytes.length > MAX_LOAD_BYTES) return tooLarge;
|
|
79
|
+
return { ok: true, code: bytes.toString("utf8"), sourceFile: resolved.path };
|
|
80
|
+
} catch (error) {
|
|
81
|
+
const code = errorCode(error);
|
|
82
|
+
if (code === "ENOENT" || code === "ENOTDIR") return { ok: false, message: `file not found: ${target}` };
|
|
83
|
+
if (code === "EACCES" || code === "EPERM") return { ok: false, message: `permission denied: ${target}` };
|
|
84
|
+
return { ok: false, message: `%load could not read ${target}${code === undefined ? "" : ` (${code})`}` };
|
|
85
|
+
} finally {
|
|
86
|
+
if (fd !== undefined) closeSync(fd);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import type { JsEnvironments } from "../environments/js-environments.ts";
|
|
2
|
+
import type { PythonEnvironments } from "../environments/python-environments.ts";
|
|
3
|
+
import { TIMEOUT_PAUSE_OP, TIMEOUT_RESUME_OP } from "../timeouts/bridge-timeout.ts";
|
|
4
|
+
import { MagicCellError, parseMagicCell } from "./magic-cells.ts";
|
|
5
|
+
import type { EvalLanguage, HostCellExecutor } from "./types.ts";
|
|
6
|
+
|
|
7
|
+
export type MagicCellPlan =
|
|
8
|
+
| { readonly kind: "ordinary" }
|
|
9
|
+
| { readonly kind: "host"; readonly executor: HostCellExecutor }
|
|
10
|
+
| { readonly kind: "refused"; readonly message: string }
|
|
11
|
+
| { readonly kind: "load"; readonly target: string };
|
|
12
|
+
|
|
13
|
+
export function planMagicCell(
|
|
14
|
+
language: EvalLanguage,
|
|
15
|
+
code: string,
|
|
16
|
+
environments: PythonEnvironments | undefined,
|
|
17
|
+
jsEnvironments?: JsEnvironments,
|
|
18
|
+
): MagicCellPlan {
|
|
19
|
+
let magic: ReturnType<typeof parseMagicCell>;
|
|
20
|
+
try {
|
|
21
|
+
magic = parseMagicCell(language, code);
|
|
22
|
+
} catch (error) {
|
|
23
|
+
if (error instanceof MagicCellError) return { kind: "refused", message: error.message };
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
if (magic === undefined) return { kind: "ordinary" };
|
|
27
|
+
if (magic.kind === "load") return { kind: "load", target: magic.target };
|
|
28
|
+
if (language === "js") return planJsMagic(magic, jsEnvironments);
|
|
29
|
+
if (environments === undefined) {
|
|
30
|
+
return {
|
|
31
|
+
kind: "refused",
|
|
32
|
+
message: "environment_installer_unavailable: this session has no Python interpreter to install packages for",
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
if (magic.kind === "environment") {
|
|
36
|
+
const mode = magic.mode;
|
|
37
|
+
return {
|
|
38
|
+
kind: "host",
|
|
39
|
+
executor: async () => {
|
|
40
|
+
await environments.setMode(mode);
|
|
41
|
+
const where = mode === "project" ? "the session directory" : "this session's managed revisions";
|
|
42
|
+
return { ok: true, valueRepr: `environment: ${mode} (${where})` };
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const requirements = magic.args;
|
|
47
|
+
return {
|
|
48
|
+
kind: "host",
|
|
49
|
+
// Install time is parked time for the run budget, the same accounting as a bridge call.
|
|
50
|
+
executor: async ({ signal, emit }) => {
|
|
51
|
+
emit({ type: "status", event: { op: TIMEOUT_PAUSE_OP } });
|
|
52
|
+
try {
|
|
53
|
+
const receipt = await environments.install(requirements, signal, (stream, data) =>
|
|
54
|
+
emit({ type: "text", stream, data }),
|
|
55
|
+
);
|
|
56
|
+
const packages = receipt.resolved.length > 0 ? receipt.resolved.join(", ") : "nothing new";
|
|
57
|
+
return {
|
|
58
|
+
ok: true,
|
|
59
|
+
valueRepr: `installed ${packages} into ${receipt.mode} (revision ${receipt.revision}); already-imported modules stay cached until reset`,
|
|
60
|
+
};
|
|
61
|
+
} catch (error) {
|
|
62
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
63
|
+
return { ok: false, error: { message } };
|
|
64
|
+
} finally {
|
|
65
|
+
emit({ type: "status", event: { op: TIMEOUT_RESUME_OP } });
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function planJsMagic(
|
|
72
|
+
magic: Exclude<NonNullable<ReturnType<typeof parseMagicCell>>, { kind: "load" }>,
|
|
73
|
+
environments: JsEnvironments | undefined,
|
|
74
|
+
): MagicCellPlan {
|
|
75
|
+
if (environments === undefined) {
|
|
76
|
+
return {
|
|
77
|
+
kind: "refused",
|
|
78
|
+
message: "environment_installer_unavailable: this session has no JavaScript package environment",
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
if (magic.kind === "environment") {
|
|
82
|
+
const mode = magic.mode;
|
|
83
|
+
return {
|
|
84
|
+
kind: "host",
|
|
85
|
+
executor: async () => {
|
|
86
|
+
await environments.setMode(mode);
|
|
87
|
+
const where = mode === "project" ? "the session directory" : "this session's managed revisions";
|
|
88
|
+
return { ok: true, valueRepr: `environment: ${mode} (${where})` };
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
if (magic.kind !== "js-add") return { kind: "refused", message: "this magic is not available in JavaScript" };
|
|
93
|
+
const requested = magic.args;
|
|
94
|
+
return {
|
|
95
|
+
kind: "host",
|
|
96
|
+
executor: async ({ signal, emit }) => {
|
|
97
|
+
emit({ type: "status", event: { op: TIMEOUT_PAUSE_OP } });
|
|
98
|
+
try {
|
|
99
|
+
const receipt = await environments.install(
|
|
100
|
+
requested,
|
|
101
|
+
signal,
|
|
102
|
+
(stream, data) => emit({ type: "text", stream, data }),
|
|
103
|
+
magic.installer,
|
|
104
|
+
);
|
|
105
|
+
const added = receipt.added.length > 0 ? receipt.added.join(", ") : "nothing new";
|
|
106
|
+
const where =
|
|
107
|
+
receipt.revision === undefined ? receipt.mode : `${receipt.mode} (revision ${receipt.revision})`;
|
|
108
|
+
const shadow =
|
|
109
|
+
receipt.shadowed.length > 0
|
|
110
|
+
? `; environment_resolution_conflict: ${receipt.shadowed.join(", ")} still ${receipt.shadowed.length === 1 ? "resolves" : "resolve"} from the project's node_modules first`
|
|
111
|
+
: "";
|
|
112
|
+
return {
|
|
113
|
+
ok: true,
|
|
114
|
+
valueRepr: `added ${added} with ${receipt.installer} into ${where}${shadow}; already-imported modules stay cached until reset`,
|
|
115
|
+
};
|
|
116
|
+
} catch (error) {
|
|
117
|
+
return { ok: false, error: { message: error instanceof Error ? error.message : String(error) } };
|
|
118
|
+
} finally {
|
|
119
|
+
emit({ type: "status", event: { op: TIMEOUT_RESUME_OP } });
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { EvalLanguage } from "./types.ts";
|
|
2
|
+
|
|
3
|
+
export type MagicCell =
|
|
4
|
+
| { readonly kind: "pip"; readonly args: string }
|
|
5
|
+
| { readonly kind: "environment"; readonly mode: "managed" | "project" }
|
|
6
|
+
| { readonly kind: "load"; readonly target: string }
|
|
7
|
+
| { readonly kind: "js-add"; readonly installer: "bun" | "npm"; readonly args: string };
|
|
8
|
+
|
|
9
|
+
const HOST_MAGICS = ["pip", "environment", "load", "bun", "npm"] as const;
|
|
10
|
+
const LOAD_LANGUAGES: ReadonlySet<EvalLanguage> = new Set(["py", "js"]);
|
|
11
|
+
const COMMENT_PREFIX: Partial<Record<EvalLanguage, string>> = { py: "#", js: "//" };
|
|
12
|
+
type HostMagic = (typeof HOST_MAGICS)[number];
|
|
13
|
+
|
|
14
|
+
export class MagicCellError extends Error {
|
|
15
|
+
readonly name = "MagicCellError";
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function joinContinuations(lines: readonly string[]): string[] {
|
|
19
|
+
const joined: string[] = [];
|
|
20
|
+
let pending: string | undefined;
|
|
21
|
+
for (const line of lines) {
|
|
22
|
+
const current = pending === undefined ? line : `${pending} ${line.trim()}`;
|
|
23
|
+
if (current.trimEnd().endsWith("\\")) pending = current.trimEnd().slice(0, -1).trimEnd();
|
|
24
|
+
else {
|
|
25
|
+
joined.push(current);
|
|
26
|
+
pending = undefined;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
if (pending !== undefined) joined.push(pending);
|
|
30
|
+
return joined;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function hostMagicOf(language: EvalLanguage, line: string): HostMagic | undefined {
|
|
34
|
+
const match = /^%([A-Za-z]+)(?:\s|$)/.exec(line.trim());
|
|
35
|
+
const name = match?.[1];
|
|
36
|
+
const magic = HOST_MAGICS.find((candidate) => candidate === name);
|
|
37
|
+
if (magic === "load") return LOAD_LANGUAGES.has(language) ? magic : undefined;
|
|
38
|
+
if (magic === "bun" || magic === "npm") return language === "js" ? magic : undefined;
|
|
39
|
+
if (magic === "environment") return language === "py" || language === "js" ? magic : undefined;
|
|
40
|
+
return language === "py" ? magic : undefined;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* A cell whose first code line (blank and comment lines skipped) is a host magic runs on the host instead of
|
|
45
|
+
* the interpreter: `%pip` and `%environment` in Python, `%load` in Python and JavaScript. In Python a trailing
|
|
46
|
+
* backslash continues the line. Any other cell is ordinary code, so a magic-looking line later in the cell
|
|
47
|
+
* (say, inside a string) is left alone. A magic followed by more code is refused, because the host step must
|
|
48
|
+
* finish before the code that depends on it runs.
|
|
49
|
+
*/
|
|
50
|
+
export function parseMagicCell(language: EvalLanguage, code: string): MagicCell | undefined {
|
|
51
|
+
const comment = COMMENT_PREFIX[language];
|
|
52
|
+
// Comment lines go before Python's backslash continuations are joined: Python never continues a comment line.
|
|
53
|
+
const raw = code.split("\n").filter((line) => comment === undefined || !line.trim().startsWith(comment));
|
|
54
|
+
const lines = (language === "py" ? joinContinuations(raw) : raw).filter((line) => line.trim() !== "");
|
|
55
|
+
const first = lines[0] ?? "";
|
|
56
|
+
const magic = hostMagicOf(language, first);
|
|
57
|
+
if (magic === undefined) return undefined;
|
|
58
|
+
if (lines.length > 1)
|
|
59
|
+
throw new MagicCellError(`put %${magic} on its own cell, then run the code that uses it in the next cell`);
|
|
60
|
+
const args = first.trim().slice(`%${magic}`.length).trim();
|
|
61
|
+
if (magic === "pip") return { kind: "pip", args };
|
|
62
|
+
if (magic === "bun" || magic === "npm") {
|
|
63
|
+
const [verb, ...rest] = args.split(/\s+/);
|
|
64
|
+
if (verb !== "add" && verb !== "install") {
|
|
65
|
+
throw new MagicCellError(`%${magic} supports only add: %${magic} add <package ...>`);
|
|
66
|
+
}
|
|
67
|
+
return { kind: "js-add", installer: magic, args: rest.join(" ") };
|
|
68
|
+
}
|
|
69
|
+
if (magic === "load") {
|
|
70
|
+
if (args === "") throw new MagicCellError("%load takes one argument: the path of the file to run");
|
|
71
|
+
return { kind: "load", target: args };
|
|
72
|
+
}
|
|
73
|
+
if (args === "managed" || args === "project") return { kind: "environment", mode: args };
|
|
74
|
+
throw new MagicCellError("%environment takes one argument: managed or project");
|
|
75
|
+
}
|