@code-yeongyu/senpi-codemode 2026.10.7 → 2026.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +16 -2
- package/package.json +5 -4
- package/src/bridge/kernel-tools-protocol.ts +8 -0
- package/src/bridges/eval-virtual-schemas.ts +21 -0
- package/src/environments/editable-check.ts +116 -0
- package/src/environments/install-lock.ts +223 -0
- package/src/environments/py-environment.ts +98 -0
- package/src/environments/py-installer.ts +187 -0
- package/src/environments/revision-store.ts +95 -0
- package/src/extension/kernel-registration.ts +5 -0
- package/src/extension/session-manager.ts +15 -1
- package/src/kernels/js/kernel-tools-errors.js +2 -0
- package/src/kernels/js/kernel-tools-errors.ts +2 -0
- package/src/kernels/py/kernel-contract.ts +11 -2
- package/src/kernels/py/kernel-tools-host.ts +97 -0
- package/src/kernels/py/kernel.ts +40 -8
- package/src/kernels/py/prelude.py +659 -9
- package/src/kernels/py/process.ts +15 -4
- package/src/kernels/py/startup.ts +58 -20
- package/src/kernels/py/transport.ts +36 -7
- package/src/kernels/sandbox/vendor/pi-codemode/LICENSE +21 -0
- package/src/kernels/sandbox/vendor/pi-codemode/VENDORED.md +42 -0
- package/src/kernels/sandbox/vendor/pi-codemode/identifier.ts +14 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/host.ts +514 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/prelude-source.ts +431 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/protocol.ts +61 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/worker.ts +231 -0
- package/src/kernels/sandbox/vendor/pi-codemode/types.ts +185 -0
- package/src/kernels/sandbox/vendor/pi-codemode/wasm.ts +37 -0
- package/src/kernels/shared/kernel-tools-pump.ts +6 -0
- package/src/kernels/shared/process-cpu.ts +69 -0
- package/src/kernels/shared/subprocess-queue.ts +1 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { devNull } from "node:os";
|
|
3
|
+
import { terminateProcessTrees } from "../kernels/js/process-tree-host.ts";
|
|
4
|
+
|
|
5
|
+
export type EnvironmentErrorCode =
|
|
6
|
+
| "environment_install_failed"
|
|
7
|
+
| "environment_install_timeout"
|
|
8
|
+
| "environment_install_cancelled"
|
|
9
|
+
| "environment_installer_unavailable"
|
|
10
|
+
| "environment_resolution_conflict"
|
|
11
|
+
| "environment_language_mismatch";
|
|
12
|
+
|
|
13
|
+
export class EnvironmentError extends Error {
|
|
14
|
+
readonly name = "EnvironmentError";
|
|
15
|
+
readonly code: EnvironmentErrorCode;
|
|
16
|
+
|
|
17
|
+
constructor(code: EnvironmentErrorCode, message: string) {
|
|
18
|
+
super(`${code}: ${message}`);
|
|
19
|
+
this.code = code;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// pip accepts any unambiguous prefix of a long option and grouped short flags, so a denylist of destination
|
|
24
|
+
// options can't be complete. Only these options are accepted, each spelled out in full.
|
|
25
|
+
const ALLOWED_FLAGS = new Set([
|
|
26
|
+
"--upgrade",
|
|
27
|
+
"--no-deps",
|
|
28
|
+
"--pre",
|
|
29
|
+
"--force-reinstall",
|
|
30
|
+
"--no-index",
|
|
31
|
+
"--no-cache-dir",
|
|
32
|
+
"--only-binary",
|
|
33
|
+
"--no-binary",
|
|
34
|
+
"--prefer-binary",
|
|
35
|
+
"--index-url",
|
|
36
|
+
"--extra-index-url",
|
|
37
|
+
"--find-links",
|
|
38
|
+
"--constraint",
|
|
39
|
+
"--requirement",
|
|
40
|
+
"--quiet",
|
|
41
|
+
"--verbose",
|
|
42
|
+
]);
|
|
43
|
+
const FLAGS_WITH_VALUE = new Set([
|
|
44
|
+
"--only-binary",
|
|
45
|
+
"--no-binary",
|
|
46
|
+
"--index-url",
|
|
47
|
+
"--extra-index-url",
|
|
48
|
+
"--find-links",
|
|
49
|
+
"--constraint",
|
|
50
|
+
"--requirement",
|
|
51
|
+
]);
|
|
52
|
+
const SHORT_ALIASES: Readonly<Record<string, string>> = {
|
|
53
|
+
"-U": "--upgrade",
|
|
54
|
+
"-i": "--index-url",
|
|
55
|
+
"-f": "--find-links",
|
|
56
|
+
"-c": "--constraint",
|
|
57
|
+
"-r": "--requirement",
|
|
58
|
+
"-q": "--quiet",
|
|
59
|
+
"-v": "--verbose",
|
|
60
|
+
};
|
|
61
|
+
const STDERR_TAIL_BYTES = 4_096;
|
|
62
|
+
const PIP_TREE_GRACE_MS = 2_000;
|
|
63
|
+
|
|
64
|
+
export function parsePipRequirements(text: string): string[] {
|
|
65
|
+
const args = text
|
|
66
|
+
.trim()
|
|
67
|
+
.split(/\s+/)
|
|
68
|
+
.filter((arg) => arg !== "");
|
|
69
|
+
const command = args[0] === "install" ? args.slice(1) : undefined;
|
|
70
|
+
if (command === undefined) {
|
|
71
|
+
throw new EnvironmentError("environment_install_failed", "only `%pip install <requirements>` is supported");
|
|
72
|
+
}
|
|
73
|
+
if (command.length === 0) throw new EnvironmentError("environment_install_failed", "name at least one requirement");
|
|
74
|
+
const normalized: string[] = [];
|
|
75
|
+
for (let index = 0; index < command.length; index++) {
|
|
76
|
+
const arg = command[index] ?? "";
|
|
77
|
+
if (!arg.startsWith("-")) {
|
|
78
|
+
normalized.push(arg);
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
const [spelled = arg, attached] = arg.split(/=(.*)/su, 2);
|
|
82
|
+
const flag = SHORT_ALIASES[spelled] ?? spelled;
|
|
83
|
+
if (!ALLOWED_FLAGS.has(flag)) {
|
|
84
|
+
throw new EnvironmentError(
|
|
85
|
+
"environment_install_failed",
|
|
86
|
+
`${spelled} is not allowed: %pip accepts ${[...ALLOWED_FLAGS].join(", ")} (each spelled out in full); packages always install into the session's environment root`,
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
if (!FLAGS_WITH_VALUE.has(flag)) {
|
|
90
|
+
if (attached !== undefined)
|
|
91
|
+
throw new EnvironmentError("environment_install_failed", `${spelled} takes no value`);
|
|
92
|
+
normalized.push(flag);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const value = attached ?? command[++index];
|
|
96
|
+
if (value === undefined || value === "" || value.startsWith("-")) {
|
|
97
|
+
throw new EnvironmentError("environment_install_failed", `${spelled} needs a value`);
|
|
98
|
+
}
|
|
99
|
+
normalized.push(`${flag}=${value}`);
|
|
100
|
+
}
|
|
101
|
+
const named = normalized.some((arg) => !arg.startsWith("-") || arg.startsWith("--requirement="));
|
|
102
|
+
if (!named) throw new EnvironmentError("environment_install_failed", "name at least one requirement");
|
|
103
|
+
return normalized;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* pip's environment with every PIP_* variable removed and its config file pointed at nothing: `--isolated`
|
|
108
|
+
* alone still honours PIP_CONFIG_FILE, and a configured target, root or prefix would install outside the revision.
|
|
109
|
+
*/
|
|
110
|
+
export function isolatedPipEnv(): NodeJS.ProcessEnv {
|
|
111
|
+
const env: NodeJS.ProcessEnv = {};
|
|
112
|
+
for (const [key, value] of Object.entries(process.env)) {
|
|
113
|
+
if (!key.toUpperCase().startsWith("PIP_")) env[key] = value;
|
|
114
|
+
}
|
|
115
|
+
// pip skips every config file when PIP_CONFIG_FILE equals Python's os.devnull: "nul" on Windows, not Node's "\\\\.\\nul".
|
|
116
|
+
return { ...env, PIP_CONFIG_FILE: process.platform === "win32" ? "nul" : devNull, PYTHONNOUSERSITE: "1" };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function runPipInstall(input: {
|
|
120
|
+
readonly interpreter: string;
|
|
121
|
+
readonly root: string;
|
|
122
|
+
readonly args: readonly string[];
|
|
123
|
+
readonly cwd: string;
|
|
124
|
+
readonly signal: AbortSignal;
|
|
125
|
+
readonly onOutput?: (stream: "stdout" | "stderr", data: string) => void;
|
|
126
|
+
}): Promise<void> {
|
|
127
|
+
const argv = [
|
|
128
|
+
"-m",
|
|
129
|
+
"pip",
|
|
130
|
+
"install",
|
|
131
|
+
// Ignore pip's config files and PIP_* variables: a configured `root` or `prefix` would write outside the revision.
|
|
132
|
+
"--isolated",
|
|
133
|
+
"--disable-pip-version-check",
|
|
134
|
+
"--no-input",
|
|
135
|
+
"--target",
|
|
136
|
+
input.root,
|
|
137
|
+
...input.args,
|
|
138
|
+
];
|
|
139
|
+
return new Promise((resolve, reject) => {
|
|
140
|
+
if (input.signal.aborted) {
|
|
141
|
+
reject(new EnvironmentError("environment_install_cancelled", "the install was cancelled before it started"));
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
const child = spawn(input.interpreter, argv, {
|
|
145
|
+
cwd: input.cwd,
|
|
146
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
147
|
+
env: isolatedPipEnv(),
|
|
148
|
+
});
|
|
149
|
+
let stderrTail = "";
|
|
150
|
+
child.stdout.setEncoding("utf8").on("data", (data: string) => input.onOutput?.("stdout", data));
|
|
151
|
+
child.stderr.setEncoding("utf8").on("data", (data: string) => {
|
|
152
|
+
stderrTail = (stderrTail + data).slice(-STDERR_TAIL_BYTES);
|
|
153
|
+
input.onOutput?.("stderr", data);
|
|
154
|
+
});
|
|
155
|
+
// pip's build backends run in their own subprocesses; stopping only pip would leave them writing.
|
|
156
|
+
const onAbort = () => {
|
|
157
|
+
const pid = child.pid;
|
|
158
|
+
if (pid === undefined) child.kill("SIGKILL");
|
|
159
|
+
else void terminateProcessTrees([pid], { graceMs: PIP_TREE_GRACE_MS }).catch(() => child.kill("SIGKILL"));
|
|
160
|
+
};
|
|
161
|
+
input.signal.addEventListener("abort", onAbort, { once: true });
|
|
162
|
+
child.once("error", (error) => {
|
|
163
|
+
input.signal.removeEventListener("abort", onAbort);
|
|
164
|
+
reject(
|
|
165
|
+
new EnvironmentError("environment_installer_unavailable", `${input.interpreter} -m pip: ${error.message}`),
|
|
166
|
+
);
|
|
167
|
+
});
|
|
168
|
+
child.once("close", (code, signal) => {
|
|
169
|
+
input.signal.removeEventListener("abort", onAbort);
|
|
170
|
+
if (input.signal.aborted) {
|
|
171
|
+
reject(new EnvironmentError("environment_install_cancelled", "the install was cancelled; pip was stopped"));
|
|
172
|
+
} else if (code === 0) resolve();
|
|
173
|
+
else if (/No module named pip/.test(stderrTail)) {
|
|
174
|
+
reject(new EnvironmentError("environment_installer_unavailable", `${input.interpreter} has no pip module`));
|
|
175
|
+
} else if (/ResolutionImpossible|conflicting dependencies/.test(stderrTail)) {
|
|
176
|
+
reject(new EnvironmentError("environment_resolution_conflict", stderrTail.trim()));
|
|
177
|
+
} else {
|
|
178
|
+
reject(
|
|
179
|
+
new EnvironmentError(
|
|
180
|
+
"environment_install_failed",
|
|
181
|
+
stderrTail.trim() || `pip exited with ${code ?? signal}`,
|
|
182
|
+
),
|
|
183
|
+
);
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
});
|
|
187
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { cp, mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { type LockWaitNotice, withRootLock } from "./install-lock.ts";
|
|
5
|
+
import { EnvironmentError } from "./py-installer.ts";
|
|
6
|
+
|
|
7
|
+
const POINTER = "active";
|
|
8
|
+
const REVISION = /^rev-(\d+)$/;
|
|
9
|
+
|
|
10
|
+
export interface Revision {
|
|
11
|
+
readonly number: number;
|
|
12
|
+
readonly dir: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export async function readActiveRevision(base: string): Promise<Revision | undefined> {
|
|
16
|
+
try {
|
|
17
|
+
const name = (await readFile(join(base, POINTER), "utf8")).trim();
|
|
18
|
+
const match = REVISION.exec(name);
|
|
19
|
+
return match?.[1] === undefined ? undefined : { number: Number(match[1]), dir: join(base, name) };
|
|
20
|
+
} catch (error) {
|
|
21
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") return undefined;
|
|
22
|
+
throw error;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Builds the next revision under the root's lock: a staging copy of the active revision (so packages
|
|
28
|
+
* accumulate) is handed to `build`, renamed to `rev-<n>` only after `build` succeeds, and published by an
|
|
29
|
+
* atomic replace of the `active` pointer. A failed, cancelled or interrupted build is deleted; the
|
|
30
|
+
* previous revision stays active and is never modified, so a running kernel never sees a partial install.
|
|
31
|
+
*/
|
|
32
|
+
export async function publishNextRevision(
|
|
33
|
+
base: string,
|
|
34
|
+
build: (staging: string, previous: Revision | undefined) => Promise<void>,
|
|
35
|
+
signal?: AbortSignal,
|
|
36
|
+
onLockWait?: (notice: LockWaitNotice) => void,
|
|
37
|
+
): Promise<{ readonly revision: Revision; readonly previous: Revision | undefined }> {
|
|
38
|
+
await mkdir(base, { recursive: true });
|
|
39
|
+
return withRootLock(
|
|
40
|
+
base,
|
|
41
|
+
async () => {
|
|
42
|
+
const previous = await readActiveRevision(base);
|
|
43
|
+
const number = Math.max(previous?.number ?? 0, await highestRevision(base)) + 1;
|
|
44
|
+
const staging = join(base, `.staging-rev-${number}-${process.pid}-${randomUUID()}`);
|
|
45
|
+
const dir = join(base, `rev-${number}`);
|
|
46
|
+
try {
|
|
47
|
+
if (previous === undefined) await mkdir(staging, { recursive: true });
|
|
48
|
+
else await cp(previous.dir, staging, { recursive: true, verbatimSymlinks: true });
|
|
49
|
+
await build(staging, previous);
|
|
50
|
+
signal?.throwIfAborted();
|
|
51
|
+
await renameRetrying(staging, dir);
|
|
52
|
+
const pointer = join(base, `.${POINTER}-${process.pid}-${randomUUID()}`);
|
|
53
|
+
await writeFile(pointer, `rev-${number}\n`);
|
|
54
|
+
await renameRetrying(pointer, join(base, POINTER));
|
|
55
|
+
} catch (error) {
|
|
56
|
+
await rm(staging, { recursive: true, force: true });
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
return { revision: { number, dir }, previous };
|
|
60
|
+
},
|
|
61
|
+
signal,
|
|
62
|
+
onLockWait,
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const RENAME_RETRY_DELAYS_MS = [20, 50, 100, 200, 400, 800];
|
|
67
|
+
|
|
68
|
+
// On Windows a rename fails with EPERM/EBUSY/EACCES while another process (an indexer, antivirus) holds a
|
|
69
|
+
// handle on the source or the replaced file; those holds are brief, so the rename is retried before failing.
|
|
70
|
+
async function renameRetrying(from: string, to: string): Promise<void> {
|
|
71
|
+
for (let attempt = 0; ; attempt++) {
|
|
72
|
+
try {
|
|
73
|
+
await rename(from, to);
|
|
74
|
+
return;
|
|
75
|
+
} catch (error) {
|
|
76
|
+
const code = error instanceof Error && "code" in error ? error.code : undefined;
|
|
77
|
+
const transient = code === "EPERM" || code === "EBUSY" || code === "EACCES";
|
|
78
|
+
const delay = RENAME_RETRY_DELAYS_MS[attempt];
|
|
79
|
+
if (!transient || delay === undefined) {
|
|
80
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
81
|
+
throw new EnvironmentError("environment_install_failed", `could not publish the new revision: ${reason}`);
|
|
82
|
+
}
|
|
83
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async function highestRevision(base: string): Promise<number> {
|
|
89
|
+
let highest = 0;
|
|
90
|
+
for (const entry of await readdir(base)) {
|
|
91
|
+
const match = REVISION.exec(entry);
|
|
92
|
+
if (match?.[1] !== undefined) highest = Math.max(highest, Number(match[1]));
|
|
93
|
+
}
|
|
94
|
+
return highest;
|
|
95
|
+
}
|
|
@@ -4,6 +4,7 @@ import type { BridgeConnectionConfig, KernelToHostMessage } from "../bridge/prot
|
|
|
4
4
|
import { JuliaKernel } from "../kernels/jl/kernel.ts";
|
|
5
5
|
import type { JavaScriptKernel } from "../kernels/js/context-manager.ts";
|
|
6
6
|
import { PythonKernel } from "../kernels/py/kernel.ts";
|
|
7
|
+
import type { PeerKernelToolsDescribe } from "../kernels/py/kernel-tools-host.ts";
|
|
7
8
|
import { defaultSpawn } from "../kernels/py/process.ts";
|
|
8
9
|
import { RubyKernel } from "../kernels/rb/kernel.ts";
|
|
9
10
|
import type { SessionEnvironment } from "../kernels/session-env.ts";
|
|
@@ -27,6 +28,7 @@ export interface SubprocessKernelStart {
|
|
|
27
28
|
readonly connection: BridgeConnectionConfig;
|
|
28
29
|
readonly onMessage: (message: KernelToHostMessage) => void;
|
|
29
30
|
};
|
|
31
|
+
readonly peerKernelToolsDescribe?: PeerKernelToolsDescribe;
|
|
30
32
|
}
|
|
31
33
|
|
|
32
34
|
export function registerKernel(sessionId: string, language: EvalLanguage, memory: RegisteredKernelSource): string {
|
|
@@ -82,6 +84,9 @@ export async function startSubprocessKernel(start: SubprocessKernelStart): Promi
|
|
|
82
84
|
...shared,
|
|
83
85
|
interpreterPath: start.interpreterPath,
|
|
84
86
|
memory,
|
|
87
|
+
...(start.peerKernelToolsDescribe === undefined
|
|
88
|
+
? {}
|
|
89
|
+
: { peerKernelToolsDescribe: start.peerKernelToolsDescribe }),
|
|
85
90
|
spawnProcess: (options) => {
|
|
86
91
|
const child = defaultSpawn(options);
|
|
87
92
|
pid = child.pid;
|
|
@@ -6,6 +6,7 @@ import type { CompletionRequest, CompletionResult } from "../completion/handler.
|
|
|
6
6
|
import { resolveKernelMemoryThresholds } from "../config/memory-settings.ts";
|
|
7
7
|
import { collectOrphanedChildren } from "../host-sdk.ts";
|
|
8
8
|
import { JavaScriptKernel } from "../kernels/js/context-manager.ts";
|
|
9
|
+
import type { KernelToolsDescribeResult } from "../kernels/js/kernel-tools-types.ts";
|
|
9
10
|
import type { KernelLifecycle } from "../kernels/shared/kernel-death.ts";
|
|
10
11
|
import type { EvalKernel, EvalLanguage } from "../tool/types.ts";
|
|
11
12
|
import { type BridgeDispatchContext, dispatchBridgeCompletion } from "./bridge-dispatch.ts";
|
|
@@ -276,7 +277,20 @@ class DefaultCodemodeSessionManager implements CodemodeSessionManager {
|
|
|
276
277
|
...lifecycle,
|
|
277
278
|
};
|
|
278
279
|
const memory = resolveKernelMemoryThresholds(this.#options.settings.memory);
|
|
279
|
-
return await startSubprocessKernel({
|
|
280
|
+
return await startSubprocessKernel({
|
|
281
|
+
language,
|
|
282
|
+
interpreterPath: detected.path,
|
|
283
|
+
memory,
|
|
284
|
+
shared,
|
|
285
|
+
peerKernelToolsDescribe: (names) => this.#javaScriptKernelToolsDescribe(names),
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** Only an already running JavaScript kernel can define a colliding name; none is started for this. */
|
|
290
|
+
#javaScriptKernelToolsDescribe(names: readonly string[]): Promise<KernelToolsDescribeResult> | undefined {
|
|
291
|
+
const js = this.#kernels.get("js");
|
|
292
|
+
if (!js || !("describeKernelTools" in js) || typeof js.describeKernelTools !== "function") return undefined;
|
|
293
|
+
return (js.describeKernelTools as (names: readonly string[]) => Promise<KernelToolsDescribeResult>)(names);
|
|
280
294
|
}
|
|
281
295
|
|
|
282
296
|
#foreignKernelToolNames(): string[] {
|
|
@@ -8,6 +8,8 @@ export const KERNEL_TOOL_ERROR_CODES = [
|
|
|
8
8
|
"kernel_tool_failed",
|
|
9
9
|
"kernel_tool_recursion",
|
|
10
10
|
"kernel_tool_host_denied",
|
|
11
|
+
"kernel_tool_cancelled",
|
|
12
|
+
"kernel_tool_loop_mismatch",
|
|
11
13
|
] as const;
|
|
12
14
|
|
|
13
15
|
export type KernelToolErrorCode = (typeof KERNEL_TOOL_ERROR_CODES)[number];
|
|
@@ -3,7 +3,8 @@ import type { BridgeConnectionConfig, KernelToHostMessage } from "../../bridge/p
|
|
|
3
3
|
import type { EvalKernelRunInput } from "../../tool/types.ts";
|
|
4
4
|
import type { SessionEnvironment } from "../session-env.ts";
|
|
5
5
|
import type { KernelLifecycle } from "../shared/kernel-death.ts";
|
|
6
|
-
import type {
|
|
6
|
+
import type { PeerKernelToolsDescribe } from "./kernel-tools-host.ts";
|
|
7
|
+
import type { KernelSpawnProcess, KillProcessGroup } from "./process.ts";
|
|
7
8
|
import type { PythonStartupStage } from "./startup.ts";
|
|
8
9
|
import type { PythonTransportResult } from "./transport.ts";
|
|
9
10
|
|
|
@@ -15,14 +16,22 @@ export interface PythonKernelStartOptions extends KernelLifecycle {
|
|
|
15
16
|
readonly env?: NodeJS.ProcessEnv;
|
|
16
17
|
/** Per-session PI_* values merged into the interpreter environment at spawn. */
|
|
17
18
|
readonly sessionEnv?: SessionEnvironment;
|
|
18
|
-
/**
|
|
19
|
+
/** How long a starting interpreter may show no stage change, output or CPU use before startup fails. */
|
|
19
20
|
readonly startupTimeoutMs?: number;
|
|
21
|
+
/** Total startup backstop for an interpreter that stays busy but never becomes ready. */
|
|
22
|
+
readonly startupCeilingMs?: number;
|
|
23
|
+
/** Reads the interpreter's CPU time; the default reads the process group (Unix) or the process (Windows). */
|
|
24
|
+
readonly readCpuTime?: (pid: number | undefined) => bigint | undefined;
|
|
25
|
+
/** Signals the interpreter's process group when it is retired; the default is `process.kill(-pid, signal)`. */
|
|
26
|
+
readonly killProcessGroup?: KillProcessGroup;
|
|
20
27
|
/** Observes bootstrap control events without mixing them into cell output callbacks. */
|
|
21
28
|
readonly onStartupProgress?: (stage: PythonStartupStage) => void;
|
|
22
29
|
readonly onMessage?: (message: KernelToHostMessage) => void;
|
|
23
30
|
readonly spawnProcess?: KernelSpawnProcess;
|
|
24
31
|
/** Post-cell collection, notice, and ceiling thresholds sent on `init`; absent leaves memory unmanaged. */
|
|
25
32
|
readonly memory?: KernelMemoryThresholds;
|
|
33
|
+
/** The JavaScript kernel's describe, so a name defined in both languages is a describe-time collision. */
|
|
34
|
+
readonly peerKernelToolsDescribe?: PeerKernelToolsDescribe;
|
|
26
35
|
}
|
|
27
36
|
|
|
28
37
|
export type PythonKernelRunOptions = EvalKernelRunInput;
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import type { HostToKernelMessage, KernelToHostMessage } from "../../bridge/protocol.ts";
|
|
2
|
+
import { kernelToolError } from "../js/kernel-tools-errors.ts";
|
|
3
|
+
import type {
|
|
4
|
+
KernelToolsDescribeResult,
|
|
5
|
+
KernelToolsInvokeOptions,
|
|
6
|
+
KernelToolsInvokeRequest,
|
|
7
|
+
} from "../js/kernel-tools-types.ts";
|
|
8
|
+
import { KernelToolHostPump } from "../shared/kernel-tools-pump.ts";
|
|
9
|
+
|
|
10
|
+
export type KernelToolReplyEvent = {
|
|
11
|
+
readonly requestId: string;
|
|
12
|
+
readonly ok: boolean;
|
|
13
|
+
readonly code?: string;
|
|
14
|
+
readonly message?: string;
|
|
15
|
+
readonly output?: string;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
function replyEvent(message: Extract<KernelToHostMessage, { type: "kernel-tool-invoke-reply" }>): KernelToolReplyEvent {
|
|
19
|
+
return {
|
|
20
|
+
requestId: message.requestId,
|
|
21
|
+
ok: message.ok,
|
|
22
|
+
...(message.ok ? {} : { message: message.error.message }),
|
|
23
|
+
...(!message.ok && message.error.code !== undefined ? { code: message.error.code } : {}),
|
|
24
|
+
...(message.output === undefined ? {} : { output: message.output }),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type PeerKernelToolsDescribe = (names: readonly string[]) => Promise<KernelToolsDescribeResult> | undefined;
|
|
29
|
+
|
|
30
|
+
/** The host half of Python kernel tools: frames over the live transport, names, and generation fencing. */
|
|
31
|
+
export class PythonKernelTools {
|
|
32
|
+
readonly #pump: KernelToolHostPump;
|
|
33
|
+
readonly #peerDescribe: PeerKernelToolsDescribe | undefined;
|
|
34
|
+
#names: readonly string[] = [];
|
|
35
|
+
|
|
36
|
+
constructor(options: {
|
|
37
|
+
readonly post: (message: HostToKernelMessage) => void;
|
|
38
|
+
readonly isOpen: () => boolean;
|
|
39
|
+
readonly peerDescribe?: PeerKernelToolsDescribe;
|
|
40
|
+
}) {
|
|
41
|
+
this.#pump = new KernelToolHostPump(options.post, options.isOpen);
|
|
42
|
+
this.#peerDescribe = options.peerDescribe;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
get events(): EventTarget {
|
|
46
|
+
return this.#pump.events;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
listNames(): readonly string[] {
|
|
50
|
+
return this.#names;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
consume(message: KernelToHostMessage): boolean {
|
|
54
|
+
if (message.type === "kernel-tools-defined") {
|
|
55
|
+
this.#names = [...message.names];
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
if (message.type === "kernel-tool-invoke-reply") {
|
|
59
|
+
// Published even when the waiter is gone (a cancelled call), so its explicit outcome stays observable.
|
|
60
|
+
this.#pump.events.dispatchEvent(
|
|
61
|
+
new CustomEvent<KernelToolReplyEvent>("kernelToolReply", { detail: replyEvent(message) }),
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
return this.#pump.consume(message);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The interpreter is gone (restart, reset, close, death): its definitions and in-flight calls are too. */
|
|
68
|
+
retire(reason: "kernel_tool_stale" | "tools_unavailable", message: string): void {
|
|
69
|
+
this.#names = [];
|
|
70
|
+
this.#pump.rejectAll(kernelToolError(reason, message));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async describe(names: readonly string[]): Promise<KernelToolsDescribeResult> {
|
|
74
|
+
const own = await this.#pump.describe(names);
|
|
75
|
+
const peer = await this.#peerDescribe?.(names)?.catch(() => undefined);
|
|
76
|
+
if (!peer) return own;
|
|
77
|
+
const definedInJs = new Set(peer.results.filter((entry) => entry.ok).map((entry) => entry.name));
|
|
78
|
+
return {
|
|
79
|
+
results: own.results.map((entry) =>
|
|
80
|
+
entry.ok && definedInJs.has(entry.name)
|
|
81
|
+
? {
|
|
82
|
+
name: entry.name,
|
|
83
|
+
ok: false,
|
|
84
|
+
error: {
|
|
85
|
+
code: "tool_name_collision",
|
|
86
|
+
message: `Kernel tool ${entry.name} is defined in both js and py`,
|
|
87
|
+
},
|
|
88
|
+
}
|
|
89
|
+
: entry,
|
|
90
|
+
),
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async invoke(request: KernelToolsInvokeRequest, options?: AbortSignal | KernelToolsInvokeOptions): Promise<unknown> {
|
|
95
|
+
return await this.#pump.invoke(request, options);
|
|
96
|
+
}
|
|
97
|
+
}
|
package/src/kernels/py/kernel.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
// allow: SIZE_OK — one persistent Python lifecycle state machine owns queue, generation, and retirement.
|
|
2
2
|
import type { KernelInterruptHandle, PendingCell } from "../../tool/types.ts";
|
|
3
|
-
import type {
|
|
4
|
-
|
|
3
|
+
import type {
|
|
4
|
+
KernelToolsDescribeResult,
|
|
5
|
+
KernelToolsInvokeOptions,
|
|
6
|
+
KernelToolsInvokeRequest,
|
|
7
|
+
} from "../js/kernel-tools-types.ts";
|
|
5
8
|
import { describeExit } from "../shared/kernel-death.ts";
|
|
6
9
|
import { KernelMemoryHost } from "../shared/kernel-memory-host.ts";
|
|
7
10
|
import { KernelPreludeTracker } from "../shared/kernel-prelude-plan.ts";
|
|
8
11
|
import { runHostCell } from "./host-cell.ts";
|
|
9
12
|
import type { PendingRun, PythonKernelRunOptions, PythonKernelStartOptions, ResultMessage } from "./kernel-contract.ts";
|
|
10
|
-
import {
|
|
13
|
+
import { PythonKernelTools } from "./kernel-tools-host.ts";
|
|
14
|
+
import { pythonStartupCeilingMs, pythonStartupHangGuardMs } from "./startup.ts";
|
|
11
15
|
import { failedPythonResult, PythonKernelTransport } from "./transport.ts";
|
|
12
16
|
|
|
13
17
|
export type { PythonKernelRunOptions, PythonKernelStartOptions } from "./kernel-contract.ts";
|
|
@@ -15,6 +19,15 @@ export type { KernelChild, KernelSpawnOptions, KernelSpawnProcess } from "./proc
|
|
|
15
19
|
|
|
16
20
|
const interruptEscalationMs = 5_000;
|
|
17
21
|
|
|
22
|
+
// Every interpreter this process starts gets its own generation, so a kernel tool descriptor taken from one
|
|
23
|
+
// never resolves in another, whether it was replaced by reset, by a lazy restart after a crash, or by a new
|
|
24
|
+
// kernel instance an owner created.
|
|
25
|
+
let lastInterpreterGeneration = 0;
|
|
26
|
+
function nextInterpreterGeneration(): number {
|
|
27
|
+
lastInterpreterGeneration += 1;
|
|
28
|
+
return lastInterpreterGeneration;
|
|
29
|
+
}
|
|
30
|
+
|
|
18
31
|
export class PythonKernel {
|
|
19
32
|
readonly #options: PythonKernelStartOptions;
|
|
20
33
|
#transport: PythonKernelTransport | null = null;
|
|
@@ -29,11 +42,17 @@ export class PythonKernel {
|
|
|
29
42
|
readonly #memory: KernelMemoryHost | null;
|
|
30
43
|
#generation = 0;
|
|
31
44
|
#closed = false;
|
|
45
|
+
readonly #tools: PythonKernelTools;
|
|
32
46
|
#dead = false;
|
|
33
47
|
|
|
34
48
|
private constructor(options: PythonKernelStartOptions) {
|
|
35
49
|
this.#options = options;
|
|
36
50
|
this.#memory = options.memory === undefined ? null : new KernelMemoryHost("py", options.memory);
|
|
51
|
+
this.#tools = new PythonKernelTools({
|
|
52
|
+
post: (message) => this.#transport?.post(message),
|
|
53
|
+
isOpen: () => this.#transport !== null && !this.#closed,
|
|
54
|
+
...(options.peerKernelToolsDescribe === undefined ? {} : { peerDescribe: options.peerKernelToolsDescribe }),
|
|
55
|
+
});
|
|
37
56
|
}
|
|
38
57
|
|
|
39
58
|
static async start(options: PythonKernelStartOptions): Promise<PythonKernel> {
|
|
@@ -43,7 +62,11 @@ export class PythonKernel {
|
|
|
43
62
|
}
|
|
44
63
|
|
|
45
64
|
listKernelToolNames(): readonly string[] {
|
|
46
|
-
return
|
|
65
|
+
return this.#tools.listNames();
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
get kernelToolEvents(): EventTarget {
|
|
69
|
+
return this.#tools.events;
|
|
47
70
|
}
|
|
48
71
|
|
|
49
72
|
isAlive(): boolean {
|
|
@@ -58,12 +81,15 @@ export class PythonKernel {
|
|
|
58
81
|
});
|
|
59
82
|
}
|
|
60
83
|
|
|
61
|
-
describeKernelTools(
|
|
62
|
-
return
|
|
84
|
+
describeKernelTools(names: readonly string[]): Promise<KernelToolsDescribeResult> {
|
|
85
|
+
return this.#tools.describe(names);
|
|
63
86
|
}
|
|
64
87
|
|
|
65
|
-
invokeKernelTool(
|
|
66
|
-
|
|
88
|
+
invokeKernelTool(
|
|
89
|
+
request: KernelToolsInvokeRequest,
|
|
90
|
+
options?: AbortSignal | KernelToolsInvokeOptions,
|
|
91
|
+
): Promise<unknown> {
|
|
92
|
+
return this.#tools.invoke(request, options);
|
|
67
93
|
}
|
|
68
94
|
|
|
69
95
|
run(input: PythonKernelRunOptions): Promise<ResultMessage> {
|
|
@@ -131,6 +157,7 @@ export class PythonKernel {
|
|
|
131
157
|
if (this.#failure) throw this.#failure;
|
|
132
158
|
if (this.#closed) throw new Error("Python kernel is closed");
|
|
133
159
|
const generation = ++this.#generation;
|
|
160
|
+
this.#tools.retire("kernel_tool_stale", "Python kernel was reset; its kernel tools are gone");
|
|
134
161
|
const prior = this.#starting;
|
|
135
162
|
const operation = (async () => {
|
|
136
163
|
await prior?.catch(() => undefined);
|
|
@@ -156,6 +183,7 @@ export class PythonKernel {
|
|
|
156
183
|
if (this.#closePromise) return await this.#closePromise;
|
|
157
184
|
this.#closed = true;
|
|
158
185
|
this.#generation += 1;
|
|
186
|
+
this.#tools.retire("tools_unavailable", "Python kernel is closed");
|
|
159
187
|
this.#settleAllPending("Python kernel closed");
|
|
160
188
|
const starting = this.#starting;
|
|
161
189
|
this.#closePromise = (async () => {
|
|
@@ -283,6 +311,7 @@ export class PythonKernel {
|
|
|
283
311
|
...this.#options,
|
|
284
312
|
onMessage: (message) => {
|
|
285
313
|
if (message.type === "result") return;
|
|
314
|
+
if (generation === this.#generation && this.#tools.consume(message)) return;
|
|
286
315
|
const callback =
|
|
287
316
|
message.type === "ready" || message.type === "init-failed" || message.type === "closed"
|
|
288
317
|
? this.#options.onMessage
|
|
@@ -290,6 +319,8 @@ export class PythonKernel {
|
|
|
290
319
|
callback?.(message);
|
|
291
320
|
},
|
|
292
321
|
startupTimeoutMs: this.#options.startupTimeoutMs ?? pythonStartupHangGuardMs,
|
|
322
|
+
startupCeilingMs: this.#options.startupCeilingMs ?? pythonStartupCeilingMs,
|
|
323
|
+
kernelGeneration: nextInterpreterGeneration(),
|
|
293
324
|
isOwned: () => !this.#closed && generation === this.#generation,
|
|
294
325
|
onRetirementFailure: (transport, error) => {
|
|
295
326
|
if (!this.#transport) this.#transport = transport;
|
|
@@ -324,6 +355,7 @@ export class PythonKernel {
|
|
|
324
355
|
#onExit(transport: PythonKernelTransport, error: Error, reason: string): void {
|
|
325
356
|
if (this.#transport !== transport) return;
|
|
326
357
|
this.#transport = null;
|
|
358
|
+
this.#tools.retire("kernel_tool_stale", "Python kernel died; its kernel tools are gone");
|
|
327
359
|
// With an owner listening, a death is final for this instance: the cells that never started stay
|
|
328
360
|
// queued for `drainPending` and the owner replaces it. Without one it respawns lazily, as before.
|
|
329
361
|
const onDeath = this.#options.onDeath;
|