@code-yeongyu/senpi-codemode 2026.10.9 → 2026.10.10-10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -0
- package/README.md +161 -11
- package/package.json +6 -5
- package/src/bridge/protocol.ts +8 -3
- package/src/bridge/reserved.ts +2 -0
- package/src/bridges/eval-environment-schemas.ts +48 -0
- package/src/bridges/eval-virtual-schemas.ts +11 -4
- package/src/bridges/reserved-dispatch.ts +13 -1
- package/src/config/executable-settings.json +5 -0
- package/src/config/project-trust.ts +51 -0
- package/src/config/settings-overrides.ts +2 -0
- package/src/environments/js-environments.ts +208 -0
- package/src/environments/js-installer.ts +268 -0
- package/src/environments/js-revision-files.ts +91 -0
- package/src/environments/no-links.ts +132 -0
- package/src/environments/packages-install.ts +98 -0
- package/src/environments/project-resolves.ts +54 -0
- package/src/environments/py-environment.ts +7 -3
- package/src/environments/py-installer.ts +5 -1
- package/src/environments/python-environments.ts +1 -1
- package/src/environments/revision-store.ts +20 -4
- package/src/extension/bridge-tool-call.ts +5 -1
- package/src/extension/eval-notifier.ts +13 -2
- package/src/extension/kernel-registration.ts +7 -2
- package/src/extension/runtime-factory.ts +51 -13
- package/src/extension/runtime-info.ts +6 -2
- package/src/extension/session-manager-contract.ts +3 -0
- package/src/extension/session-manager.ts +16 -2
- package/src/index.ts +3 -2
- package/src/interpreters/detect.ts +34 -1
- package/src/kernels/AGENTS.md +2 -0
- package/src/kernels/js/cell-dispatch.ts +44 -0
- package/src/kernels/js/cell-ownership.js +268 -0
- package/src/kernels/js/cell-run-context.js +89 -0
- package/src/kernels/js/context-manager.ts +46 -51
- package/src/kernels/js/control-frames.ts +24 -0
- package/src/kernels/js/host-entries.ts +62 -0
- package/src/kernels/js/kernel-contract.ts +16 -1
- package/src/kernels/js/kernel-memory-bridge.ts +1 -1
- package/src/kernels/js/kernel-memory.ts +79 -0
- package/src/kernels/js/kernel-tools-registry.js +1 -0
- package/src/kernels/js/local-module-loader.ts +17 -6
- package/src/kernels/js/process-drain-scan.d.ts +8 -0
- package/src/kernels/js/process-drain-scan.js +28 -0
- package/src/kernels/js/process-entry.js +466 -0
- package/src/kernels/js/process-memory-host.ts +83 -0
- package/src/kernels/js/process-worker.ts +298 -0
- package/src/kernels/js/rejection-reports.js +61 -0
- package/src/kernels/js/run-queue.ts +7 -0
- package/src/kernels/js/worker-core.js +53 -8
- package/src/kernels/js/worker-cwd.js +5 -2
- package/src/kernels/js/worker-host.ts +9 -2
- package/src/kernels/js/worker-indirect-eval.js +37 -13
- package/src/kernels/js/worker-package-resolve.js +153 -0
- package/src/kernels/js/worker-require.d.ts +19 -0
- package/src/kernels/js/worker-require.js +52 -0
- package/src/kernels/js/worker-runtime.js +43 -9
- package/src/kernels/js/worker-shadow-guard.js +2 -6
- package/src/kernels/js/worker-shell-capture.js +5 -0
- package/src/kernels/js/worker-slot.ts +9 -0
- package/src/kernels/js/worker-startup.ts +49 -1
- package/src/kernels/js/worker-webview.js +15 -1
- package/src/kernels/py/host-cell.ts +8 -31
- package/src/kernels/py/prelude.py +33 -2
- package/src/kernels/py/process.ts +4 -0
- package/src/kernels/sandbox/sandbox-cell.ts +141 -0
- package/src/kernels/sandbox/vendor/pi-codemode/VENDORED.md +1 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/prelude-source.ts +14 -1
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/worker.ts +4 -1
- package/src/kernels/sandbox/vendor/pi-codemode/types.ts +2 -0
- package/src/kernels/shared/host-cell.ts +57 -0
- package/src/kernels/shared/subprocess-process.ts +14 -0
- package/src/prompt/eval-prompt.ts +8 -1
- package/src/tool/AGENTS.md +1 -0
- package/src/tool/cell-handler.ts +12 -11
- package/src/tool/cell-reserved-dispatch.ts +1 -0
- package/src/tool/cell-runtime.ts +4 -2
- package/src/tool/detached-cell-contract.ts +3 -0
- package/src/tool/detached-cell-manager.ts +4 -8
- package/src/tool/detached-cell-notification.ts +11 -63
- package/src/tool/detached-cell-snapshot.ts +17 -0
- package/src/tool/detached-eval-result.ts +13 -7
- package/src/tool/detached-notification-queue.ts +1 -2
- package/src/tool/eval-request.ts +23 -1
- package/src/tool/eval-tool-options.ts +3 -0
- package/src/tool/eval-tool.ts +17 -4
- package/src/tool/interrupt-note.ts +4 -5
- package/src/tool/live-headline.ts +60 -0
- package/src/tool/load-cell.ts +18 -3
- package/src/tool/magic-cell-host.ts +60 -2
- package/src/tool/magic-cells.ts +12 -2
- package/src/tool/managed-cell.ts +0 -4
- package/src/tool/render.ts +74 -15
- package/src/tool/run-eval-cell.ts +46 -4
- package/src/tool/runtime-label.ts +2 -1
- package/src/tool/types.ts +23 -1
|
@@ -0,0 +1,466 @@
|
|
|
1
|
+
// JavaScript kernel process-mode entry: worker-core over the framed subprocess transport.
|
|
2
|
+
//
|
|
3
|
+
// Trust model: process mode isolates CRASHES (memory, segfaults, out-of-memory), not hostile code. A cell runs in this
|
|
4
|
+
// process and can reach everything it holds, exactly like worker mode; hostile code belongs in isolate: true sandbox
|
|
5
|
+
// cells. The frame token below guards against ACCIDENTAL corruption of the channel (a cell's raw fd writes, a child
|
|
6
|
+
// process's output), not against a cell that sets out to forge frames.
|
|
7
|
+
//
|
|
8
|
+
// Channel: the host writes a random token as the first line of fd 0 and frames after it. Where the runtime allows it
|
|
9
|
+
// (Bun on macOS/Linux) the frame reader moves to a private duplicate of fd 0 and fd 0 becomes /dev/null, and fd 1 is
|
|
10
|
+
// re-pointed at a blocking pipe whose bytes a reader thread writes out as text frames; frames go out on a duplicate
|
|
11
|
+
// of the original fd 1 taken before the re-point, under one lock shared by both threads. Every frame is written as
|
|
12
|
+
// "<token> <json>", and the host parses only lines that carry it.
|
|
13
|
+
//
|
|
14
|
+
// Lifetime: the kernel exits the moment its control channel reaches end-of-file (the host closed it, exited or was
|
|
15
|
+
// killed). A watchdog thread also checks the parent pid and kills this process when the host is gone, so a cell
|
|
16
|
+
// that never yields (a busy loop, a blocking call) cannot keep the kernel alive past its host.
|
|
17
|
+
import { closeSync, createReadStream, openSync, writeSync } from "node:fs";
|
|
18
|
+
import { createRequire } from "node:module";
|
|
19
|
+
import { createInterface } from "node:readline";
|
|
20
|
+
import { Readable } from "node:stream";
|
|
21
|
+
import { Worker } from "node:worker_threads";
|
|
22
|
+
import { scanDrainText } from "./process-drain-scan.js";
|
|
23
|
+
|
|
24
|
+
// Captured before any cell runs, so a cell that later replaces these never sees a frame being built.
|
|
25
|
+
const stringify = JSON.stringify;
|
|
26
|
+
const writeBytes = writeSync;
|
|
27
|
+
const exit = process.exit.bind(process);
|
|
28
|
+
const fromString = (text) => Buffer.from(text, "utf8");
|
|
29
|
+
|
|
30
|
+
// The host refuses a frame over 10 MiB; text is split well below it (a char can escape to 6 bytes of JSON).
|
|
31
|
+
const TEXT_CHUNK_CHARS = 1 << 20;
|
|
32
|
+
const FRAME_LIMIT_BYTES = 9 * 1024 * 1024;
|
|
33
|
+
// Values JSON cannot carry are sent as markers the host turns back into the value, matching worker mode. Each marker
|
|
34
|
+
// carries the frame token, so a cell's own data can never be taken for one.
|
|
35
|
+
const BIGINT_MARKER = "\u0000senpi:bigint:";
|
|
36
|
+
const UNDEFINED_MARKER = "\u0000senpi:undefined:";
|
|
37
|
+
const DRAIN_MARKER = "\u0000senpi-drain:";
|
|
38
|
+
|
|
39
|
+
// Set when libc could not be loaded; shown once, as stderr output of the first cell, so the user sees why.
|
|
40
|
+
let libcUnavailableNotice;
|
|
41
|
+
|
|
42
|
+
function libcSymbols() {
|
|
43
|
+
if (process.platform === "win32") return null;
|
|
44
|
+
const libcPath = { darwin: "/usr/lib/libSystem.B.dylib", linux: "libc.so.6" }[process.platform];
|
|
45
|
+
if (libcPath === undefined) return null;
|
|
46
|
+
let ffi;
|
|
47
|
+
try {
|
|
48
|
+
ffi = createRequire(import.meta.url)("bun:ffi");
|
|
49
|
+
} catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
try {
|
|
53
|
+
return loadLibcSymbols(ffi, libcPath);
|
|
54
|
+
} catch (error) {
|
|
55
|
+
// dlopen can fail where libc is not where this expects it (a minimal image, an unusual libc). The kernel then
|
|
56
|
+
// runs as it does on Windows, framing on fd 1 without the raw-output pipe, instead of dying before it can say why.
|
|
57
|
+
libcUnavailableNotice = `[senpi-codemode] process kernel: libc could not be loaded (${error instanceof Error ? error.message : String(error)}); raw fd 1 output from child processes is not captured\n`;
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function loadLibcSymbols(ffi, libcPath) {
|
|
63
|
+
const symbols = ffi.dlopen(libcPath, {
|
|
64
|
+
dup: { args: [ffi.FFIType.i32], returns: ffi.FFIType.i32 },
|
|
65
|
+
dup2: { args: [ffi.FFIType.i32, ffi.FFIType.i32], returns: ffi.FFIType.i32 },
|
|
66
|
+
fcntl: { args: [ffi.FFIType.i32, ffi.FFIType.i32, ffi.FFIType.i32], returns: ffi.FFIType.i32 },
|
|
67
|
+
pipe: { args: [ffi.FFIType.ptr], returns: ffi.FFIType.i32 },
|
|
68
|
+
...(process.platform === "linux"
|
|
69
|
+
? {
|
|
70
|
+
prctl: {
|
|
71
|
+
args: [ffi.FFIType.i32, ffi.FFIType.u64, ffi.FFIType.u64, ffi.FFIType.u64, ffi.FFIType.u64],
|
|
72
|
+
returns: ffi.FFIType.i32,
|
|
73
|
+
},
|
|
74
|
+
}
|
|
75
|
+
: {}),
|
|
76
|
+
}).symbols;
|
|
77
|
+
if (!(process.platform === "darwin" && process.arch === "arm64")) return symbols;
|
|
78
|
+
// fcntl is variadic. bun:ffi makes only non-variadic calls, and on Apple arm64 a variadic argument is read from the
|
|
79
|
+
// stack, not a register: declared with six padding arguments, the value is the ninth argument, which is the first
|
|
80
|
+
// stack slot. Elsewhere the three-argument form passes it where fcntl reads it.
|
|
81
|
+
const stackArgs = ffi.dlopen(libcPath, {
|
|
82
|
+
fcntl: {
|
|
83
|
+
args: [
|
|
84
|
+
ffi.FFIType.i32,
|
|
85
|
+
ffi.FFIType.i32,
|
|
86
|
+
ffi.FFIType.i64,
|
|
87
|
+
ffi.FFIType.i64,
|
|
88
|
+
ffi.FFIType.i64,
|
|
89
|
+
ffi.FFIType.i64,
|
|
90
|
+
ffi.FFIType.i64,
|
|
91
|
+
ffi.FFIType.i64,
|
|
92
|
+
ffi.FFIType.i64,
|
|
93
|
+
],
|
|
94
|
+
returns: ffi.FFIType.i32,
|
|
95
|
+
},
|
|
96
|
+
}).symbols;
|
|
97
|
+
return { ...symbols, fcntlStack: stackArgs.fcntl };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const libc = libcSymbols();
|
|
101
|
+
|
|
102
|
+
// On Linux a crash is reported through the kernel's core-dump handler first; where that is a pipe to
|
|
103
|
+
// systemd-coredump or apport, the dying child is held until the dump is consumed, which can freeze its cell for
|
|
104
|
+
// about 30 s. Marking this process non-dumpable skips the dump, so the crash (signal included) is reported at once.
|
|
105
|
+
// exec resets the flag, so a cell's own subprocesses dump as usual. SENPI_KERNEL_CORE_DUMPS=1 keeps dumps.
|
|
106
|
+
const PR_SET_DUMPABLE = 4;
|
|
107
|
+
if (libc?.prctl !== undefined && process.env.SENPI_KERNEL_CORE_DUMPS !== "1") libc.prctl(PR_SET_DUMPABLE, 0, 0, 0, 0);
|
|
108
|
+
|
|
109
|
+
/** The control channel's fd: a private duplicate of fd 0 where possible, so a cell reading fd 0 reads /dev/null. */
|
|
110
|
+
function privateControlFd() {
|
|
111
|
+
if (libc === null) return 0;
|
|
112
|
+
const duplicate = libc.dup(0);
|
|
113
|
+
if (duplicate === -1) return 0;
|
|
114
|
+
const devNull = openSync("/dev/null", "r");
|
|
115
|
+
// dup2 returns the new descriptor (0) on success and -1 on failure.
|
|
116
|
+
if (libc.dup2(devNull, 0) === -1) {
|
|
117
|
+
closeSync(devNull);
|
|
118
|
+
return duplicate;
|
|
119
|
+
}
|
|
120
|
+
closeSync(devNull);
|
|
121
|
+
return duplicate;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const F_GETFL = 3;
|
|
125
|
+
const F_SETFL = 4;
|
|
126
|
+
const O_NONBLOCK = process.platform === "darwin" ? 0x4 : 0x800;
|
|
127
|
+
const variadicOnStack = process.platform === "darwin" && process.arch === "arm64";
|
|
128
|
+
|
|
129
|
+
function setFileStatusFlags(fd, flags) {
|
|
130
|
+
if (libc === null) return;
|
|
131
|
+
if (variadicOnStack) libc.fcntlStack(fd, F_SETFL, 0, 0, 0, 0, 0, 0, flags);
|
|
132
|
+
else libc.fcntl(fd, F_SETFL, flags);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Keeps fd 1 blocking. O_NONBLOCK belongs to the pipe's open file description, which every child inheriting fd 1
|
|
137
|
+
* shares: bun sets it the first time a process touches process.stdout and never clears it, and a SIGKILLed node child
|
|
138
|
+
* cannot restore it. A writer that does not retry EAGAIN (`cat`, writeSync) would then lose output whenever it outran
|
|
139
|
+
* the reader thread, so this runs before every cell and before and after every child process a cell starts.
|
|
140
|
+
*/
|
|
141
|
+
function ensureBlocking() {
|
|
142
|
+
if (libc === null || textPipe === null) return;
|
|
143
|
+
const flags = libc.fcntl(1, F_GETFL, 0);
|
|
144
|
+
if (flags !== -1 && (flags & O_NONBLOCK) !== 0) setFileStatusFlags(1, flags & ~O_NONBLOCK);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Points fd 1 at a pipe this process reads, so raw fd 1 writes become text frames; null when unavailable. */
|
|
148
|
+
function repointStdoutToPipe() {
|
|
149
|
+
if (libc === null) return null;
|
|
150
|
+
const fds = new Int32Array(2);
|
|
151
|
+
if (libc.pipe(fds) !== 0) return null;
|
|
152
|
+
// dup2 returns the new descriptor (1) on success and -1 on failure.
|
|
153
|
+
if (libc.dup2(fds[1], 1) === -1) return null;
|
|
154
|
+
// Bun makes fd 1 non-blocking when process.stdout is first touched: set it up now, then keep it blocking.
|
|
155
|
+
void process.stdout;
|
|
156
|
+
return { readFd: fds[0] };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const controlFd = privateControlFd();
|
|
160
|
+
// Frames go out on a duplicate of the original fd 1, taken before the re-point. A duplicate rather than a reopen of
|
|
161
|
+
// /dev/fd/1: on Linux the spawn's stdout is a socket, which cannot be opened by path.
|
|
162
|
+
const duplicatedFd = libc === null ? -1 : libc.dup(1);
|
|
163
|
+
const frameFd = duplicatedFd === -1 ? 1 : duplicatedFd;
|
|
164
|
+
const textPipe = duplicatedFd === -1 ? null : repointStdoutToPipe();
|
|
165
|
+
ensureBlocking();
|
|
166
|
+
keepChildProcessesBlocking();
|
|
167
|
+
|
|
168
|
+
/** Wraps the ways a cell starts a child process, so fd 1 is blocking again before each starts and after it ends. */
|
|
169
|
+
function keepChildProcessesBlocking() {
|
|
170
|
+
if (textPipe === null) return;
|
|
171
|
+
const childProcess = createRequire(import.meta.url)("node:child_process");
|
|
172
|
+
for (const name of ["execSync", "execFileSync", "spawnSync"]) {
|
|
173
|
+
const original = childProcess[name];
|
|
174
|
+
childProcess[name] = function (...args) {
|
|
175
|
+
ensureBlocking();
|
|
176
|
+
try {
|
|
177
|
+
return original.apply(this, args);
|
|
178
|
+
} finally {
|
|
179
|
+
ensureBlocking();
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
for (const name of ["spawn", "exec", "execFile", "fork"]) {
|
|
184
|
+
const original = childProcess[name];
|
|
185
|
+
childProcess[name] = function (...args) {
|
|
186
|
+
ensureBlocking();
|
|
187
|
+
const child = original.apply(this, args);
|
|
188
|
+
child.once("exit", ensureBlocking);
|
|
189
|
+
return child;
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
if (globalThis.Bun === undefined) return;
|
|
193
|
+
const bun = globalThis.Bun;
|
|
194
|
+
const spawnSync = bun.spawnSync;
|
|
195
|
+
const spawn = bun.spawn;
|
|
196
|
+
// Bun.spawn and Bun.spawnSync are writable but not configurable, so they are replaced by assignment (as the
|
|
197
|
+
// per-cell shell capture also does); a redefinition would throw.
|
|
198
|
+
bun.spawnSync = (...args) => {
|
|
199
|
+
ensureBlocking();
|
|
200
|
+
try {
|
|
201
|
+
return spawnSync.apply(bun, args);
|
|
202
|
+
} finally {
|
|
203
|
+
ensureBlocking();
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
bun.spawn = (...args) => {
|
|
207
|
+
ensureBlocking();
|
|
208
|
+
const child = spawn.apply(bun, args);
|
|
209
|
+
child.exited.then(ensureBlocking, ensureBlocking);
|
|
210
|
+
return child;
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
const pause = new Int32Array(new SharedArrayBuffer(4));
|
|
214
|
+
// Frames are written by this thread and by the fd 1 reader thread; one lock keeps every frame line whole.
|
|
215
|
+
const frameLock = new Int32Array(new SharedArrayBuffer(4));
|
|
216
|
+
|
|
217
|
+
/** Writes all of `bytes`, waiting out a full pipe; a closed channel means the host is gone, so the kernel exits. */
|
|
218
|
+
function writeAll(fd, bytes) {
|
|
219
|
+
let offset = 0;
|
|
220
|
+
while (offset < bytes.length) {
|
|
221
|
+
try {
|
|
222
|
+
offset += writeBytes(fd, bytes, offset, bytes.length - offset);
|
|
223
|
+
} catch (error) {
|
|
224
|
+
if (error?.code === "EAGAIN") {
|
|
225
|
+
Atomics.wait(pause, 0, 0, 2);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
exit(0);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Without the re-point, cell output shares the channel: a leading newline ends any unterminated output line, so the
|
|
234
|
+
// frame always starts its own line.
|
|
235
|
+
const framePrefix = textPipe === null ? "\n" : "";
|
|
236
|
+
let frameToken;
|
|
237
|
+
|
|
238
|
+
function replacer(_key, value) {
|
|
239
|
+
if (typeof value === "bigint") return { [`${BIGINT_MARKER}${frameToken}`]: value.toString() };
|
|
240
|
+
if (value === undefined) return { [`${UNDEFINED_MARKER}${frameToken}`]: 1 };
|
|
241
|
+
return value;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function writeFrame(message) {
|
|
245
|
+
const bytes = fromString(stringify(message, replacer));
|
|
246
|
+
if (bytes.length <= FRAME_LIMIT_BYTES) {
|
|
247
|
+
const line = Buffer.concat([fromString(`${framePrefix}${frameToken} `), bytes, fromString("\n")]);
|
|
248
|
+
while (Atomics.compareExchange(frameLock, 0, 0, 1) !== 0) Atomics.wait(frameLock, 0, 1, 5);
|
|
249
|
+
try {
|
|
250
|
+
writeAll(frameFd, line);
|
|
251
|
+
} finally {
|
|
252
|
+
Atomics.store(frameLock, 0, 0);
|
|
253
|
+
Atomics.notify(frameLock, 0, 1);
|
|
254
|
+
}
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
const tooLarge = (what) => ({ name: "RangeError", message: `${what} is too large to send (${bytes.length} bytes)` });
|
|
258
|
+
if (message.type === "result") {
|
|
259
|
+
const error = tooLarge("cell result");
|
|
260
|
+
writeFrame({ type: "result", cellId: message.cellId, ok: false, error, durationMs: message.durationMs ?? 0 });
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
if (message.type === "tool-call") {
|
|
264
|
+
// The host never sees the call, so the cell gets the refusal as the call's own failure instead of waiting on it.
|
|
265
|
+
const reply = { type: "tool-reply", callId: message.callId, ok: false, error: tooLarge(`the ${message.toolName} call`) };
|
|
266
|
+
deliver(reply);
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const data = `[a ${message.type} message of ${bytes.length} bytes was too large to send and was dropped]\n`;
|
|
270
|
+
writeFrame({ type: "text", stream: "stderr", data });
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function sendFrame(message) {
|
|
274
|
+
if (message.type !== "text" || typeof message.data !== "string" || message.data.length <= TEXT_CHUNK_CHARS) {
|
|
275
|
+
writeFrame(message);
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
278
|
+
for (let start = 0; start < message.data.length; ) {
|
|
279
|
+
let end = Math.min(start + TEXT_CHUNK_CHARS, message.data.length);
|
|
280
|
+
// Never split a surrogate pair.
|
|
281
|
+
if (end < message.data.length && /[\uD800-\uDBFF]/.test(message.data[end - 1])) end -= 1;
|
|
282
|
+
writeFrame({ ...message, data: message.data.slice(start, end) });
|
|
283
|
+
start = end;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// A cell that reads stdin sees an already-ended stream instead of the control channel.
|
|
288
|
+
const ended = new Readable({
|
|
289
|
+
read() {
|
|
290
|
+
this.push(null);
|
|
291
|
+
},
|
|
292
|
+
});
|
|
293
|
+
Object.defineProperty(process, "stdin", { value: ended, configurable: true });
|
|
294
|
+
if (globalThis.Bun !== undefined) {
|
|
295
|
+
try {
|
|
296
|
+
Object.defineProperty(globalThis.Bun, "stdin", { get: () => ended, configurable: true });
|
|
297
|
+
} catch {
|
|
298
|
+
// Bun.stdin is already non-configurable on this runtime
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const frameReader = createReadStream("", { fd: controlFd });
|
|
303
|
+
const frameLines = createInterface({ input: frameReader });
|
|
304
|
+
// End-of-file on the control channel means the host is gone (closed, exited or killed): never outlive it.
|
|
305
|
+
frameReader.once("end", () => exit(0));
|
|
306
|
+
frameReader.once("close", () => exit(0));
|
|
307
|
+
// The same, from a thread a running cell cannot block: once the parent pid changes the host is gone.
|
|
308
|
+
const WATCHDOG = `const { workerData } = require("node:worker_threads");
|
|
309
|
+
setInterval(() => {
|
|
310
|
+
if (process.ppid !== workerData.parent) process.kill(process.pid, "SIGKILL");
|
|
311
|
+
}, 100);`;
|
|
312
|
+
new Worker(WATCHDOG, { eval: true, workerData: { parent: process.ppid } }).unref();
|
|
313
|
+
|
|
314
|
+
let lineHandler;
|
|
315
|
+
const bufferedLines = [];
|
|
316
|
+
function deliver(message) {
|
|
317
|
+
if (message.type === "run") ensureBlocking();
|
|
318
|
+
if (message.type === "run" && libcUnavailableNotice !== undefined) {
|
|
319
|
+
sendFrame({ type: "text", stream: "stderr", data: libcUnavailableNotice });
|
|
320
|
+
libcUnavailableNotice = undefined;
|
|
321
|
+
}
|
|
322
|
+
if (lineHandler === undefined) bufferedLines.push(message);
|
|
323
|
+
else lineHandler(message);
|
|
324
|
+
}
|
|
325
|
+
const tokenReceived = new Promise((resolve) => {
|
|
326
|
+
frameLines.on("line", (line) => {
|
|
327
|
+
if (frameToken === undefined) {
|
|
328
|
+
frameToken = line;
|
|
329
|
+
resolve();
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
if (line.length === 0) return;
|
|
333
|
+
deliver(JSON.parse(line));
|
|
334
|
+
});
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
const transport = {
|
|
338
|
+
send(message) {
|
|
339
|
+
if (message.type === "result" && textPipe !== null) {
|
|
340
|
+
drainSequence += 1;
|
|
341
|
+
const key = String(drainSequence);
|
|
342
|
+
pendingResults.set(key, message);
|
|
343
|
+
writeAll(1, fromString(`${DRAIN_MARKER}${key}\u0000`));
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
sendFrame(message);
|
|
347
|
+
},
|
|
348
|
+
onMessage(handler) {
|
|
349
|
+
lineHandler = handler;
|
|
350
|
+
for (const message of bufferedLines.splice(0)) handler(message);
|
|
351
|
+
return () => frameLines.close();
|
|
352
|
+
},
|
|
353
|
+
close() {
|
|
354
|
+
setTimeout(() => exit(0), 0);
|
|
355
|
+
},
|
|
356
|
+
};
|
|
357
|
+
|
|
358
|
+
await tokenReceived;
|
|
359
|
+
|
|
360
|
+
// Raw fd 1 output is read on its own thread, which writes it out as text frames itself: a cell that fills the pipe
|
|
361
|
+
// from this thread (a child process inheriting fd 1, a writeSync loop) never waits on this thread, and output is never
|
|
362
|
+
// held in memory while the cell runs. A result waits behind a drain marker written to the same pipe, so every byte a
|
|
363
|
+
// cell wrote before it returned reaches the host before its result.
|
|
364
|
+
const pendingResults = new Map();
|
|
365
|
+
let drainSequence = 0;
|
|
366
|
+
const DRAINER = `const { readSync, writeSync } = require("node:fs");
|
|
367
|
+
const { StringDecoder } = require("node:string_decoder");
|
|
368
|
+
const { parentPort, workerData } = require("node:worker_threads");
|
|
369
|
+
const { fd, frameFd, token, lock, marker } = workerData;
|
|
370
|
+
const pause = new Int32Array(new SharedArrayBuffer(4));
|
|
371
|
+
function writeAll(bytes) {
|
|
372
|
+
let offset = 0;
|
|
373
|
+
while (offset < bytes.length) {
|
|
374
|
+
try {
|
|
375
|
+
offset += writeSync(frameFd, bytes, offset, bytes.length - offset);
|
|
376
|
+
} catch (error) {
|
|
377
|
+
if (error && error.code === "EAGAIN") { Atomics.wait(pause, 0, 0, 2); continue; }
|
|
378
|
+
// The host is gone. process.exit would end only this thread, leaving the frame lock held.
|
|
379
|
+
process.kill(process.pid, "SIGKILL");
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
// One read is at most 64 KiB, so a text frame never nears the frame limit.
|
|
384
|
+
function sendText(data) {
|
|
385
|
+
const line = Buffer.from(token + " " + JSON.stringify({ type: "text", stream: "stdout", data }) + "\\n", "utf8");
|
|
386
|
+
while (Atomics.compareExchange(lock, 0, 0, 1) !== 0) Atomics.wait(lock, 0, 1, 5);
|
|
387
|
+
try { writeAll(line); } finally { Atomics.store(lock, 0, 0); Atomics.notify(lock, 0, 1); }
|
|
388
|
+
}
|
|
389
|
+
const decoder = new StringDecoder("utf8");
|
|
390
|
+
const buffer = Buffer.alloc(1 << 16);
|
|
391
|
+
let carry = "";
|
|
392
|
+
${scanDrainText.toString()}
|
|
393
|
+
for (;;) {
|
|
394
|
+
let read;
|
|
395
|
+
try {
|
|
396
|
+
read = readSync(fd, buffer, 0, buffer.length, null);
|
|
397
|
+
} catch {
|
|
398
|
+
break;
|
|
399
|
+
}
|
|
400
|
+
if (read === 0) break;
|
|
401
|
+
const scanned = scanDrainText(carry, decoder.write(buffer.subarray(0, read)), marker);
|
|
402
|
+
carry = scanned.carry;
|
|
403
|
+
for (const part of scanned.parts) {
|
|
404
|
+
if (part.key !== undefined) parentPort.postMessage(part.key);
|
|
405
|
+
else sendText(part.text);
|
|
406
|
+
}
|
|
407
|
+
}`;
|
|
408
|
+
if (textPipe !== null) {
|
|
409
|
+
const reader = new Worker(DRAINER, {
|
|
410
|
+
eval: true,
|
|
411
|
+
workerData: {
|
|
412
|
+
fd: textPipe.readFd,
|
|
413
|
+
frameFd,
|
|
414
|
+
token: frameToken,
|
|
415
|
+
lock: frameLock,
|
|
416
|
+
marker: DRAIN_MARKER,
|
|
417
|
+
},
|
|
418
|
+
});
|
|
419
|
+
reader.on("message", (key) => {
|
|
420
|
+
const result = pendingResults.get(key);
|
|
421
|
+
pendingResults.delete(key);
|
|
422
|
+
if (result !== undefined) sendFrame(result);
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
// The crash cause reaches the host on stderr, tagged with the token, before the kernel ends (worker mode reports it
|
|
427
|
+
// through the thread's error event). It ends by SIGKILL, not process.exit: a crashed kernel runs no more code, and
|
|
428
|
+
// Node's exit can stall joining its own platform threads, which would keep a dead kernel and its cell waiting (#2757).
|
|
429
|
+
function reportCrash(error) {
|
|
430
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
431
|
+
// Capped well below the host's stderr window, so the cause line is never cut off.
|
|
432
|
+
const cause = { name: error instanceof Error ? error.name : "Error", message: message.slice(0, 4096) };
|
|
433
|
+
// Wait (briefly) for a frame another thread is writing, so the channel ends on a frame boundary and the host reads
|
|
434
|
+
// the crash, not a cut line. Holding the lock past this point keeps any new frame from starting.
|
|
435
|
+
const deadline = Date.now() + 200;
|
|
436
|
+
while (Atomics.compareExchange(frameLock, 0, 0, 1) !== 0 && Date.now() < deadline) Atomics.wait(frameLock, 0, 1, 5);
|
|
437
|
+
try {
|
|
438
|
+
writeBytes(2, fromString(`\nsenpi-kernel-crash ${frameToken} ${stringify(cause)}\n`));
|
|
439
|
+
} finally {
|
|
440
|
+
process.kill(process.pid, "SIGKILL");
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
process.on("uncaughtException", reportCrash);
|
|
444
|
+
// Once the worker core is up it reports unhandled rejections to the cells (the kernel keeps running); before that, one
|
|
445
|
+
// is a startup crash.
|
|
446
|
+
process.on("unhandledRejection", (reason) => {
|
|
447
|
+
if (process.listenerCount("unhandledRejection") === 1) reportCrash(reason);
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
const cwd = process.env.SENPI_CODEMODE_PROCESS_CWD ?? process.cwd();
|
|
451
|
+
const poolWidth = Number.parseInt(process.env.SENPI_CODEMODE_PROCESS_POOL_WIDTH ?? "1", 10) || 1;
|
|
452
|
+
// Kernel plumbing, not the session's environment: cells and their children do not inherit it.
|
|
453
|
+
delete process.env.SENPI_CODEMODE_PROCESS_CWD;
|
|
454
|
+
delete process.env.SENPI_CODEMODE_PROCESS_POOL_WIDTH;
|
|
455
|
+
|
|
456
|
+
const { markKernelProcessMode } = await import("./worker-webview.js");
|
|
457
|
+
markKernelProcessMode();
|
|
458
|
+
|
|
459
|
+
const { createWorkerCore } = await import("./worker-core.js");
|
|
460
|
+
|
|
461
|
+
createWorkerCore(transport, {
|
|
462
|
+
cwd,
|
|
463
|
+
parallelPoolWidth: poolWidth,
|
|
464
|
+
cwdInstallOptions: { allowMainThread: true },
|
|
465
|
+
processModeMemory: true,
|
|
466
|
+
});
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { KernelMemoryThresholds } from "../../bridge/memory-protocol.ts";
|
|
2
|
+
import { restartNotice } from "../shared/kernel-death.ts";
|
|
3
|
+
import { KernelMemoryPolicy } from "../shared/kernel-memory.ts";
|
|
4
|
+
import type { ResultMessage } from "./kernel-contract.ts";
|
|
5
|
+
|
|
6
|
+
export interface JavaScriptProcessMemoryReading {
|
|
7
|
+
readonly liveBytes: number;
|
|
8
|
+
readonly measure: "footprint";
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type JavaScriptProcessFootprintReader = (pid: number) => { readonly bytes: number } | undefined;
|
|
12
|
+
|
|
13
|
+
// Host-side memory for a process-mode js kernel: the in-heap worker reading cannot cross a process
|
|
14
|
+
// boundary, so the host reads the child's footprint (the py/rb/jl pattern) and the shared policy
|
|
15
|
+
// applies the notice and the ceiling. A crashed child re-announces itself through the restart notice
|
|
16
|
+
// on the next result, mirroring ReplaceableKernel's restarted tag.
|
|
17
|
+
export class JavaScriptProcessMemoryHost {
|
|
18
|
+
readonly #policy: KernelMemoryPolicy;
|
|
19
|
+
readonly #readFootprint: JavaScriptProcessFootprintReader | undefined;
|
|
20
|
+
#lastLiveBytes: number | undefined;
|
|
21
|
+
#restarted: string | null = null;
|
|
22
|
+
|
|
23
|
+
constructor(thresholds: KernelMemoryThresholds, readFootprint?: JavaScriptProcessFootprintReader) {
|
|
24
|
+
this.#readFootprint = readFootprint;
|
|
25
|
+
this.#policy = new KernelMemoryPolicy("js", thresholds, { collects: readFootprint === undefined });
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
get thresholds(): KernelMemoryThresholds {
|
|
29
|
+
return this.#policy.thresholds;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
lastLiveBytes(pid: () => number | undefined): number | undefined {
|
|
33
|
+
const reading = this.#read(pid);
|
|
34
|
+
if (reading !== undefined) this.#lastLiveBytes = reading.liveBytes;
|
|
35
|
+
return this.#lastLiveBytes;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
query(pid: () => number | undefined): JavaScriptProcessMemoryReading | undefined {
|
|
39
|
+
const reading = this.#read(pid);
|
|
40
|
+
if (reading !== undefined) this.#lastLiveBytes = reading.liveBytes;
|
|
41
|
+
return reading;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
settled(message: ResultMessage, pid: () => number | undefined): ResultMessage {
|
|
45
|
+
const reading = this.#read(pid);
|
|
46
|
+
if (reading !== undefined) this.#lastLiveBytes = reading.liveBytes;
|
|
47
|
+
let result: ResultMessage = message;
|
|
48
|
+
if (reading !== undefined) {
|
|
49
|
+
const report = { liveBytes: reading.liveBytes, measure: "footprint" as const };
|
|
50
|
+
result = { ...result, memory: this.#policy.annotate(report) };
|
|
51
|
+
}
|
|
52
|
+
if (this.#restarted !== null) {
|
|
53
|
+
const notice = this.#restarted;
|
|
54
|
+
this.#restarted = null;
|
|
55
|
+
result = { ...result, notice, kernelState: "restarted" };
|
|
56
|
+
}
|
|
57
|
+
return result;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
claimRecycle(idle: boolean): boolean {
|
|
61
|
+
if (!idle || !this.#policy.recyclePending) return false;
|
|
62
|
+
this.#policy.recycleStarted();
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
markRestarted(error: Error): void {
|
|
67
|
+
this.#restarted = restartNotice("js", error.message);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
workerLost(): void {
|
|
71
|
+
this.#policy.kernelRetired();
|
|
72
|
+
this.#lastLiveBytes = undefined;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
#read(pid: () => number | undefined): JavaScriptProcessMemoryReading | undefined {
|
|
76
|
+
if (this.#readFootprint === undefined) return undefined;
|
|
77
|
+
const current = pid();
|
|
78
|
+
if (current === undefined) return undefined;
|
|
79
|
+
const footprint = this.#readFootprint(current);
|
|
80
|
+
if (footprint === undefined) return undefined;
|
|
81
|
+
return { liveBytes: Math.round(footprint.bytes), measure: "footprint" };
|
|
82
|
+
}
|
|
83
|
+
}
|