@code-yeongyu/senpi-codemode 2026.10.6 → 2026.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +20 -2
- package/package.json +5 -4
- package/src/bridge/kernel-tools-protocol.ts +8 -0
- package/src/bridge/reserved.ts +10 -0
- package/src/bridges/eval-virtual-schemas.ts +100 -0
- package/src/bridges/handle-bridge.ts +72 -0
- package/src/bridges/reserved-dispatch.ts +20 -2
- package/src/bridges/schema-bridge.ts +4 -0
- package/src/completion/handle.ts +69 -0
- package/src/completion/tool-bridge.ts +31 -12
- package/src/environments/editable-check.ts +116 -0
- package/src/environments/install-lock.ts +223 -0
- package/src/environments/py-environment.ts +98 -0
- package/src/environments/py-installer.ts +187 -0
- package/src/environments/revision-store.ts +95 -0
- package/src/extension/bridge-dispatch.ts +41 -0
- package/src/extension/bridge-tool-call.ts +5 -1
- package/src/extension/host-facts.ts +18 -0
- package/src/extension/kernel-registration.ts +5 -0
- package/src/extension/removed-tool-hints.ts +15 -0
- package/src/extension/runtime-factory.ts +3 -0
- package/src/extension/session-manager-contract.ts +3 -0
- package/src/extension/session-manager.ts +32 -5
- package/src/extension/start-recovery.ts +111 -0
- package/src/handles/completion-handles.ts +202 -0
- package/src/handles/handle-args.ts +103 -0
- package/src/handles/handle-registry.ts +164 -0
- package/src/handles/wait.ts +214 -0
- package/src/handles/watch-queue.ts +102 -0
- package/src/index.ts +99 -94
- package/src/kernels/jl/handles.jl +124 -0
- package/src/kernels/jl/prelude.jl +9 -0
- package/src/kernels/jl/runner.jl +6 -1
- package/src/kernels/js/handles.js +93 -0
- package/src/kernels/js/kernel-tools-define.d.ts +6 -0
- package/src/kernels/js/kernel-tools-define.js +3 -1
- package/src/kernels/js/kernel-tools-errors.js +2 -0
- package/src/kernels/js/kernel-tools-errors.ts +2 -0
- package/src/kernels/js/kernel-tools-registry.d.ts +2 -0
- package/src/kernels/js/kernel-tools-registry.js +18 -9
- package/src/kernels/js/kernel-tools-schema.js +6 -2
- package/src/kernels/js/worker-runtime.js +10 -1
- package/src/kernels/py/host-cell.ts +42 -0
- package/src/kernels/py/kernel-contract.ts +13 -2
- package/src/kernels/py/kernel-tools-host.ts +97 -0
- package/src/kernels/py/kernel.ts +88 -12
- package/src/kernels/py/prelude.py +797 -11
- package/src/kernels/py/process.ts +15 -4
- package/src/kernels/py/startup.ts +58 -20
- package/src/kernels/py/transport.ts +36 -7
- package/src/kernels/rb/handles.rb +97 -0
- package/src/kernels/rb/paths.rb +37 -0
- package/src/kernels/rb/prelude.rb +15 -38
- package/src/kernels/rb/runner.rb +7 -7
- package/src/kernels/sandbox/vendor/pi-codemode/LICENSE +21 -0
- package/src/kernels/sandbox/vendor/pi-codemode/VENDORED.md +42 -0
- package/src/kernels/sandbox/vendor/pi-codemode/identifier.ts +14 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/host.ts +514 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/prelude-source.ts +431 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/protocol.ts +61 -0
- package/src/kernels/sandbox/vendor/pi-codemode/runtime/worker.ts +231 -0
- package/src/kernels/sandbox/vendor/pi-codemode/types.ts +185 -0
- package/src/kernels/sandbox/vendor/pi-codemode/wasm.ts +37 -0
- package/src/kernels/shared/kernel-tools-pump.ts +6 -0
- package/src/kernels/shared/process-cpu.ts +69 -0
- package/src/kernels/shared/subprocess-queue.ts +1 -0
- package/src/tool/cell-handler.ts +21 -18
- package/src/tool/cell-reserved-dispatch.ts +51 -0
- package/src/tool/eval-tool-options.ts +3 -0
- package/src/tool/run-eval-cell.ts +2 -0
- package/src/tool/types.ts +15 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
// Vendored from https://github.com/earendil-works/pi (packages/codemode/src) at v1.0.1, commit a7229ddc21810d6245105978033b7df645ecc2f7.
|
|
2
|
+
// MIT license; see LICENSE beside this file. Local changes are listed in VENDORED.md.
|
|
3
|
+
/**
|
|
4
|
+
* Worker thread entry. One worker runs one script inside a fresh QuickJS VM
|
|
5
|
+
* (a separate wasm instance), relays tool calls and output to the host, and
|
|
6
|
+
* reports the result. The host terminates the worker when the script settles,
|
|
7
|
+
* times out, or is aborted; the worker exists so that a spinning script never
|
|
8
|
+
* blocks the host thread.
|
|
9
|
+
*
|
|
10
|
+
* Importing this module starts the worker. Hosts that bundle their code (for
|
|
11
|
+
* example a Bun compiled executable) add a file that imports
|
|
12
|
+
* `@earendil-works/pi-codemode/worker` as a separate entrypoint and pass its URL
|
|
13
|
+
* or embedded-module string specifier as `workerUrl`.
|
|
14
|
+
*/
|
|
15
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
16
|
+
import { JSException, type JSValueHandle, MAX_STACK_SIZE, QuickJS } from "quickjs-wasi";
|
|
17
|
+
import { PRELUDE_SOURCE } from "./prelude-source.ts";
|
|
18
|
+
import { isHostToWorkerMessage, type WorkerData, type WorkerToHostMessage } from "./protocol.ts";
|
|
19
|
+
|
|
20
|
+
function post(message: WorkerToHostMessage): void {
|
|
21
|
+
parentPort?.postMessage(message);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function crash(error: unknown): void {
|
|
25
|
+
post({ type: "crash", message: error instanceof Error ? `${error.name}: ${error.message}` : String(error) });
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* QuickJS writes engine diagnostics to fd 1 and 2, which the default shim
|
|
30
|
+
* forwards to the host's stdout and stderr. That output belongs to the host
|
|
31
|
+
* application (for example a TUI), so it is discarded. Reporting every byte as
|
|
32
|
+
* written keeps libc from retrying.
|
|
33
|
+
*/
|
|
34
|
+
function discardOutput(memory: { readonly buffer: ArrayBufferLike }) {
|
|
35
|
+
return {
|
|
36
|
+
fd_write(_fd: number, iovsPtr: number, iovsLen: number, nwrittenPtr: number): number {
|
|
37
|
+
const view = new DataView(memory.buffer);
|
|
38
|
+
let written = 0;
|
|
39
|
+
for (let i = 0; i < iovsLen; i++) {
|
|
40
|
+
written += view.getUint32(iovsPtr + i * 8 + 4, true);
|
|
41
|
+
}
|
|
42
|
+
view.setUint32(nwrittenPtr, written, true);
|
|
43
|
+
return 0;
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function describeException(error: JSException): string {
|
|
49
|
+
const head = error.message ? `${error.name}: ${error.message}` : error.name;
|
|
50
|
+
const stack = error.stack?.trimEnd();
|
|
51
|
+
return JSON.stringify({ name: error.name, message: error.message, stack: stack ? `${head}\n${stack}` : head });
|
|
52
|
+
}
|
|
53
|
+
// senpi-change begin: output streaming
|
|
54
|
+
/**
|
|
55
|
+
* Sends one output item as frames of at most `frameBytes` UTF-16 bytes, each only after taking that many bytes
|
|
56
|
+
* of credit. With no credit left the worker sleeps until the host consumes a frame or interrupts the run, so
|
|
57
|
+
* neither side ever holds more than the window. Returns false when the run was interrupted while waiting.
|
|
58
|
+
*/
|
|
59
|
+
function streamItem(
|
|
60
|
+
stream: { credits: Int32Array; frameBytes: number },
|
|
61
|
+
interrupt: Int32Array,
|
|
62
|
+
itemId: number,
|
|
63
|
+
type: "text" | "image",
|
|
64
|
+
payload: string,
|
|
65
|
+
mimeType: string | undefined,
|
|
66
|
+
): boolean {
|
|
67
|
+
const frameUnits = Math.max(1, Math.floor(stream.frameBytes / 2));
|
|
68
|
+
let offset = 0;
|
|
69
|
+
let seq = 0;
|
|
70
|
+
do {
|
|
71
|
+
let end = Math.min(payload.length, offset + frameUnits);
|
|
72
|
+
// Never split a surrogate pair across frames.
|
|
73
|
+
if (end < payload.length && end - offset > 1) {
|
|
74
|
+
const last = payload.charCodeAt(end - 1);
|
|
75
|
+
if (last >= 0xd800 && last <= 0xdbff) end--;
|
|
76
|
+
}
|
|
77
|
+
const chunk = payload.slice(offset, end);
|
|
78
|
+
const bytes = Math.max(2, chunk.length * 2);
|
|
79
|
+
for (;;) {
|
|
80
|
+
if (Atomics.load(interrupt, 0) !== 0) return false;
|
|
81
|
+
const available = Atomics.load(stream.credits, 0);
|
|
82
|
+
if (available >= bytes) {
|
|
83
|
+
if (Atomics.compareExchange(stream.credits, 0, available, available - bytes) === available) break;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
Atomics.wait(stream.credits, 0, available);
|
|
87
|
+
}
|
|
88
|
+
post({
|
|
89
|
+
type: "output-frame",
|
|
90
|
+
bytes,
|
|
91
|
+
frame: {
|
|
92
|
+
itemId,
|
|
93
|
+
seq,
|
|
94
|
+
final: end >= payload.length,
|
|
95
|
+
type,
|
|
96
|
+
chunk,
|
|
97
|
+
...(mimeType === undefined ? {} : { mimeType }),
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
offset = end;
|
|
101
|
+
seq++;
|
|
102
|
+
} while (offset < payload.length);
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
// senpi-change end
|
|
106
|
+
|
|
107
|
+
async function main(data: WorkerData): Promise<void> {
|
|
108
|
+
const interrupt = new Int32Array(data.interrupt);
|
|
109
|
+
// senpi-change begin: output streaming
|
|
110
|
+
const stream =
|
|
111
|
+
data.stream === undefined
|
|
112
|
+
? undefined
|
|
113
|
+
: { credits: new Int32Array(data.stream.credits), frameBytes: data.stream.frameBytes };
|
|
114
|
+
let nextItemId = 0;
|
|
115
|
+
// senpi-change end
|
|
116
|
+
const vm = await QuickJS.create({
|
|
117
|
+
wasm: data.wasm,
|
|
118
|
+
memoryLimit: data.memoryLimitBytes,
|
|
119
|
+
// Without a guard, deep recursion overflows the wasm stack and traps instead of throwing a
|
|
120
|
+
// catchable RangeError.
|
|
121
|
+
maxStackSize: MAX_STACK_SIZE,
|
|
122
|
+
interruptHandler: () => Atomics.load(interrupt, 0) !== 0,
|
|
123
|
+
wasi: discardOutput,
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
// Called from the prelude with primitives only.
|
|
127
|
+
const bridge = vm.newFunction("bridge", (kind, a, b, c) => {
|
|
128
|
+
switch (kind.toString()) {
|
|
129
|
+
case "call":
|
|
130
|
+
case "global":
|
|
131
|
+
post({
|
|
132
|
+
type: "call",
|
|
133
|
+
id: a.toNumber(),
|
|
134
|
+
target: kind.toString() === "call" ? "tool" : "global",
|
|
135
|
+
name: b.toString(),
|
|
136
|
+
args: c === undefined || c.isUndefined ? undefined : c.toString(),
|
|
137
|
+
});
|
|
138
|
+
break;
|
|
139
|
+
case "output":
|
|
140
|
+
// senpi-change begin: output streaming
|
|
141
|
+
if (stream !== undefined) {
|
|
142
|
+
const image = a.toString() === "image";
|
|
143
|
+
streamItem(stream, interrupt, nextItemId++, image ? "image" : "text", b.toString(), image ? c.toString() : undefined);
|
|
144
|
+
break;
|
|
145
|
+
}
|
|
146
|
+
// senpi-change end
|
|
147
|
+
post({
|
|
148
|
+
type: "output",
|
|
149
|
+
item:
|
|
150
|
+
a.toString() === "image"
|
|
151
|
+
? { type: "image", data: b.toString(), mimeType: c.toString() }
|
|
152
|
+
: { type: "text", text: b.toString() },
|
|
153
|
+
});
|
|
154
|
+
break;
|
|
155
|
+
case "done":
|
|
156
|
+
if (a.toBoolean()) {
|
|
157
|
+
post({
|
|
158
|
+
type: "done",
|
|
159
|
+
ok: true,
|
|
160
|
+
value: b === undefined || b.isUndefined ? undefined : b.toString(),
|
|
161
|
+
writes: c.toString(),
|
|
162
|
+
});
|
|
163
|
+
} else {
|
|
164
|
+
post({ type: "done", ok: false, error: b.toString() });
|
|
165
|
+
}
|
|
166
|
+
break;
|
|
167
|
+
}
|
|
168
|
+
return vm.undefined;
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
// The VM lives until the host terminates the worker, so these handles are never disposed.
|
|
172
|
+
const api = vm.withScope((scope) =>
|
|
173
|
+
scope.escape(
|
|
174
|
+
vm.callFunction(
|
|
175
|
+
vm.evalCode(PRELUDE_SOURCE, "codemode-prelude.js"),
|
|
176
|
+
vm.undefined,
|
|
177
|
+
bridge,
|
|
178
|
+
vm.newString(JSON.stringify(data.tools)),
|
|
179
|
+
vm.newString(JSON.stringify(data.globals)),
|
|
180
|
+
vm.newString(JSON.stringify(data.store)),
|
|
181
|
+
// senpi-change begin: output streaming and builtin policy
|
|
182
|
+
vm.newString(JSON.stringify({ streamOutput: stream !== undefined, store: data.storePolicy ?? "collect" })),
|
|
183
|
+
// senpi-change end
|
|
184
|
+
),
|
|
185
|
+
),
|
|
186
|
+
);
|
|
187
|
+
const settle = api.getProp("settle");
|
|
188
|
+
const run = api.getProp("run");
|
|
189
|
+
const stalled = api.getProp("stalled");
|
|
190
|
+
/** Run queued jobs, then fail a script that waits on nothing that can ever resume it. */
|
|
191
|
+
const drain = () => {
|
|
192
|
+
vm.executePendingJobs();
|
|
193
|
+
vm.callFunction(stalled, api).dispose();
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
parentPort?.on("message", (message: unknown) => {
|
|
197
|
+
if (!isHostToWorkerMessage(message)) return;
|
|
198
|
+
try {
|
|
199
|
+
vm.withScope(() => {
|
|
200
|
+
vm.callFunction(
|
|
201
|
+
settle,
|
|
202
|
+
api,
|
|
203
|
+
vm.newNumber(message.id),
|
|
204
|
+
message.ok ? vm.true : vm.false,
|
|
205
|
+
message.payload === undefined ? vm.undefined : vm.newString(message.payload),
|
|
206
|
+
);
|
|
207
|
+
});
|
|
208
|
+
drain();
|
|
209
|
+
} catch (error) {
|
|
210
|
+
crash(error);
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
// The prefix shares the first line with the script so reported line numbers
|
|
215
|
+
// match the script as written.
|
|
216
|
+
let fn: JSValueHandle;
|
|
217
|
+
try {
|
|
218
|
+
fn = vm.evalCode(`(async (tools, console) => {${data.code}\n})`, "codemode.js");
|
|
219
|
+
} catch (error) {
|
|
220
|
+
if (!(error instanceof JSException)) throw error;
|
|
221
|
+
post({ type: "done", ok: false, error: describeException(error) });
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
vm.callFunction(run, api, fn).dispose();
|
|
225
|
+
fn.dispose();
|
|
226
|
+
drain();
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if (parentPort) {
|
|
230
|
+
main(workerData as WorkerData).catch(crash);
|
|
231
|
+
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// Vendored from https://github.com/earendil-works/pi (packages/codemode/src) at v1.0.1, commit a7229ddc21810d6245105978033b7df645ecc2f7.
|
|
2
|
+
// MIT license; see LICENSE beside this file. Local changes are listed in VENDORED.md.
|
|
3
|
+
import type { CodemodeWasmModule } from "./wasm.ts";
|
|
4
|
+
|
|
5
|
+
export interface CodemodeToolContext {
|
|
6
|
+
/**
|
|
7
|
+
* Aborted when the script finishes (including unawaited calls), the
|
|
8
|
+
* execution times out, the caller aborts, or the sandbox is closed.
|
|
9
|
+
*/
|
|
10
|
+
signal: AbortSignal;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** A JSON Schema document. Only used to render declarations; values are not validated against it. */
|
|
14
|
+
export type CodemodeJsonSchema = { [key: string]: unknown } | boolean;
|
|
15
|
+
|
|
16
|
+
export interface CodemodeTool {
|
|
17
|
+
/**
|
|
18
|
+
* The script calls tools as `tools.<id>(args)`, where `<id>` is the name with characters that
|
|
19
|
+
* are not valid in identifiers replaced by `_` (see `toCodemodeIdentifier`), and also as
|
|
20
|
+
* `tools["<name>"](args)`. Globals are called as `<name>(args)` and must be identifiers, or
|
|
21
|
+
* `<namespace>.<member>`, which groups them into a frozen namespace object.
|
|
22
|
+
*/
|
|
23
|
+
name: string;
|
|
24
|
+
/** Shown as a doc comment in {@link renderDeclarations}, and listed in `ALL_TOOLS` for tools. */
|
|
25
|
+
description?: string;
|
|
26
|
+
/** Schema of the single argument. Rendered as the parameter type; `unknown` when omitted. */
|
|
27
|
+
inputSchema?: CodemodeJsonSchema;
|
|
28
|
+
/** Schema of the resolved value. Rendered as the promise type; `unknown` when omitted. */
|
|
29
|
+
outputSchema?: CodemodeJsonSchema;
|
|
30
|
+
/** Globals only: `execute` receives all call arguments as an array instead of the first one. */
|
|
31
|
+
spread?: boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Globals only: TypeScript parameter list and return type for {@link renderDeclarations}, for
|
|
34
|
+
* example `(type: string, id?: string): Promise<Model[]>`. Replaces the rendering from the schemas.
|
|
35
|
+
*/
|
|
36
|
+
signature?: string;
|
|
37
|
+
/**
|
|
38
|
+
* `args` is whatever the script passed, after a JSON round trip. The return
|
|
39
|
+
* value must be JSON-serializable; a thrown error surfaces in the script as
|
|
40
|
+
* an `Error` with the same message.
|
|
41
|
+
*/
|
|
42
|
+
execute(args: unknown, context: CodemodeToolContext): Promise<unknown> | unknown;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* One item of the script's output, in the order the script produced it: `text()` and `console.*`
|
|
47
|
+
* produce text items, `image()` image items. `data` is base64.
|
|
48
|
+
*/
|
|
49
|
+
export type CodemodeOutputItem = { type: "text"; text: string } | { type: "image"; data: string; mimeType: string };
|
|
50
|
+
|
|
51
|
+
export type CodemodeCallStatus = "ok" | "error" | "cancelled";
|
|
52
|
+
|
|
53
|
+
export interface CodemodeCall {
|
|
54
|
+
name: string;
|
|
55
|
+
status: CodemodeCallStatus;
|
|
56
|
+
durationMs: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export type CodemodeErrorKind =
|
|
60
|
+
/** The script threw or failed to parse. `name` and `stack` come from the script's error. */
|
|
61
|
+
| "script"
|
|
62
|
+
/** The overall deadline expired. The worker was terminated. */
|
|
63
|
+
| "timeout"
|
|
64
|
+
/** The caller's signal fired or the sandbox was closed. The worker was terminated. */
|
|
65
|
+
| "aborted"
|
|
66
|
+
/** The worker or VM failed outside the script's control (for example a wasm trap or a missing worker file). */
|
|
67
|
+
| "sandbox";
|
|
68
|
+
|
|
69
|
+
export interface CodemodeError {
|
|
70
|
+
kind: CodemodeErrorKind;
|
|
71
|
+
name?: string;
|
|
72
|
+
message: string;
|
|
73
|
+
stack?: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Keys the script changed with `store()`. Only successful executions report writes. */
|
|
77
|
+
export interface CodemodeStoreWrites {
|
|
78
|
+
set: Record<string, unknown>;
|
|
79
|
+
/** Keys stored as `undefined`. */
|
|
80
|
+
delete: string[];
|
|
81
|
+
}
|
|
82
|
+
// senpi-change begin: output streaming
|
|
83
|
+
/**
|
|
84
|
+
* One transport frame of an output item in `output: "stream"` mode. An item is split into frames of at most
|
|
85
|
+
* `frameBytes` (UTF-16 bytes of `chunk`); frames of one item share `itemId`, arrive in `seq` order, and the
|
|
86
|
+
* last one has `final: true`. Image frames carry base64 text and repeat the item's `mimeType`.
|
|
87
|
+
*/
|
|
88
|
+
export interface CodemodeOutputFrame {
|
|
89
|
+
itemId: number;
|
|
90
|
+
seq: number;
|
|
91
|
+
final: boolean;
|
|
92
|
+
type: "text" | "image";
|
|
93
|
+
chunk: string;
|
|
94
|
+
mimeType?: string;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Bounds observed in `output: "stream"` mode: frames delivered and the most bytes ever in flight. */
|
|
98
|
+
export interface CodemodeStreamStats {
|
|
99
|
+
frames: number;
|
|
100
|
+
maxInFlightBytes: number;
|
|
101
|
+
windowBytes: number;
|
|
102
|
+
}
|
|
103
|
+
// senpi-change end
|
|
104
|
+
|
|
105
|
+
// senpi-change begin: output streaming (replaces upstream's CodemodeResult union to add `streamed`)
|
|
106
|
+
/** `output` is kept for failed executions too, up to the failure. `exit()` completes with `value: undefined`. */
|
|
107
|
+
export type CodemodeResult = (
|
|
108
|
+
| {
|
|
109
|
+
ok: true;
|
|
110
|
+
value: unknown;
|
|
111
|
+
output: CodemodeOutputItem[];
|
|
112
|
+
calls: CodemodeCall[];
|
|
113
|
+
storeWrites: CodemodeStoreWrites;
|
|
114
|
+
}
|
|
115
|
+
| { ok: false; error: CodemodeError; output: CodemodeOutputItem[]; calls: CodemodeCall[] }
|
|
116
|
+
) & {
|
|
117
|
+
/** Set in `output: "stream"` mode, where `output` stays empty because every item went to `onOutputFrame`. */
|
|
118
|
+
streamed?: CodemodeStreamStats;
|
|
119
|
+
};
|
|
120
|
+
// senpi-change end
|
|
121
|
+
|
|
122
|
+
export interface CodemodeSandboxOptions {
|
|
123
|
+
tools?: CodemodeTool[];
|
|
124
|
+
/**
|
|
125
|
+
* Functions exposed as top-level identifiers instead of on `tools`, for host helpers such as
|
|
126
|
+
* attaching an image to the result. They behave like tools (JSON round trip, promise result)
|
|
127
|
+
* but are not recorded in `result.calls`. Names must be identifiers and may not shadow the
|
|
128
|
+
* built-in globals (`tools`, `ALL_TOOLS`, `console`, `text`, `image`, `exit`, `store`, `load`).
|
|
129
|
+
*/
|
|
130
|
+
globals?: CodemodeTool[];
|
|
131
|
+
/**
|
|
132
|
+
* Overall deadline per execution, including time spent in tools. `Infinity` disables the
|
|
133
|
+
* deadline; the execution then only ends when the script settles or is aborted.
|
|
134
|
+
* Default: 300000.
|
|
135
|
+
*/
|
|
136
|
+
timeoutMs?: number;
|
|
137
|
+
/**
|
|
138
|
+
* Maximum memory the QuickJS VM may allocate. Allocations beyond it fail inside the script as
|
|
139
|
+
* `InternalError: out of memory`. Default: no limit beyond wasm32's 4 GiB address space.
|
|
140
|
+
*/
|
|
141
|
+
memoryLimitBytes?: number;
|
|
142
|
+
/**
|
|
143
|
+
* Compiled `quickjs-wasi/quickjs.wasm`, usually from {@link loadQuickJSWasm}. Default:
|
|
144
|
+
* `loadQuickJSWasm()`, the file in the installed `quickjs-wasi` package. Pass it when that file
|
|
145
|
+
* is not on disk, for example in a Bun compiled executable.
|
|
146
|
+
*/
|
|
147
|
+
wasm?: CodemodeWasmModule | Promise<CodemodeWasmModule>;
|
|
148
|
+
/**
|
|
149
|
+
* Worker entry that imports `@earendil-works/pi-codemode/worker`. Default: this package's own
|
|
150
|
+
* worker file. Pass it when this package is bundled, since the default is resolved relative to
|
|
151
|
+
* the module that creates the sandbox. Bun compiled executables require the relative string
|
|
152
|
+
* specifier of an embedded build entrypoint; other hosts usually use a URL.
|
|
153
|
+
*/
|
|
154
|
+
workerUrl?: string | URL;
|
|
155
|
+
// senpi-change begin: output streaming and builtin policy
|
|
156
|
+
/**
|
|
157
|
+
* `"collect"` (default) keeps output items until the script settles and returns them in `result.output`.
|
|
158
|
+
* `"stream"` keeps nothing: each item is split into frames handed to `onOutputFrame` as they arrive, the
|
|
159
|
+
* worker blocks once `windowBytes` of frames are unconsumed, and the per-script output limits do not apply.
|
|
160
|
+
*/
|
|
161
|
+
output?: "collect" | "stream";
|
|
162
|
+
/** Receives frames in order in `"stream"` mode; the next frame is handed over only after this settles. */
|
|
163
|
+
onOutputFrame?: (frame: CodemodeOutputFrame) => void | Promise<void>;
|
|
164
|
+
/** Bytes of frames that may be unconsumed at once in `"stream"` mode. Default: 262144. */
|
|
165
|
+
windowBytes?: number;
|
|
166
|
+
/** Largest frame in `"stream"` mode, at most `windowBytes`. Default: 65536. */
|
|
167
|
+
frameBytes?: number;
|
|
168
|
+
/**
|
|
169
|
+
* `store: "collect"` (default) is upstream's `store()`/`load()`. `"reject"` keeps the names but holds no state:
|
|
170
|
+
* `load()` returns `undefined` and `store()` throws an error named `CodemodeStoreDisabledError`.
|
|
171
|
+
*/
|
|
172
|
+
builtins?: { store?: "collect" | "reject" };
|
|
173
|
+
// senpi-change end
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export interface CodemodeExecuteOptions {
|
|
177
|
+
signal?: AbortSignal;
|
|
178
|
+
/** Overrides the sandbox default for this execution. */
|
|
179
|
+
timeoutMs?: number;
|
|
180
|
+
/**
|
|
181
|
+
* Values the script reads with `load(key)`. Must be JSON-serializable. The script's own
|
|
182
|
+
* `store()` calls come back as `result.storeWrites`; persisting them is up to the caller.
|
|
183
|
+
*/
|
|
184
|
+
store?: Readonly<Record<string, unknown>>;
|
|
185
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// Vendored from https://github.com/earendil-works/pi (packages/codemode/src) at v1.0.1, commit a7229ddc21810d6245105978033b7df645ecc2f7.
|
|
2
|
+
// MIT license; see LICENSE beside this file. Local changes are listed in VENDORED.md.
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import { createRequire } from "node:module";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A compiled `WebAssembly.Module` of `quickjs-wasi/quickjs.wasm`. Typed opaquely because the Node
|
|
8
|
+
* type definitions do not declare the WebAssembly globals (they live in the DOM lib).
|
|
9
|
+
*/
|
|
10
|
+
export type CodemodeWasmModule = object;
|
|
11
|
+
|
|
12
|
+
interface WebAssemblyGlobal {
|
|
13
|
+
compile(bytes: Uint8Array): Promise<CodemodeWasmModule>;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const modules = new Map<string, Promise<CodemodeWasmModule>>();
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Read and compile the QuickJS wasm once per path. `path` defaults to the file in the installed
|
|
20
|
+
* `quickjs-wasi` package; pass it when that file lives elsewhere, for example embedded in a Bun
|
|
21
|
+
* compiled executable. A failed load is retried on the next call.
|
|
22
|
+
*/
|
|
23
|
+
export function loadQuickJSWasm(path?: string): Promise<CodemodeWasmModule> {
|
|
24
|
+
const resolved = path ?? createRequire(import.meta.url).resolve("quickjs-wasi/quickjs.wasm");
|
|
25
|
+
let module = modules.get(resolved);
|
|
26
|
+
if (!module) {
|
|
27
|
+
const { WebAssembly } = globalThis as unknown as { WebAssembly: WebAssemblyGlobal };
|
|
28
|
+
module = readFile(resolved)
|
|
29
|
+
.then((bytes) => WebAssembly.compile(bytes))
|
|
30
|
+
.catch((error: unknown) => {
|
|
31
|
+
modules.delete(resolved);
|
|
32
|
+
throw error;
|
|
33
|
+
});
|
|
34
|
+
modules.set(resolved, module);
|
|
35
|
+
}
|
|
36
|
+
return module;
|
|
37
|
+
}
|
|
@@ -159,6 +159,9 @@ function codeOf(
|
|
|
159
159
|
| "kernel_tool_missing"
|
|
160
160
|
| "kernel_tool_recursion"
|
|
161
161
|
| "kernel_tool_host_denied"
|
|
162
|
+
| "kernel_tool_cancelled"
|
|
163
|
+
| "kernel_tool_loop_mismatch"
|
|
164
|
+
| "tool_name_collision"
|
|
162
165
|
| "tools_unavailable"
|
|
163
166
|
| "invalid_tool_definition" {
|
|
164
167
|
if (
|
|
@@ -166,6 +169,9 @@ function codeOf(
|
|
|
166
169
|
code === "kernel_tool_missing" ||
|
|
167
170
|
code === "kernel_tool_recursion" ||
|
|
168
171
|
code === "kernel_tool_host_denied" ||
|
|
172
|
+
code === "kernel_tool_cancelled" ||
|
|
173
|
+
code === "kernel_tool_loop_mismatch" ||
|
|
174
|
+
code === "tool_name_collision" ||
|
|
169
175
|
code === "tools_unavailable" ||
|
|
170
176
|
code === "invalid_tool_definition"
|
|
171
177
|
) {
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CPU time a kernel child has used so far, in platform units that only ever grow; a watchdog compares
|
|
3
|
+
* two readings and never converts.
|
|
4
|
+
*
|
|
5
|
+
* - linux and darwin: the child's whole process group (`readProcessGroupCpuTime`), so a helper process
|
|
6
|
+
* the runner forks counts as activity. The kernel child is spawned detached, so its pid is its pgid.
|
|
7
|
+
* - win32 under Bun: the child's own kernel + user time from `GetProcessTimes` (kernel32, via `bun:ffi`).
|
|
8
|
+
* Windows has no process groups; this is the interpreter itself when it is launched directly.
|
|
9
|
+
* - anything else, or Windows under Node: `undefined`, and the caller treats the CPU as unknown.
|
|
10
|
+
*/
|
|
11
|
+
import { readProcessGroupCpuTime } from "./process-group-cpu.ts";
|
|
12
|
+
|
|
13
|
+
type BunFfi = typeof import("bun:ffi");
|
|
14
|
+
type CpuReader = (pid: number) => bigint | undefined;
|
|
15
|
+
|
|
16
|
+
const PROCESS_QUERY_LIMITED_INFORMATION = 0x1000;
|
|
17
|
+
const FILETIME_BYTES = 8;
|
|
18
|
+
|
|
19
|
+
let windowsReader: CpuReader | null | undefined;
|
|
20
|
+
|
|
21
|
+
export function readKernelCpuTime(pid: number): bigint | undefined {
|
|
22
|
+
if (!Number.isInteger(pid) || pid <= 0) return undefined;
|
|
23
|
+
if (process.platform !== "win32") return readProcessGroupCpuTime(pid);
|
|
24
|
+
if (windowsReader === undefined) windowsReader = createWindowsReader() ?? null;
|
|
25
|
+
return windowsReader?.(pid);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function createWindowsReader(): CpuReader | undefined {
|
|
29
|
+
const ffi = process.getBuiltinModule("bun:ffi") as BunFfi | undefined;
|
|
30
|
+
if (ffi === undefined) return undefined;
|
|
31
|
+
try {
|
|
32
|
+
return bindGetProcessTimes(ffi);
|
|
33
|
+
} catch {
|
|
34
|
+
// kernel32 always exports these; a binding failure means this runtime cannot call it at all.
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function bindGetProcessTimes({ dlopen, FFIType, ptr }: BunFfi): CpuReader {
|
|
40
|
+
const library = dlopen("kernel32.dll", {
|
|
41
|
+
OpenProcess: { args: [FFIType.u32, FFIType.i32, FFIType.u32], returns: FFIType.ptr },
|
|
42
|
+
GetProcessTimes: {
|
|
43
|
+
args: [FFIType.ptr, FFIType.ptr, FFIType.ptr, FFIType.ptr, FFIType.ptr],
|
|
44
|
+
returns: FFIType.i32,
|
|
45
|
+
},
|
|
46
|
+
CloseHandle: { args: [FFIType.ptr], returns: FFIType.i32 },
|
|
47
|
+
});
|
|
48
|
+
const times = new Uint8Array(FILETIME_BYTES * 4);
|
|
49
|
+
const view = new DataView(times.buffer);
|
|
50
|
+
return (pid) => {
|
|
51
|
+
const handle = library.symbols.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid);
|
|
52
|
+
if (!handle) return undefined;
|
|
53
|
+
try {
|
|
54
|
+
times.fill(0);
|
|
55
|
+
const base = ptr(times);
|
|
56
|
+
const ok = library.symbols.GetProcessTimes(
|
|
57
|
+
handle,
|
|
58
|
+
base,
|
|
59
|
+
base + FILETIME_BYTES,
|
|
60
|
+
base + FILETIME_BYTES * 2,
|
|
61
|
+
base + FILETIME_BYTES * 3,
|
|
62
|
+
);
|
|
63
|
+
if (ok === 0) return undefined;
|
|
64
|
+
return view.getBigUint64(FILETIME_BYTES * 2, true) + view.getBigUint64(FILETIME_BYTES * 3, true);
|
|
65
|
+
} finally {
|
|
66
|
+
library.symbols.CloseHandle(handle);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
}
|
package/src/tool/cell-handler.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { appendSchemaHint } from "../bridges/schema-hint.ts";
|
|
|
14
14
|
import type { CompletionRequest, CompletionResult } from "../completion/handler.ts";
|
|
15
15
|
import { handleCompletionToolCall } from "../completion/tool-bridge.ts";
|
|
16
16
|
import type { ResolvedCodemodeSettings } from "../config/settings.ts";
|
|
17
|
+
import type { HandleRegistry } from "../handles/handle-registry.ts";
|
|
17
18
|
import type { KernelToolsCapability } from "../kernels/js/kernel-tools-types.ts";
|
|
18
19
|
import {
|
|
19
20
|
boundToolCallArgs,
|
|
@@ -24,6 +25,7 @@ import {
|
|
|
24
25
|
type ToolCallCapture,
|
|
25
26
|
toolCallResultPreview,
|
|
26
27
|
} from "./call-capture.ts";
|
|
28
|
+
import { completionCallOptions, reservedDispatchContext } from "./cell-reserved-dispatch.ts";
|
|
27
29
|
import { CellResultBuilder, type CellState } from "./cell-runtime.ts";
|
|
28
30
|
import { type EvalImageResizer, marshalToolResult, toolResultIsError } from "./image.ts";
|
|
29
31
|
import { upsertStatusEvent } from "./status-events.ts";
|
|
@@ -48,6 +50,10 @@ export interface CellBridgeRuntime {
|
|
|
48
50
|
readonly imageResizer?: EvalImageResizer;
|
|
49
51
|
/** This cell's live kernel-tool capability; only a JS kernel has one (#1754). */
|
|
50
52
|
readonly kernelTools?: KernelToolsCapability;
|
|
53
|
+
/** The session generation's handle registry behind `wait()` / `handle()` / completion handles. */
|
|
54
|
+
readonly handles?: HandleRegistry;
|
|
55
|
+
/** Absolute wall-clock deadline of this cell; a completion handle it creates is bounded by it. */
|
|
56
|
+
readonly hardDeadlineMs?: number;
|
|
51
57
|
}
|
|
52
58
|
|
|
53
59
|
export class CellHandler {
|
|
@@ -159,17 +165,12 @@ export class CellHandler {
|
|
|
159
165
|
await this.#deliverToolReply(
|
|
160
166
|
message,
|
|
161
167
|
async () => ({
|
|
162
|
-
value: await runReservedTool(
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
listTools: this.#runtime.listTools,
|
|
169
|
-
signal: this.#state.signal,
|
|
170
|
-
emitStatus: (event) => this.#recordStatus(event),
|
|
171
|
-
marshalToolResult,
|
|
172
|
-
}),
|
|
168
|
+
value: await runReservedTool(
|
|
169
|
+
message.toolName,
|
|
170
|
+
reservedDispatchContext(message, this.#runtime, this.#state.signal, (event) =>
|
|
171
|
+
this.#recordStatus(event),
|
|
172
|
+
),
|
|
173
|
+
),
|
|
173
174
|
toolCallOk: true,
|
|
174
175
|
}),
|
|
175
176
|
capture,
|
|
@@ -177,13 +178,15 @@ export class CellHandler {
|
|
|
177
178
|
return;
|
|
178
179
|
}
|
|
179
180
|
if (message.toolName === "completion" && this.#runtime.complete) {
|
|
180
|
-
const result = await handleCompletionToolCall(
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
181
|
+
const result = await handleCompletionToolCall(
|
|
182
|
+
completionCallOptions(
|
|
183
|
+
message,
|
|
184
|
+
this.#kernel,
|
|
185
|
+
this.#runtime,
|
|
186
|
+
this.#runtime.complete,
|
|
187
|
+
() => this.#state.active,
|
|
188
|
+
),
|
|
189
|
+
);
|
|
187
190
|
if (!this.#state.active) return;
|
|
188
191
|
recordToolCall(this.#state.toolCalls, result.ok, capture, undefined, result.ok ? undefined : result.error);
|
|
189
192
|
this.#resultBuilder.emitUpdate(false);
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { ExtensionContext } from "@code-yeongyu/senpi";
|
|
2
|
+
import type { KernelToHostMessage } from "../bridge/protocol.ts";
|
|
3
|
+
import type { ReservedDispatchContext } from "../bridges/reserved-dispatch.ts";
|
|
4
|
+
import type { CompletionRequest, CompletionResult } from "../completion/handler.ts";
|
|
5
|
+
import type { CompletionToolCallOptions } from "../completion/tool-bridge.ts";
|
|
6
|
+
import type { CellBridgeRuntime } from "./cell-handler.ts";
|
|
7
|
+
import { marshalToolResult } from "./image.ts";
|
|
8
|
+
import type { EvalKernel, EvalStatusEvent } from "./types.ts";
|
|
9
|
+
|
|
10
|
+
type ToolCallMessage = Extract<KernelToHostMessage, { type: "tool-call" }>;
|
|
11
|
+
|
|
12
|
+
/** The in-process (JS kernel) dispatch context for a reserved helper call; mirrors the bridge route for py/rb/jl. */
|
|
13
|
+
export function reservedDispatchContext(
|
|
14
|
+
message: ToolCallMessage,
|
|
15
|
+
runtime: CellBridgeRuntime,
|
|
16
|
+
signal: AbortSignal,
|
|
17
|
+
emitStatus: (event: EvalStatusEvent) => void,
|
|
18
|
+
): ReservedDispatchContext {
|
|
19
|
+
const host = runtime.ctx.evalHandleHost;
|
|
20
|
+
return {
|
|
21
|
+
callId: message.callId,
|
|
22
|
+
args: message.args,
|
|
23
|
+
executeTool: runtime.executeTool,
|
|
24
|
+
taskToolName: runtime.settings.taskTools.task,
|
|
25
|
+
taskOutputToolName: runtime.settings.taskTools.output,
|
|
26
|
+
listTools: runtime.listTools,
|
|
27
|
+
signal,
|
|
28
|
+
emitStatus,
|
|
29
|
+
marshalToolResult,
|
|
30
|
+
...(runtime.handles === undefined ? {} : { handles: runtime.handles }),
|
|
31
|
+
...(host === undefined ? {} : { evalHandleHost: host }),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function completionCallOptions(
|
|
36
|
+
message: ToolCallMessage,
|
|
37
|
+
kernel: EvalKernel,
|
|
38
|
+
runtime: CellBridgeRuntime,
|
|
39
|
+
complete: (request: CompletionRequest, ctx: ExtensionContext) => Promise<CompletionResult>,
|
|
40
|
+
isActive: () => boolean,
|
|
41
|
+
): CompletionToolCallOptions {
|
|
42
|
+
return {
|
|
43
|
+
message,
|
|
44
|
+
kernel,
|
|
45
|
+
complete,
|
|
46
|
+
ctx: runtime.ctx,
|
|
47
|
+
isActive,
|
|
48
|
+
...(runtime.handles === undefined ? {} : { handles: runtime.handles }),
|
|
49
|
+
...(runtime.hardDeadlineMs === undefined ? {} : { hardDeadlineMs: runtime.hardDeadlineMs }),
|
|
50
|
+
};
|
|
51
|
+
}
|