@code-yeongyu/senpi-codemode 2026.9.4-3 → 2026.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +2 -2
- package/package.json +4 -4
- package/src/extension/session-manager.ts +8 -4
- package/src/kernels/js/display-image.js +162 -0
- package/src/kernels/js/prelude.ts +2 -2
- package/src/kernels/js/worker-runtime.js +29 -60
- package/src/kernels/py/prelude.py +66 -1
- package/src/prompt/eval-prompt.ts +3 -3
- package/src/tool/types.ts +2 -2
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -112,12 +112,12 @@ options object and asynchronous helpers are `await`-able.
|
|
|
112
112
|
|
|
113
113
|
| Helper | Contract |
|
|
114
114
|
| --- | --- |
|
|
115
|
-
| `display(value)` | Emits text, structured JSON, markdown, or
|
|
115
|
+
| `display(value)` | Emits text, structured JSON, markdown, or image display data. Images reach the model only through `display`: pass a figure, raw image bytes (PNG/JPEG/GIF/WebP/BMP sniffed), a `data:` URL, a `Blob`-like or `Bun.Image` value, a marshalled tool result, or one of its `images[i]` frames. |
|
|
116
116
|
| `print(value, ...)` | Emits text output. |
|
|
117
117
|
| `read(path, offset?, limit?)` | Reads text with 1-indexed line slicing. `local://` paths resolve under the session artifact root. |
|
|
118
118
|
| `write(path, content)` | Creates parent directories and writes text. `local://` paths persist in the session artifact root. |
|
|
119
119
|
| `env(key?, value?)` | Reads all kernel environment values, one value, or sets one value. |
|
|
120
|
-
| `tool.<name>(args)` | Invokes an active Senpi tool through the normal `pi.executeTool` pipeline
|
|
120
|
+
| `tool.<name>(args)` | Invokes an active Senpi tool through the normal `pi.executeTool` pipeline and returns `{ text, images?, details?, hasError? }` in every kernel; image blocks arrive as `images[i] = { mimeType, dataBase64 }`. |
|
|
121
121
|
| `tool_schema(name?)` | Returns a tool's parameter schema without calling it; omit `name` to list tool names. |
|
|
122
122
|
| `completion(prompt, model?, system?, schema?)` | Requests a one-shot host completion; `schema` asks the host to parse structured output. |
|
|
123
123
|
| `agent(prompt, ...)` | Delegates to the configured active `taskTools.task` tool. Supports background handles and structured JSON results. |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@code-yeongyu/senpi-codemode",
|
|
3
|
-
"version": "2026.9.
|
|
3
|
+
"version": "2026.9.5",
|
|
4
4
|
"description": "Source-only senpi extension package for codemode evaluation tools",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.ts",
|
|
@@ -30,14 +30,14 @@
|
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
32
|
"@babel/parser": "8.0.4",
|
|
33
|
-
"@earendil-works/pi-ai": "npm:@code-yeongyu/senpi-ai@2026.9.
|
|
33
|
+
"@earendil-works/pi-ai": "npm:@code-yeongyu/senpi-ai@2026.9.5",
|
|
34
34
|
"typebox": "1.3.18"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
|
-
"@code-yeongyu/senpi": "2026.9.
|
|
37
|
+
"@code-yeongyu/senpi": "2026.9.5"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
|
-
"@code-yeongyu/senpi": "2026.9.
|
|
40
|
+
"@code-yeongyu/senpi": "2026.9.5"
|
|
41
41
|
},
|
|
42
42
|
"keywords": [
|
|
43
43
|
"senpi",
|
|
@@ -92,12 +92,16 @@ class DefaultCodemodeSessionManager implements CodemodeSessionManager {
|
|
|
92
92
|
});
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
// Subprocess kernels (py/rb/jl) reach the host only through this route, so
|
|
96
|
-
//
|
|
97
|
-
//
|
|
95
|
+
// Subprocess kernels (py/rb/jl) reach the host only through this route, so every reply
|
|
96
|
+
// must match the in-process JS path in tool/cell-handler.ts: reserved helper names dispatch
|
|
97
|
+
// through runReservedTool (forwarding them made agent() fail with "Unknown tool __agent__"),
|
|
98
|
+
// and ordinary tool results are marshalled to { text, images, details, hasError } — the raw
|
|
99
|
+
// { content } shape left python cells unable to reach tool.read image blocks.
|
|
98
100
|
async #call(request: { toolName: string; args: unknown; callId: string; signal: AbortSignal }): Promise<unknown> {
|
|
99
101
|
if (!isReservedToolName(request.toolName)) {
|
|
100
|
-
return
|
|
102
|
+
return marshalToolResult(
|
|
103
|
+
await this.#options.executeTool(request.toolName, request.args, { signal: request.signal }),
|
|
104
|
+
);
|
|
101
105
|
}
|
|
102
106
|
const taskTools = this.#options.settings.taskTools ?? defaultCodemodeSettings.taskTools;
|
|
103
107
|
return await runReservedTool(request.toolName, {
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
const BASE64_STRICT_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u;
|
|
2
|
+
const DECIMAL_CSV_RE = /^\d{1,3}(?:,\d{1,3})*$/u;
|
|
3
|
+
const DATA_URL_RE = /^data:([^;,]+)(?:;[^,]*)?;base64,([\s\S]*)$/u;
|
|
4
|
+
|
|
5
|
+
const IMAGE_SIGNATURES = [
|
|
6
|
+
{ mimeType: "image/png", offset: 0, bytes: [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] },
|
|
7
|
+
{ mimeType: "image/jpeg", offset: 0, bytes: [0xff, 0xd8, 0xff] },
|
|
8
|
+
{ mimeType: "image/gif", offset: 0, bytes: [0x47, 0x49, 0x46, 0x38] },
|
|
9
|
+
{ mimeType: "image/webp", offset: 8, bytes: [0x57, 0x45, 0x42, 0x50] },
|
|
10
|
+
{ mimeType: "image/bmp", offset: 0, bytes: [0x42, 0x4d] },
|
|
11
|
+
];
|
|
12
|
+
|
|
13
|
+
export function sniffImageMimeType(bytes) {
|
|
14
|
+
for (const signature of IMAGE_SIGNATURES) {
|
|
15
|
+
if (bytes.length < signature.offset + signature.bytes.length) continue;
|
|
16
|
+
if (signature.bytes.every((byte, index) => bytes[signature.offset + index] === byte)) return signature.mimeType;
|
|
17
|
+
}
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function isBinaryData(value) {
|
|
22
|
+
return value instanceof ArrayBuffer || ArrayBuffer.isView(value);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function bytesOf(value) {
|
|
26
|
+
if (value instanceof Uint8Array) return value;
|
|
27
|
+
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
28
|
+
if (ArrayBuffer.isView(value)) return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function normalizeBase64(text) {
|
|
33
|
+
const compact = text.replace(/\s+/gu, "").replace(/-/gu, "+").replace(/_/gu, "/");
|
|
34
|
+
const padded = compact.length % 4 === 0 ? compact : `${compact}${"=".repeat(4 - (compact.length % 4))}`;
|
|
35
|
+
return padded.length > 0 && BASE64_STRICT_RE.test(padded) ? padded : undefined;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function decimalCsvBase64(text) {
|
|
39
|
+
const parts = text.split(",");
|
|
40
|
+
const bytes = new Uint8Array(parts.length);
|
|
41
|
+
for (let index = 0; index < parts.length; index += 1) {
|
|
42
|
+
const byte = Number(parts[index]);
|
|
43
|
+
if (!Number.isInteger(byte) || byte < 0 || byte > 255) return undefined;
|
|
44
|
+
bytes[index] = byte;
|
|
45
|
+
}
|
|
46
|
+
return Buffer.from(bytes).toString("base64");
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function serializedBufferBase64(data) {
|
|
50
|
+
const bytes = new Uint8Array(data.length);
|
|
51
|
+
for (let index = 0; index < data.length; index += 1) {
|
|
52
|
+
const byte = data[index];
|
|
53
|
+
if (typeof byte !== "number" || !Number.isInteger(byte) || byte < 0 || byte > 255) return undefined;
|
|
54
|
+
bytes[index] = byte;
|
|
55
|
+
}
|
|
56
|
+
return Buffer.from(bytes).toString("base64");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function imagePayload(data) {
|
|
60
|
+
if (typeof data === "string") {
|
|
61
|
+
const dataUrl = DATA_URL_RE.exec(data);
|
|
62
|
+
if (dataUrl) {
|
|
63
|
+
const dataBase64 = normalizeBase64(dataUrl[2]);
|
|
64
|
+
return dataBase64 === undefined ? undefined : { dataBase64, mimeType: dataUrl[1] };
|
|
65
|
+
}
|
|
66
|
+
const dataBase64 = normalizeBase64(data);
|
|
67
|
+
if (dataBase64 !== undefined) return { dataBase64 };
|
|
68
|
+
return DECIMAL_CSV_RE.test(data) ? wrap(decimalCsvBase64(data)) : undefined;
|
|
69
|
+
}
|
|
70
|
+
const bytes = bytesOf(data);
|
|
71
|
+
if (bytes !== undefined) return { dataBase64: Buffer.from(bytes).toString("base64"), mimeType: sniffImageMimeType(bytes) };
|
|
72
|
+
if (isSerializedBuffer(data)) return wrap(serializedBufferBase64(data.data));
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function wrap(dataBase64) {
|
|
77
|
+
return dataBase64 === undefined ? undefined : { dataBase64 };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function isSerializedBuffer(data) {
|
|
81
|
+
return (
|
|
82
|
+
typeof data === "object" &&
|
|
83
|
+
data !== null &&
|
|
84
|
+
Object.getPrototypeOf(data) === Object.prototype &&
|
|
85
|
+
data.type === "Buffer" &&
|
|
86
|
+
Array.isArray(data.data)
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function describeImageData(data) {
|
|
91
|
+
if (data === null) return "null";
|
|
92
|
+
if (data instanceof Uint8Array) return "Uint8Array";
|
|
93
|
+
if (data instanceof ArrayBuffer) return "ArrayBuffer";
|
|
94
|
+
if (ArrayBuffer.isView(data)) return data.constructor.name;
|
|
95
|
+
if (typeof data === "string") return `string(${data.length})`;
|
|
96
|
+
return typeof data;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function isImageFrame(value) {
|
|
100
|
+
return (
|
|
101
|
+
typeof value === "object" &&
|
|
102
|
+
value !== null &&
|
|
103
|
+
typeof value.mimeType === "string" &&
|
|
104
|
+
typeof value.dataBase64 === "string"
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function isMarshalledToolResult(value) {
|
|
109
|
+
return typeof value.text === "string" && Array.isArray(value.images) && value.images.every(isImageFrame);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function isEncodableImage(value) {
|
|
113
|
+
return !isBinaryData(value) && (typeof value.arrayBuffer === "function" || typeof value.bytes === "function");
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function frame(mimeType, dataBase64) {
|
|
117
|
+
return { kind: "frame", mimeType, dataBase64 };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function dropped(reason) {
|
|
121
|
+
return [{ kind: "text", text: `[display: image dropped — ${reason}]` }];
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function resolveDisplayOps(value) {
|
|
125
|
+
if (typeof value !== "object" || value === null) return undefined;
|
|
126
|
+
if (value.type === "image" && typeof value.mimeType === "string") {
|
|
127
|
+
const payload = imagePayload(value.data);
|
|
128
|
+
return payload === undefined
|
|
129
|
+
? dropped(
|
|
130
|
+
`\`data\` must be a base64 string, data: URL, Uint8Array/Buffer, or ArrayBuffer; got ${describeImageData(value.data)}`,
|
|
131
|
+
)
|
|
132
|
+
: [frame(value.mimeType, payload.dataBase64)];
|
|
133
|
+
}
|
|
134
|
+
if (isImageFrame(value)) return [frame(value.mimeType, value.dataBase64)];
|
|
135
|
+
if (isMarshalledToolResult(value)) {
|
|
136
|
+
const frames = value.images.map((image) => frame(image.mimeType, image.dataBase64));
|
|
137
|
+
return value.text === "" ? frames : [{ kind: "text", text: value.text }, ...frames];
|
|
138
|
+
}
|
|
139
|
+
if (isBinaryData(value)) {
|
|
140
|
+
const payload = imagePayload(value);
|
|
141
|
+
return [frame(payload.mimeType ?? "application/octet-stream", payload.dataBase64)];
|
|
142
|
+
}
|
|
143
|
+
if (isEncodableImage(value)) return [{ kind: "encode", value }];
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export async function encodeDisplayImage(value) {
|
|
148
|
+
let raw;
|
|
149
|
+
try {
|
|
150
|
+
raw = typeof value.bytes === "function" ? await value.bytes() : await value.arrayBuffer();
|
|
151
|
+
} catch (error) {
|
|
152
|
+
if (!(error instanceof Error)) throw error;
|
|
153
|
+
return dropped(`encoding failed: ${error.message}`);
|
|
154
|
+
}
|
|
155
|
+
const bytes = bytesOf(raw);
|
|
156
|
+
if (bytes === undefined) return dropped(`encoder returned ${describeImageData(raw)}, not bytes`);
|
|
157
|
+
const declared = typeof value.type === "string" && value.type.startsWith("image/") ? value.type : undefined;
|
|
158
|
+
const mimeType = declared ?? sniffImageMimeType(bytes);
|
|
159
|
+
return mimeType === undefined
|
|
160
|
+
? dropped("bytes carry no recognizable image signature")
|
|
161
|
+
: [frame(mimeType, Buffer.from(bytes).toString("base64"))];
|
|
162
|
+
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
export const JAVASCRIPT_KERNEL_PRELUDE = [
|
|
2
2
|
"print(...values): write stdout text.",
|
|
3
|
-
"display(value): emit JSON,
|
|
3
|
+
"display(value): emit JSON, markdown, or image display output; image bytes, data: URLs, Blob-like values, tool results, and their images[i] frames all render as images.",
|
|
4
4
|
"log(message): emit a progress log line.",
|
|
5
5
|
"phase(title): emit a progress phase.",
|
|
6
6
|
"env(key?, value?): read, set, or list environment values.",
|
|
7
7
|
"read(path, options?): read UTF-8 text; plain paths use cwd and local:// uses the session local root.",
|
|
8
8
|
"write(path, content): write UTF-8 or binary data and return the resolved path.",
|
|
9
|
-
"tool.<name>(args): request a host tool call through the bridge.",
|
|
9
|
+
"tool.<name>(args): request a host tool call through the bridge; resolves to { text, images?, details?, hasError? }.",
|
|
10
10
|
"completion(prompt, options?): request a host completion bridge call.",
|
|
11
11
|
"output(...ids, options?): retrieve task output through the reserved output bridge.",
|
|
12
12
|
"agent(prompt, options?): delegate work through the reserved agent bridge.",
|
|
@@ -2,13 +2,12 @@
|
|
|
2
2
|
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
3
3
|
import { dirname, isAbsolute, join, normalize, resolve, sep } from "node:path";
|
|
4
4
|
import { inspect } from "node:util";
|
|
5
|
+
import { encodeDisplayImage, resolveDisplayOps } from "./display-image.js";
|
|
5
6
|
import { awaitMaybePromise, indirectEval, wrapUserCode } from "./worker-indirect-eval.js";
|
|
6
7
|
import { installShellCapture } from "./worker-shell-capture.js";
|
|
7
8
|
|
|
8
9
|
const PREPARED_CELL_PREFIX = "/*senpi:prepared-cell*/";
|
|
9
10
|
const INTERNAL_URL = /^([a-z][a-z0-9+.-]*):\/\/(.*)$/iu;
|
|
10
|
-
const BASE64_STRICT_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u;
|
|
11
|
-
const DECIMAL_CSV_RE = /^\d{1,3}(?:,\d{1,3})*$/u;
|
|
12
11
|
|
|
13
12
|
export class JsWorkerRuntime {
|
|
14
13
|
#cwd;
|
|
@@ -16,6 +15,7 @@ export class JsWorkerRuntime {
|
|
|
16
15
|
#localRoots;
|
|
17
16
|
#env = new Map();
|
|
18
17
|
#hooks = null;
|
|
18
|
+
#pendingDisplays = [];
|
|
19
19
|
|
|
20
20
|
constructor(options) {
|
|
21
21
|
this.#cwd = options.cwd;
|
|
@@ -36,12 +36,23 @@ export class JsWorkerRuntime {
|
|
|
36
36
|
({ prelude, code: cellCode } = prepared);
|
|
37
37
|
}
|
|
38
38
|
if (prelude) indirectEval(prelude, `${cellId}:prelude`);
|
|
39
|
-
|
|
39
|
+
const value = await awaitMaybePromise(indirectEval(wrapUserCode(cellCode), cellId));
|
|
40
|
+
await this.#drainPendingDisplays();
|
|
41
|
+
return value;
|
|
40
42
|
} finally {
|
|
43
|
+
this.#pendingDisplays = [];
|
|
41
44
|
this.#hooks = null;
|
|
42
45
|
}
|
|
43
46
|
}
|
|
44
47
|
|
|
48
|
+
async #drainPendingDisplays() {
|
|
49
|
+
while (this.#pendingDisplays.length > 0) {
|
|
50
|
+
const pending = this.#pendingDisplays;
|
|
51
|
+
this.#pendingDisplays = [];
|
|
52
|
+
await Promise.all(pending);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
45
56
|
#installGlobals() {
|
|
46
57
|
globalThis.print = (...values) => this.#emitText("stdout", `${values.map(formatValue).join(" ")}\n`);
|
|
47
58
|
globalThis.display = value => this.#display(value);
|
|
@@ -112,22 +123,8 @@ export class JsWorkerRuntime {
|
|
|
112
123
|
this.#hooks?.emit({ type: "display", mimeType: "text/markdown", dataBase64: encodeBase64(value.text) });
|
|
113
124
|
return;
|
|
114
125
|
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
if (dataBase64 !== undefined) {
|
|
118
|
-
this.#hooks?.emit({ type: "display", mimeType: value.mimeType, dataBase64 });
|
|
119
|
-
return;
|
|
120
|
-
}
|
|
121
|
-
this.#emitText(
|
|
122
|
-
"stdout",
|
|
123
|
-
`[display: image dropped — \`data\` must be a base64 string, Uint8Array/Buffer, or ArrayBuffer; got ${describeImageData(value.data)}]\n`,
|
|
124
|
-
);
|
|
125
|
-
return;
|
|
126
|
-
}
|
|
127
|
-
if (typeof value.mimeType === "string" && typeof value.dataBase64 === "string") {
|
|
128
|
-
this.#hooks?.emit({ type: "display", mimeType: value.mimeType, dataBase64: value.dataBase64 });
|
|
129
|
-
return;
|
|
130
|
-
}
|
|
126
|
+
const ops = resolveDisplayOps(value);
|
|
127
|
+
if (ops !== undefined) return this.#applyDisplayOps(ops);
|
|
131
128
|
try {
|
|
132
129
|
this.#hooks?.emit({ type: "display", mimeType: "application/json", dataBase64: encodeBase64(JSON.stringify(value)) });
|
|
133
130
|
} catch (error) {
|
|
@@ -139,6 +136,19 @@ export class JsWorkerRuntime {
|
|
|
139
136
|
this.#emitText("stdout", `${String(value)}\n`);
|
|
140
137
|
}
|
|
141
138
|
|
|
139
|
+
#applyDisplayOps(ops) {
|
|
140
|
+
let pending;
|
|
141
|
+
for (const op of ops) {
|
|
142
|
+
if (op.kind === "frame") this.#hooks?.emit({ type: "display", mimeType: op.mimeType, dataBase64: op.dataBase64 });
|
|
143
|
+
else if (op.kind === "text") this.#emitText("stdout", `${op.text}\n`);
|
|
144
|
+
else {
|
|
145
|
+
pending = encodeDisplayImage(op.value).then((encoded) => this.#applyDisplayOps(encoded));
|
|
146
|
+
this.#pendingDisplays.push(pending);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return pending;
|
|
150
|
+
}
|
|
151
|
+
|
|
142
152
|
#envHelper(key, value) {
|
|
143
153
|
if (key === undefined || key === null || key === "") {
|
|
144
154
|
const merged = Object.fromEntries(Object.entries({ ...process.env, ...Object.fromEntries(this.#env) }).sort());
|
|
@@ -357,47 +367,6 @@ async function writeData(value) {
|
|
|
357
367
|
throw new TypeError("write() expects string, Blob, ArrayBuffer, or TypedArray data");
|
|
358
368
|
}
|
|
359
369
|
|
|
360
|
-
function imageBase64(data) {
|
|
361
|
-
if (typeof data === "string") {
|
|
362
|
-
if (isStrictBase64(data)) return data;
|
|
363
|
-
if (!DECIMAL_CSV_RE.test(data)) return undefined;
|
|
364
|
-
const parts = data.split(",");
|
|
365
|
-
const bytes = new Uint8Array(parts.length);
|
|
366
|
-
for (let index = 0; index < parts.length; index += 1) {
|
|
367
|
-
const byte = Number(parts[index]);
|
|
368
|
-
if (!Number.isInteger(byte) || byte < 0 || byte > 255) return undefined;
|
|
369
|
-
bytes[index] = byte;
|
|
370
|
-
}
|
|
371
|
-
return Buffer.from(bytes).toString("base64");
|
|
372
|
-
}
|
|
373
|
-
if (data instanceof Uint8Array) return Buffer.from(data).toString("base64");
|
|
374
|
-
if (data instanceof ArrayBuffer) return Buffer.from(data).toString("base64");
|
|
375
|
-
if (ArrayBuffer.isView(data)) return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString("base64");
|
|
376
|
-
if (isPlainObject(data) && data.type === "Buffer" && Array.isArray(data.data)) {
|
|
377
|
-
const bytes = new Uint8Array(data.data.length);
|
|
378
|
-
for (let index = 0; index < data.data.length; index += 1) {
|
|
379
|
-
const byte = data.data[index];
|
|
380
|
-
if (typeof byte !== "number" || !Number.isInteger(byte) || byte < 0 || byte > 255) return undefined;
|
|
381
|
-
bytes[index] = byte;
|
|
382
|
-
}
|
|
383
|
-
return Buffer.from(bytes).toString("base64");
|
|
384
|
-
}
|
|
385
|
-
return undefined;
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
function isStrictBase64(value) {
|
|
389
|
-
return value.length > 0 && value.length % 4 === 0 && BASE64_STRICT_RE.test(value);
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
function describeImageData(data) {
|
|
393
|
-
if (data === null) return "null";
|
|
394
|
-
if (data instanceof Uint8Array) return "Uint8Array";
|
|
395
|
-
if (data instanceof ArrayBuffer) return "ArrayBuffer";
|
|
396
|
-
if (ArrayBuffer.isView(data)) return data.constructor.name;
|
|
397
|
-
if (typeof data === "string") return `string(${data.length})`;
|
|
398
|
-
return typeof data;
|
|
399
|
-
}
|
|
400
|
-
|
|
401
370
|
function chunkToString(chunk, encoding) {
|
|
402
371
|
if (typeof chunk === "string") return chunk;
|
|
403
372
|
if (chunk instanceof Uint8Array) return Buffer.from(chunk).toString(encoding ?? "utf8");
|
|
@@ -91,6 +91,68 @@ def b64_text(value: str) -> str:
|
|
|
91
91
|
return base64.b64encode(value.encode("utf-8")).decode("ascii")
|
|
92
92
|
|
|
93
93
|
|
|
94
|
+
_IMAGE_SIGNATURES: tuple[tuple[str, int, bytes], ...] = (
|
|
95
|
+
("image/png", 0, b"\x89PNG\r\n\x1a\n"),
|
|
96
|
+
("image/jpeg", 0, b"\xff\xd8\xff"),
|
|
97
|
+
("image/gif", 0, b"GIF8"),
|
|
98
|
+
("image/webp", 8, b"WEBP"),
|
|
99
|
+
("image/bmp", 0, b"BM"),
|
|
100
|
+
)
|
|
101
|
+
_DATA_URL_RE = re.compile(r"^data:([^;,]+)(?:;[^,]*)?;base64,(.*)$", re.DOTALL)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _sniff_image_mime_type(data: bytes) -> str | None:
|
|
105
|
+
for mime_type, offset, magic in _IMAGE_SIGNATURES:
|
|
106
|
+
if data[offset : offset + len(magic)] == magic:
|
|
107
|
+
return mime_type
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _image_base64(data: Any) -> tuple[str, str | None] | None:
|
|
112
|
+
if isinstance(data, (bytes, bytearray)):
|
|
113
|
+
raw = bytes(data)
|
|
114
|
+
return base64.b64encode(raw).decode("ascii"), _sniff_image_mime_type(raw)
|
|
115
|
+
if not isinstance(data, str):
|
|
116
|
+
return None
|
|
117
|
+
declared: str | None = None
|
|
118
|
+
match = _DATA_URL_RE.match(data)
|
|
119
|
+
if match:
|
|
120
|
+
declared, data = match.group(1), match.group(2)
|
|
121
|
+
compact = re.sub(r"\s+", "", data).replace("-", "+").replace("_", "/")
|
|
122
|
+
compact += "=" * (-len(compact) % 4)
|
|
123
|
+
try:
|
|
124
|
+
base64.b64decode(compact, validate=True)
|
|
125
|
+
except (ValueError, TypeError):
|
|
126
|
+
return None
|
|
127
|
+
return (compact, declared) if compact else None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _display_image_dict(value: dict[str, Any]) -> bool:
|
|
131
|
+
mime_type = value.get("mimeType")
|
|
132
|
+
payload = value.get("dataBase64") if "dataBase64" in value else value.get("data")
|
|
133
|
+
if not isinstance(mime_type, str) or not mime_type.startswith("image/") or payload is None:
|
|
134
|
+
return False
|
|
135
|
+
encoded = _image_base64(payload)
|
|
136
|
+
if encoded is None:
|
|
137
|
+
print(f"[display: image dropped \u2014 `data` must be base64, a data: URL, or bytes; got {type(payload).__name__}]")
|
|
138
|
+
return True
|
|
139
|
+
emit({"type": "display", "mimeType": mime_type, "dataBase64": encoded[0]})
|
|
140
|
+
return True
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _display_tool_result(value: dict[str, Any]) -> bool:
|
|
144
|
+
text, images = value.get("text"), value.get("images")
|
|
145
|
+
if not isinstance(text, str) or not isinstance(images, list):
|
|
146
|
+
return False
|
|
147
|
+
if not all(isinstance(image, dict) and isinstance(image.get("mimeType"), str) and isinstance(image.get("dataBase64"), str) for image in images):
|
|
148
|
+
return False
|
|
149
|
+
if text:
|
|
150
|
+
print(text)
|
|
151
|
+
for image in images:
|
|
152
|
+
emit({"type": "display", "mimeType": image["mimeType"], "dataBase64": image["dataBase64"]})
|
|
153
|
+
return True
|
|
154
|
+
|
|
155
|
+
|
|
94
156
|
def _emit_display(mime_type: str, data: Any) -> None:
|
|
95
157
|
if isinstance(data, (bytes, bytearray)):
|
|
96
158
|
encoded = base64.b64encode(bytes(data)).decode("ascii")
|
|
@@ -188,11 +250,14 @@ def _rich_bundle(value: Any) -> dict[str, Any]:
|
|
|
188
250
|
|
|
189
251
|
|
|
190
252
|
def display(value: Any) -> None:
|
|
253
|
+
if isinstance(value, dict) and (_display_tool_result(value) or _display_image_dict(value)):
|
|
254
|
+
return
|
|
191
255
|
if isinstance(value, (dict, list, tuple)):
|
|
192
256
|
_emit_display("application/json", value)
|
|
193
257
|
return
|
|
194
258
|
if isinstance(value, (bytes, bytearray)):
|
|
195
|
-
|
|
259
|
+
raw = bytes(value)
|
|
260
|
+
_emit_display(_sniff_image_mime_type(raw) or "application/octet-stream", raw)
|
|
196
261
|
return
|
|
197
262
|
bundle = _rich_bundle(value)
|
|
198
263
|
if bundle and _display_bundle(bundle):
|
|
@@ -112,7 +112,7 @@ On error, fix and re-run only the failing step; a normal error keeps state, whil
|
|
|
112
112
|
{{#ifAll py js}}Same helpers + arg order, both runtimes. Python: sync, options = trailing kwargs. JS: async/\`await\`able, options = ONE trailing object literal, never positional (extras throw).{{else}}{{#if py}}Sync; options = trailing kwargs.{{/if}}{{#if js}}Async/\`await\`able; options = ONE trailing object literal, never positional (extras throw).{{/if}}{{/ifAll}}{{#if rb}} Ruby: sync, options = trailing keyword args.{{/if}}{{#if jl}} Julia: sync, options = trailing keyword args.{{/if}}
|
|
113
113
|
\`\`\`
|
|
114
114
|
display(value) → None
|
|
115
|
-
Cell output
|
|
115
|
+
Cell output. Images reach you only through display: pass a figure, image bytes, a tool result, or its \`images[i]\`.
|
|
116
116
|
print(value, ...) → None
|
|
117
117
|
Text output.
|
|
118
118
|
read(path, offset?=1, limit?=None) → str
|
|
@@ -123,8 +123,8 @@ env(key?=None, value?=None) → str | None | dict
|
|
|
123
123
|
No args → full env dict; one → value; two → set \`key=value\`.
|
|
124
124
|
{{#if spawns}}output(*ids, format?="raw", offset?=None, limit?=None) → str | dict | list[dict]
|
|
125
125
|
Task/agent output by id. Reads immediately: running tasks return their status; \`format\` \`"raw"\` = full, \`"tail"\` = trailing.
|
|
126
|
-
{{/if}}tool.<name>(args) →
|
|
127
|
-
Invoke any session tool; \`
|
|
126
|
+
{{/if}}tool.<name>(args) → { text, images?, details?, hasError? }
|
|
127
|
+
Invoke any session tool; image results (e.g. \`tool.read\` on a png) arrive in \`images[i]\` as { mimeType, dataBase64 }.
|
|
128
128
|
tool_schema(name?) → dict
|
|
129
129
|
Parameter schema of a tool (omit \`name\` to list tool names); a failed \`tool.<name>()\` call also returns the expected parameters.
|
|
130
130
|
completion(prompt, model?="default", system?=None, schema?=None) → str | dict
|
package/src/tool/types.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { type TUnsafe, Type } from "typebox";
|
|
|
3
3
|
import type { HostToKernelMessage, KernelToHostMessage } from "../bridge/protocol.ts";
|
|
4
4
|
import type { TruncationMeta } from "../output/output-meta.ts";
|
|
5
5
|
|
|
6
|
-
export const evalLanguageOrder = ["
|
|
6
|
+
export const evalLanguageOrder = ["js", "py", "rb", "jl"] as const;
|
|
7
7
|
export type EvalLanguage = (typeof evalLanguageOrder)[number];
|
|
8
8
|
export type EnabledEvalLanguages = Readonly<Record<EvalLanguage, boolean>>;
|
|
9
9
|
|
|
@@ -43,7 +43,7 @@ const fullEvalInputSchema = Type.Object({
|
|
|
43
43
|
}),
|
|
44
44
|
),
|
|
45
45
|
language: Type.Optional(
|
|
46
|
-
Type.Union([Type.Literal("
|
|
46
|
+
Type.Union([Type.Literal("js"), Type.Literal("py"), Type.Literal("rb"), Type.Literal("jl")]),
|
|
47
47
|
),
|
|
48
48
|
code: Type.Optional(Type.String({ description: "Cell body, verbatim." })),
|
|
49
49
|
summary: Type.Optional(
|