@hexdecimal16/nexo 0.4.13 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/cli/src/common.js +1 -1
- package/cli/src/main.js +17 -2
- package/core/src/engine/context-trim.js +39 -0
- package/core/src/engine/messages.js +14 -1
- package/core/src/engine/run.js +36 -8
- package/core/src/providers/claude-p.js +75 -19
- package/core/src/providers/codex-p.js +16 -3
- package/core/src/providers/kiro-p.js +14 -6
- package/core/src/session/mentions.js +32 -8
- package/core/src/session/session.js +17 -13
- package/package.json +1 -1
- package/sdk/src/index.js +2 -0
- package/store/src/run-dir.js +10 -0
- package/tools/src/builtin.js +39 -8
- package/tools/src/images.js +174 -0
- package/tools/src/index.js +1 -0
- package/tui/src/app.js +51 -2
- package/tui/src/commands.js +1 -1
- package/tui/src/image-attachments.js +142 -0
- package/web/public/lib/span-view.ts +18 -1
- package/web/public/pages/run.ts +8 -1
- package/web/public/run.html +1 -0
- package/web/public/style.css +2 -0
package/package.json
CHANGED
package/sdk/src/index.js
CHANGED
|
@@ -3,5 +3,7 @@ export { SETTING_GROUPS, settingGroupLabel } from "../../core/src/index.js";
|
|
|
3
3
|
export { parseSkillInvocation } from "../../skills/src/index.js";
|
|
4
4
|
export { allowAllPrompter, nonInteractivePrompter } from "../../tools/src/index.js";
|
|
5
5
|
export { createRuntime, openLogger } from "./runtime.js";
|
|
6
|
+
/** Image helpers for `RunRequest.images`, `session.turn(text, { images })` and `session.steer(text, images)`. */
|
|
7
|
+
export { describeImage, isImageFile, loadImageFile, MAX_IMAGE_BYTES, MAX_IMAGE_EDGE, prepareImage, withAttachedImagesNote } from "../../tools/src/index.js";
|
|
6
8
|
export { NexoError } from "../../telemetry/src/index.js";
|
|
7
9
|
export { trackRunTree } from "./run-tree.js";
|
package/store/src/run-dir.js
CHANGED
|
@@ -102,6 +102,16 @@ export class RunDir {
|
|
|
102
102
|
this.writeConfined(this.toolOutputsDir, `${spanId}.txt`, content);
|
|
103
103
|
return `tool_outputs/${spanId}.txt`;
|
|
104
104
|
}
|
|
105
|
+
/**
|
|
106
|
+
* Saves a base64 image the run saw (an attached input, a screenshot a tool returned) and returns its path relative to
|
|
107
|
+
* the run directory, e.g. `inputs/image-1.png`. Image bytes are not redacted.
|
|
108
|
+
*/
|
|
109
|
+
writeImage(relPath, base64) {
|
|
110
|
+
const target = confine(this.root, relPath);
|
|
111
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
112
|
+
writeFileSync(target, Buffer.from(base64, "base64"));
|
|
113
|
+
return relPath;
|
|
114
|
+
}
|
|
105
115
|
resolveFile(relPath) {
|
|
106
116
|
return confine(this.root, relPath);
|
|
107
117
|
}
|
package/tools/src/builtin.js
CHANGED
|
@@ -6,6 +6,7 @@ import { NexoError } from "../../telemetry/src/index.js";
|
|
|
6
6
|
import { Type } from "typebox";
|
|
7
7
|
import { canonicalPath, nexoHomeGuard, resolveConfined, SECRET_PATH } from "./confine.js";
|
|
8
8
|
import { htmlToText, looksLikeHtml } from "./html-text.js";
|
|
9
|
+
import { describeImage, isImageFile, loadImageFile, MAX_IMAGE_EDGE } from "./images.js";
|
|
9
10
|
export const MAX_BASH_OUTPUT_BYTES = 1_000_000;
|
|
10
11
|
const MAX_READ_LINES = 2000;
|
|
11
12
|
const MAX_FETCH_CHARS = 100_000;
|
|
@@ -177,39 +178,69 @@ export function createBuiltinTools(ctx) {
|
|
|
177
178
|
}
|
|
178
179
|
throw new Error(`File not found: ${path}`);
|
|
179
180
|
};
|
|
180
|
-
|
|
181
|
+
/** A read path made absolute: saved tool outputs, or a file anywhere outside the secret-file rule. */
|
|
182
|
+
const locateRead = (path) => {
|
|
181
183
|
const outputs = ctx.toolOutputsDir;
|
|
182
184
|
const savedOutput = outputs !== undefined && path.startsWith(TOOL_OUTPUT_PREFIX);
|
|
183
185
|
const abs = savedOutput ? resolveSavedOutput(outputs, path) : resolveReadPath(path);
|
|
184
186
|
guard?.checkRead(abs);
|
|
185
187
|
if (!existsSync(abs))
|
|
186
188
|
throw new Error(`File not found: ${path}`);
|
|
189
|
+
return abs;
|
|
190
|
+
};
|
|
191
|
+
const readText = (abs, offset, limit) => {
|
|
187
192
|
const lines = readFileSync(abs, "utf8").split("\n");
|
|
188
193
|
const start = (offset ?? 1) - 1;
|
|
189
194
|
return lines.slice(start, start + (limit ?? MAX_READ_LINES)).join("\n");
|
|
190
195
|
};
|
|
196
|
+
/** A file's content blocks: its lines as text, or for an image a line describing it and the image itself. */
|
|
197
|
+
const readOne = (path, offset, limit) => {
|
|
198
|
+
const abs = locateRead(path);
|
|
199
|
+
if (isImageFile(abs)) {
|
|
200
|
+
const loaded = loadImageFile(abs);
|
|
201
|
+
return [{ type: "text", text: `Image ${describeImage(path, loaded)}:` }, loaded.image];
|
|
202
|
+
}
|
|
203
|
+
return [{ type: "text", text: readText(abs, offset, limit) }];
|
|
204
|
+
};
|
|
191
205
|
const read = {
|
|
192
206
|
name: "read",
|
|
193
207
|
label: "Read file",
|
|
194
|
-
description: `Read text files. Relative paths start at the workspace; absolute paths are read where they are, except key, token and credential files (use bash for those). Pass paths to read several files in one call. Paths starting with ${TOOL_OUTPUT_PREFIX} open saved large tool outputs.`,
|
|
208
|
+
description: `Read text files and images (PNG, JPEG, GIF, WebP; large images are scaled down to ${MAX_IMAGE_EDGE} px). Relative paths start at the workspace; absolute paths are read where they are, except key, token and credential files (use bash for those). Pass paths to read several files in one call. Paths starting with ${TOOL_OUTPUT_PREFIX} open saved large tool outputs.`,
|
|
195
209
|
parameters: ReadParams,
|
|
196
210
|
execute: async (_id, params) => {
|
|
197
211
|
if (params.paths !== undefined) {
|
|
198
212
|
if (params.path !== undefined)
|
|
199
213
|
throw new Error("Give either path or paths, not both");
|
|
200
|
-
const
|
|
214
|
+
const content = [];
|
|
215
|
+
let failures = 0;
|
|
216
|
+
for (const p of params.paths) {
|
|
201
217
|
try {
|
|
202
|
-
|
|
218
|
+
const blocks = readOne(p.path, p.offset, p.limit);
|
|
219
|
+
const first = blocks[0];
|
|
220
|
+
if (first?.type === "text")
|
|
221
|
+
content.push({ type: "text", text: `${content.length > 0 ? "\n" : ""}==> ${p.path} <==\n${first.text}` }, ...blocks.slice(1));
|
|
222
|
+
else
|
|
223
|
+
content.push({ type: "text", text: `==> ${p.path} <==` }, ...blocks);
|
|
203
224
|
}
|
|
204
225
|
catch (error) {
|
|
205
|
-
|
|
226
|
+
failures++;
|
|
227
|
+
content.push({ type: "text", text: `${content.length > 0 ? "\n" : ""}==> ${p.path} <==\n[error] ${error instanceof Error ? error.message : String(error)}` });
|
|
206
228
|
}
|
|
207
|
-
}
|
|
208
|
-
|
|
229
|
+
}
|
|
230
|
+
// Adjacent text blocks are joined so text-only reads keep their single-block shape.
|
|
231
|
+
const merged = [];
|
|
232
|
+
for (const block of content) {
|
|
233
|
+
const last = merged.at(-1);
|
|
234
|
+
if (block.type === "text" && last?.type === "text")
|
|
235
|
+
last.text += `\n${block.text}`;
|
|
236
|
+
else
|
|
237
|
+
merged.push(block.type === "text" ? { ...block } : block);
|
|
238
|
+
}
|
|
239
|
+
return { content: merged, details: undefined, ...(failures === params.paths.length ? { isError: true } : {}) };
|
|
209
240
|
}
|
|
210
241
|
if (params.path === undefined)
|
|
211
242
|
throw new Error("Give path or paths");
|
|
212
|
-
return
|
|
243
|
+
return { content: readOne(params.path, params.offset, params.limit), details: undefined };
|
|
213
244
|
},
|
|
214
245
|
};
|
|
215
246
|
const write = {
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { basename, join } from "node:path";
|
|
5
|
+
/** File extensions treated as images when a path is dropped or pasted (the bytes still decide). */
|
|
6
|
+
export const IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".webp"];
|
|
7
|
+
/**
|
|
8
|
+
* Long-edge limit images are scaled down to before they reach a model. Claude's high-resolution tier stops at 2,576 px
|
|
9
|
+
* and many-image requests at 3,000 px; 2,000 px keeps text legible while staying under every provider's limit.
|
|
10
|
+
*/
|
|
11
|
+
export const MAX_IMAGE_EDGE = 2000;
|
|
12
|
+
/** Largest image sent as is, in raw bytes (about 4.7 MB base64): under Bedrock's 5 MB base64 per-image limit. */
|
|
13
|
+
export const MAX_IMAGE_BYTES = 3_500_000;
|
|
14
|
+
/** Images larger than this are refused outright rather than read into memory. */
|
|
15
|
+
const MAX_SOURCE_BYTES = 50 * 1024 * 1024;
|
|
16
|
+
/** The image type from the file's first bytes, or null when they are not a supported image. */
|
|
17
|
+
export function sniffImageMime(bytes) {
|
|
18
|
+
const b = bytes;
|
|
19
|
+
if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47)
|
|
20
|
+
return "image/png";
|
|
21
|
+
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff)
|
|
22
|
+
return "image/jpeg";
|
|
23
|
+
if (b.length >= 6 && b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x38)
|
|
24
|
+
return "image/gif";
|
|
25
|
+
if (b.length >= 12 && ascii(b, 0, 4) === "RIFF" && ascii(b, 8, 12) === "WEBP")
|
|
26
|
+
return "image/webp";
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
function ascii(b, from, to) {
|
|
30
|
+
return String.fromCharCode(...b.subarray(from, to));
|
|
31
|
+
}
|
|
32
|
+
/** Pixel size read from the image header; null when the header is not understood. */
|
|
33
|
+
export function imageDimensions(bytes, mime) {
|
|
34
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
35
|
+
try {
|
|
36
|
+
if (mime === "image/png")
|
|
37
|
+
return { width: view.getUint32(16), height: view.getUint32(20) };
|
|
38
|
+
if (mime === "image/gif")
|
|
39
|
+
return { width: view.getUint16(6, true), height: view.getUint16(8, true) };
|
|
40
|
+
if (mime === "image/webp") {
|
|
41
|
+
const chunk = ascii(bytes, 12, 16);
|
|
42
|
+
if (chunk === "VP8X")
|
|
43
|
+
return { width: 1 + (view.getUint32(24, true) & 0xffffff), height: 1 + (view.getUint32(27, true) & 0xffffff) };
|
|
44
|
+
if (chunk === "VP8L") {
|
|
45
|
+
const bits = view.getUint32(21, true);
|
|
46
|
+
return { width: 1 + (bits & 0x3fff), height: 1 + ((bits >> 14) & 0x3fff) };
|
|
47
|
+
}
|
|
48
|
+
if (chunk === "VP8 ")
|
|
49
|
+
return { width: view.getUint16(26, true) & 0x3fff, height: view.getUint16(28, true) & 0x3fff };
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
// JPEG: walk the segments to the first start-of-frame marker.
|
|
53
|
+
let offset = 2;
|
|
54
|
+
while (offset + 9 < bytes.length) {
|
|
55
|
+
if (bytes[offset] !== 0xff)
|
|
56
|
+
return null;
|
|
57
|
+
const marker = bytes[offset + 1];
|
|
58
|
+
if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
|
|
59
|
+
return { width: view.getUint16(offset + 7), height: view.getUint16(offset + 5) };
|
|
60
|
+
}
|
|
61
|
+
offset += 2 + view.getUint16(offset + 2);
|
|
62
|
+
}
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function hasCommand(command) {
|
|
70
|
+
try {
|
|
71
|
+
execFileSync(process.platform === "win32" ? "where" : "which", [command], { stdio: "ignore" });
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Scales an image down with the platform's own tools (`sips` on macOS, ImageMagick elsewhere) so its long edge is at
|
|
80
|
+
* most `maxEdge`, as JPEG when the source is not a PNG or when PNG stays too large. Null when no tool is available or
|
|
81
|
+
* the tool fails; the caller then sends the original.
|
|
82
|
+
*/
|
|
83
|
+
function downscale(bytes, mime, maxEdge, maxBytes) {
|
|
84
|
+
const dir = mkdtempSync(join(tmpdir(), "nexo-image-"));
|
|
85
|
+
try {
|
|
86
|
+
const src = join(dir, `in.${mime.slice(6)}`);
|
|
87
|
+
writeFileSync(src, bytes);
|
|
88
|
+
const attempt = (format) => {
|
|
89
|
+
const out = join(dir, `out.${format === "png" ? "png" : "jpg"}`);
|
|
90
|
+
if (process.platform === "darwin" && hasCommand("sips")) {
|
|
91
|
+
execFileSync("sips", ["-Z", String(maxEdge), "-s", "format", format, ...(format === "jpeg" ? ["-s", "formatOptions", "85"] : []), src, "--out", out], { stdio: "ignore", timeout: 30_000 });
|
|
92
|
+
}
|
|
93
|
+
else if (hasCommand("magick") || hasCommand("convert")) {
|
|
94
|
+
const tool = hasCommand("magick") ? "magick" : "convert";
|
|
95
|
+
execFileSync(tool, [`${src}[0]`, "-resize", `${maxEdge}x${maxEdge}>`, ...(format === "jpeg" ? ["-quality", "85"] : []), out], { stdio: "ignore", timeout: 30_000 });
|
|
96
|
+
}
|
|
97
|
+
else
|
|
98
|
+
return null;
|
|
99
|
+
return readFileSync(out);
|
|
100
|
+
};
|
|
101
|
+
// PNG keeps screenshots and text crisp; fall back to JPEG when PNG is still too large.
|
|
102
|
+
const png = mime === "image/png" ? attempt("png") : null;
|
|
103
|
+
if (png && png.length <= maxBytes)
|
|
104
|
+
return { bytes: png, mime: "image/png" };
|
|
105
|
+
const jpeg = attempt("jpeg");
|
|
106
|
+
return jpeg ? { bytes: jpeg, mime: "image/jpeg" } : null;
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
rmSync(dir, { recursive: true, force: true });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Prepares image bytes for a model: checks the type by content, and scales the image down when its long edge or size
|
|
117
|
+
* passes the limits. Throws when the bytes are not a PNG, JPEG, GIF or WebP image.
|
|
118
|
+
*/
|
|
119
|
+
export function prepareImage(bytes, options = {}) {
|
|
120
|
+
const mime = sniffImageMime(bytes);
|
|
121
|
+
if (!mime)
|
|
122
|
+
throw new Error("Not a supported image (PNG, JPEG, GIF or WebP)");
|
|
123
|
+
const maxEdge = options.maxEdge ?? MAX_IMAGE_EDGE;
|
|
124
|
+
const maxBytes = options.maxBytes ?? MAX_IMAGE_BYTES;
|
|
125
|
+
const size = imageDimensions(bytes, mime);
|
|
126
|
+
const tooLarge = (size !== null && Math.max(size.width, size.height) > maxEdge) || bytes.length > maxBytes;
|
|
127
|
+
// Animated GIFs would lose their frames; send them as they are.
|
|
128
|
+
const scaled = tooLarge && mime !== "image/gif" ? downscale(bytes, mime, maxEdge, maxBytes) : null;
|
|
129
|
+
if (scaled) {
|
|
130
|
+
const after = imageDimensions(scaled.bytes, scaled.mime);
|
|
131
|
+
return {
|
|
132
|
+
image: { type: "image", data: scaled.bytes.toString("base64"), mimeType: scaled.mime },
|
|
133
|
+
width: after?.width ?? null,
|
|
134
|
+
height: after?.height ?? null,
|
|
135
|
+
...(size ? { original: { ...size, bytes: bytes.length } } : {}),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
return { image: { type: "image", data: Buffer.from(bytes).toString("base64"), mimeType: mime }, width: size?.width ?? null, height: size?.height ?? null };
|
|
139
|
+
}
|
|
140
|
+
/** True when the file starts with image bytes. False for missing files, directories and anything else. */
|
|
141
|
+
export function isImageFile(path) {
|
|
142
|
+
try {
|
|
143
|
+
if (!statSync(path).isFile())
|
|
144
|
+
return false;
|
|
145
|
+
const head = readFileSync(path).subarray(0, 16);
|
|
146
|
+
return sniffImageMime(head) !== null;
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
/** Reads an image file and prepares it for a model (see prepareImage). Throws for a missing, huge or non-image file. */
|
|
153
|
+
export function loadImageFile(path, options = {}) {
|
|
154
|
+
const stat = statSync(path);
|
|
155
|
+
if (!stat.isFile())
|
|
156
|
+
throw new Error(`${path} is not a file`);
|
|
157
|
+
if (stat.size > MAX_SOURCE_BYTES)
|
|
158
|
+
throw new Error(`${basename(path)} is ${(stat.size / 1024 / 1024).toFixed(1)} MB; images over 50 MB are not read`);
|
|
159
|
+
return prepareImage(readFileSync(path), options);
|
|
160
|
+
}
|
|
161
|
+
/** One-line description of a loaded image for the text next to it, e.g. `shot.png (1456×819 PNG, scaled from 3840×2160)`. */
|
|
162
|
+
export function describeImage(name, loaded) {
|
|
163
|
+
const kind = loaded.image.mimeType.slice(6).toUpperCase();
|
|
164
|
+
const dims = loaded.width !== null && loaded.height !== null ? `${loaded.width}×${loaded.height} ` : "";
|
|
165
|
+
const scaled = loaded.original ? `, scaled from ${loaded.original.width}×${loaded.original.height}` : "";
|
|
166
|
+
return `${name} (${dims}${kind}${scaled})`;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* The line added to a message that carries images, naming each in order (e.g. `shot.png (1456×819 PNG)`), so the model
|
|
170
|
+
* can tell which image the user means. Empty labels give the text unchanged.
|
|
171
|
+
*/
|
|
172
|
+
export function withAttachedImagesNote(text, labels) {
|
|
173
|
+
return labels.length === 0 ? text : `${text}\n\n[nexo] Attached images, in order: ${labels.join("; ")}.`;
|
|
174
|
+
}
|
package/tools/src/index.js
CHANGED
package/tui/src/app.js
CHANGED
|
@@ -9,6 +9,7 @@ import { createApprovalPrompter } from "./approvals.js";
|
|
|
9
9
|
import { helpText, isSkillCommand, parseSlashCommand, SLASH_COMMANDS, skillSlashCommands } from "./commands.js";
|
|
10
10
|
import { Block, DynamicLine, QueuedList, SafeContainer, SafeMarkdown, SubagentLine, SubagentPanel, Ticker, ToolLine, ViewMode } from "./components.js";
|
|
11
11
|
import { createAutocompleteProvider } from "./file-autocomplete.js";
|
|
12
|
+
import { ImageAttachments, readClipboardImage, withImageLabels } from "./image-attachments.js";
|
|
12
13
|
import { createSessionPickerScreen } from "./session-picker.js";
|
|
13
14
|
import { SettingsScreen } from "./settings-screen.js";
|
|
14
15
|
import { escapeControl, escapeControlKeepLines, formatError, oneLine, toolSummary, usd } from "./text.js";
|
|
@@ -118,6 +119,9 @@ class InteractiveApp {
|
|
|
118
119
|
runningShell = false;
|
|
119
120
|
/** Messages typed while a turn or command runs; sent one at a time once it ends, moved back to the editor on abort or failure. */
|
|
120
121
|
queued = [];
|
|
122
|
+
/** Images attached to drafts (dropped paths, ctrl+v); messages carry the ones whose `[Image #n]` markers they contain. */
|
|
123
|
+
images = new ImageAttachments();
|
|
124
|
+
replacingPaths = false;
|
|
121
125
|
queueView = new QueuedList(() => this.queued.map((q) => q.text));
|
|
122
126
|
turnCost = 0;
|
|
123
127
|
lastTurnCost = 0;
|
|
@@ -157,6 +161,7 @@ class InteractiveApp {
|
|
|
157
161
|
this.editor = new Editor(this.tui, editorTheme);
|
|
158
162
|
this.editor.borderColor = (border) => (this.shellMode() ? shellBorder(border) : editorTheme.borderColor(border));
|
|
159
163
|
this.editor.onSubmit = (text) => this.onSubmit(text);
|
|
164
|
+
this.editor.onChange = (text) => this.onEditorChange(text);
|
|
160
165
|
this.prompter = createApprovalPrompter({ show: (box) => this.showApproval(box) }, { now: this.now, redact: (text) => this.settings.redact(text) });
|
|
161
166
|
const root = new SafeContainer();
|
|
162
167
|
root.addChild(this.chat);
|
|
@@ -394,6 +399,12 @@ class InteractiveApp {
|
|
|
394
399
|
this.tui.requestRender(true);
|
|
395
400
|
return { consume: true };
|
|
396
401
|
}
|
|
402
|
+
if (matchesKey(data, "ctrl+v") && this.screen === "chat" && this.session) {
|
|
403
|
+
if (isKeyRepeat(data))
|
|
404
|
+
return { consume: true };
|
|
405
|
+
this.pasteClipboardImage();
|
|
406
|
+
return { consume: true };
|
|
407
|
+
}
|
|
397
408
|
if (matchesKey(data, "ctrl+d") && this.screen === "chat" && this.session && !this.running && this.editor.getText() === "") {
|
|
398
409
|
this.requestExit();
|
|
399
410
|
return { consume: true };
|
|
@@ -407,6 +418,42 @@ class InteractiveApp {
|
|
|
407
418
|
}
|
|
408
419
|
return undefined;
|
|
409
420
|
}
|
|
421
|
+
/** A file dragged into the terminal arrives as its path: an image path becomes an `[Image #n]` attachment. */
|
|
422
|
+
onEditorChange(text) {
|
|
423
|
+
if (this.replacingPaths || !/\.(png|jpe?g|gif|webp)\b/i.test(text))
|
|
424
|
+
return;
|
|
425
|
+
// Replace in the expanded text: setText drops the editor's collapsed-paste markers, so their content is kept inline.
|
|
426
|
+
const replaced = this.images.replaceDroppedPaths(this.editor.getExpandedText());
|
|
427
|
+
if (!replaced)
|
|
428
|
+
return;
|
|
429
|
+
if (replaced.attached.length > 0) {
|
|
430
|
+
this.replacingPaths = true;
|
|
431
|
+
try {
|
|
432
|
+
this.editor.setText(replaced.text);
|
|
433
|
+
}
|
|
434
|
+
finally {
|
|
435
|
+
this.replacingPaths = false;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
for (const path of replaced.failed)
|
|
439
|
+
this.say(`could not attach ${path}`);
|
|
440
|
+
this.tui.requestRender();
|
|
441
|
+
}
|
|
442
|
+
/** ctrl+v: attaches the image on the clipboard (a screenshot) at the cursor. */
|
|
443
|
+
pasteClipboardImage() {
|
|
444
|
+
const bytes = readClipboardImage();
|
|
445
|
+
if (!bytes) {
|
|
446
|
+
this.say(process.platform === "darwin" || process.platform === "linux" ? "no image on the clipboard" : "pasting images from the clipboard is not supported here; drag the file in instead");
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
try {
|
|
450
|
+
this.editor.insertTextAtCursor(this.images.attachBytes(bytes, "clipboard image"));
|
|
451
|
+
}
|
|
452
|
+
catch (error) {
|
|
453
|
+
this.say(`could not attach the clipboard image: ${error instanceof Error ? error.message : String(error)}`);
|
|
454
|
+
}
|
|
455
|
+
this.tui.requestRender();
|
|
456
|
+
}
|
|
410
457
|
onSubmit(raw) {
|
|
411
458
|
const text = raw.trim();
|
|
412
459
|
if (!text || !this.session)
|
|
@@ -426,7 +473,8 @@ class InteractiveApp {
|
|
|
426
473
|
if (this.running) {
|
|
427
474
|
// Plain text goes into the running turn at its next step; `!` commands and anything typed during a command wait.
|
|
428
475
|
// A skill command waits for its own turn: steering would drop the skill pin.
|
|
429
|
-
const
|
|
476
|
+
const { images, labels } = this.images.resolve(text);
|
|
477
|
+
const steered = !this.runningShell && !text.startsWith("!") && !text.startsWith("/") && this.session.steer(withImageLabels(text, labels), images);
|
|
430
478
|
this.queued.push({ text, steered });
|
|
431
479
|
this.say(`queued (${this.queued.length}) · ${steered ? "sent to the agent at its next step" : `sent when the ${this.runningShell ? "command" : "turn"} ends`}`);
|
|
432
480
|
return;
|
|
@@ -526,7 +574,8 @@ class InteractiveApp {
|
|
|
526
574
|
this.startTicker();
|
|
527
575
|
let result = null;
|
|
528
576
|
try {
|
|
529
|
-
|
|
577
|
+
const { images, labels } = this.images.resolve(text);
|
|
578
|
+
result = await session.turn(withImageLabels(text, labels), { signal: abort.signal, onEvent: (event) => this.onEvent(event), ...(images.length > 0 ? { images } : {}) });
|
|
530
579
|
}
|
|
531
580
|
catch (error) {
|
|
532
581
|
this.sayError(toErrorPayload(error));
|
package/tui/src/commands.js
CHANGED
|
@@ -50,6 +50,6 @@ export function helpText() {
|
|
|
50
50
|
const width = Math.max(...SLASH_COMMANDS.map((c) => `/${c.name}${c.argumentHint ? ` ${c.argumentHint}` : ""}`.length));
|
|
51
51
|
const lines = SLASH_COMMANDS.map((c) => `${`/${c.name}${c.argumentHint ? ` ${c.argumentHint}` : ""}`.padEnd(width)} ${c.description ?? ""}`);
|
|
52
52
|
lines.push("", `${"!<command>".padEnd(width)} Run a shell command; its output goes to the model with your next message`);
|
|
53
|
-
lines.push("", "enter send · shift+enter newline · ↑↓ history · ctrl+o expand tool output · esc/ctrl-c stop a turn · esc twice clears the draft (↑ recalls it) · ctrl-c twice or ctrl-d quit");
|
|
53
|
+
lines.push("", "enter send · shift+enter newline · ↑↓ history · ctrl+o expand tool output · drag an image in or ctrl+v to attach it · esc/ctrl-c stop a turn · esc twice clears the draft (↑ recalls it) · ctrl-c twice or ctrl-d quit");
|
|
54
54
|
return lines.join("\n");
|
|
55
55
|
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { basename, extname } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { describeImage, IMAGE_EXTENSIONS, withAttachedImagesNote, isImageFile, loadImageFile, prepareImage } from "../../tools/src/index.js";
|
|
6
|
+
/**
|
|
7
|
+
* A path token starting at a word boundary: single- or double-quoted, or unquoted with backslash-escaped spaces, the
|
|
8
|
+
* forms terminals use when a file is dragged in (iTerm and Terminal escape, Ghostty and VS Code quote).
|
|
9
|
+
*/
|
|
10
|
+
const PATH_TOKEN = /(^|\s)('[^'\n]+'|"[^"\n]+"|(?:\\.|[^\s\\'"])+)/g;
|
|
11
|
+
function unquote(token) {
|
|
12
|
+
if ((token.startsWith("'") && token.endsWith("'")) || (token.startsWith('"') && token.endsWith('"')))
|
|
13
|
+
return token.slice(1, -1);
|
|
14
|
+
return token.replace(/\\(.)/g, "$1");
|
|
15
|
+
}
|
|
16
|
+
/** The absolute file path a token names, or null: only absolute, `~/` and `file://` paths count, so typed words never do. */
|
|
17
|
+
function tokenPath(token) {
|
|
18
|
+
const raw = unquote(token);
|
|
19
|
+
let path;
|
|
20
|
+
if (raw.startsWith("file://")) {
|
|
21
|
+
try {
|
|
22
|
+
path = fileURLToPath(raw);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
else if (raw.startsWith("~/"))
|
|
29
|
+
path = `${homedir()}${raw.slice(1)}`;
|
|
30
|
+
else if (raw.startsWith("/") || /^[A-Za-z]:[\\/]/.test(raw))
|
|
31
|
+
path = raw;
|
|
32
|
+
else
|
|
33
|
+
return null;
|
|
34
|
+
return IMAGE_EXTENSIONS.includes(extname(path).toLowerCase()) ? path : null;
|
|
35
|
+
}
|
|
36
|
+
/** Image files named by dropped or pasted paths in `text`, in order. Paths to missing or non-image files are left out. */
|
|
37
|
+
export function findDroppedImagePaths(text, isImage = isImageFile) {
|
|
38
|
+
const out = [];
|
|
39
|
+
for (const match of text.matchAll(PATH_TOKEN)) {
|
|
40
|
+
const token = match[2];
|
|
41
|
+
const path = tokenPath(token);
|
|
42
|
+
if (!path || !isImage(path))
|
|
43
|
+
continue;
|
|
44
|
+
const start = (match.index ?? 0) + match[1].length;
|
|
45
|
+
out.push({ start, end: start + token.length, path });
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
const MARKER = /\[Image #(\d+)\]/g;
|
|
50
|
+
export const imageMarker = (n) => `[Image #${n}]`;
|
|
51
|
+
/**
|
|
52
|
+
* Images attached to the draft. Each one gets a `[Image #n]` marker in the editor; a submitted message carries the
|
|
53
|
+
* images whose markers it still contains, in the order they appear, so deleting a marker drops its image. Attachments
|
|
54
|
+
* live as long as the TUI, so a message recalled from history or put back from the queue keeps its images.
|
|
55
|
+
*/
|
|
56
|
+
export class ImageAttachments {
|
|
57
|
+
byId = new Map();
|
|
58
|
+
next = 1;
|
|
59
|
+
add(loaded, name) {
|
|
60
|
+
const id = this.next++;
|
|
61
|
+
this.byId.set(id, { image: loaded.image, label: describeImage(name, loaded) });
|
|
62
|
+
return imageMarker(id);
|
|
63
|
+
}
|
|
64
|
+
/** Attaches an image file and returns its marker. Throws for an unreadable or non-image file. */
|
|
65
|
+
attachFile(path) {
|
|
66
|
+
return this.add(loadImageFile(path), basename(path));
|
|
67
|
+
}
|
|
68
|
+
/** Attaches image bytes (a clipboard image) and returns its marker. */
|
|
69
|
+
attachBytes(bytes, name) {
|
|
70
|
+
return this.add(prepareImage(bytes), name);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Replaces every dropped image path in `text` with its marker. Returns null when there is none, so the editor is
|
|
74
|
+
* left alone; `failed` lists paths that looked like images but could not be read.
|
|
75
|
+
*/
|
|
76
|
+
replaceDroppedPaths(text) {
|
|
77
|
+
const dropped = findDroppedImagePaths(text);
|
|
78
|
+
if (dropped.length === 0)
|
|
79
|
+
return null;
|
|
80
|
+
let out = "";
|
|
81
|
+
let at = 0;
|
|
82
|
+
const attached = [];
|
|
83
|
+
const failed = [];
|
|
84
|
+
for (const d of dropped) {
|
|
85
|
+
out += text.slice(at, d.start);
|
|
86
|
+
try {
|
|
87
|
+
const marker = this.attachFile(d.path);
|
|
88
|
+
out += marker;
|
|
89
|
+
attached.push(marker);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
out += text.slice(d.start, d.end);
|
|
93
|
+
failed.push(d.path);
|
|
94
|
+
}
|
|
95
|
+
at = d.end;
|
|
96
|
+
}
|
|
97
|
+
out += text.slice(at);
|
|
98
|
+
return attached.length > 0 || failed.length > 0 ? { text: out, attached, failed } : null;
|
|
99
|
+
}
|
|
100
|
+
/** The images a message's markers refer to, in order (a repeated marker counts once), with a line naming each. */
|
|
101
|
+
resolve(text) {
|
|
102
|
+
const images = [];
|
|
103
|
+
const labels = [];
|
|
104
|
+
const seen = new Set();
|
|
105
|
+
for (const match of text.matchAll(MARKER)) {
|
|
106
|
+
const id = Number(match[1]);
|
|
107
|
+
const attachment = this.byId.get(id);
|
|
108
|
+
if (!attachment || seen.has(id))
|
|
109
|
+
continue;
|
|
110
|
+
seen.add(id);
|
|
111
|
+
images.push(attachment.image);
|
|
112
|
+
labels.push(`${imageMarker(id)} ${attachment.label}`);
|
|
113
|
+
}
|
|
114
|
+
return { images, labels };
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/** The message sent for a draft with images: the text as typed, plus a line saying which image each marker is. */
|
|
118
|
+
export const withImageLabels = withAttachedImagesNote;
|
|
119
|
+
/**
|
|
120
|
+
* The image on the system clipboard as PNG bytes, or null when there is none or no clipboard tool is available:
|
|
121
|
+
* AppleScript on macOS, wl-paste or xclip on Linux.
|
|
122
|
+
*/
|
|
123
|
+
export function readClipboardImage() {
|
|
124
|
+
const run = (command, args) => {
|
|
125
|
+
try {
|
|
126
|
+
const out = execFileSync(command, args, { stdio: ["ignore", "pipe", "ignore"], maxBuffer: 200 * 1024 * 1024, timeout: 10_000 });
|
|
127
|
+
return out.length > 0 ? out : null;
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
if (process.platform === "darwin") {
|
|
134
|
+
const out = run("osascript", ["-e", "try", "-e", "the clipboard as «class PNGf»", "-e", "end try"]);
|
|
135
|
+
const hex = out?.toString("utf8").match(/«data PNGf([0-9A-Fa-f]+)»/)?.[1];
|
|
136
|
+
return hex ? Buffer.from(hex, "hex") : null;
|
|
137
|
+
}
|
|
138
|
+
if (process.platform === "linux") {
|
|
139
|
+
return run("wl-paste", ["--no-newline", "--type", "image/png"]) ?? run("xclip", ["-selection", "clipboard", "-t", "image/png", "-o"]);
|
|
140
|
+
}
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Span, SpanNode } from "./api-types.ts";
|
|
2
2
|
import { statusBadge } from "./common-view.ts";
|
|
3
3
|
import { durationMs, formatDuration, formatUsd, shortId } from "./format.ts";
|
|
4
|
+
import { filePath } from "./files-view.ts";
|
|
4
5
|
import { html, prettyJson, type SafeHtml } from "./html.ts";
|
|
5
6
|
|
|
6
7
|
export function findSpan(roots: SpanNode[], id: string): SpanNode | undefined {
|
|
@@ -55,7 +56,22 @@ function renderContentBlocks(content: unknown): SafeHtml {
|
|
|
55
56
|
})}`;
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
const SHOWN_ATTRS = new Set(["content", "args", "result", "reason", "trigger", "instruction", "errorMessage", "firstChunkMs", "firstTextMs"]);
|
|
59
|
+
const SHOWN_ATTRS = new Set(["content", "args", "result", "reason", "trigger", "instruction", "errorMessage", "firstChunkMs", "firstTextMs", "images"]);
|
|
60
|
+
|
|
61
|
+
/** Paths of the images a span saved in its run folder (`attrs.images`): a run's attached inputs or a tool's returned images. */
|
|
62
|
+
export function spanImages(span: Span): string[] {
|
|
63
|
+
const images = span.attrs.images;
|
|
64
|
+
return Array.isArray(images) ? images.filter((p): p is string => typeof p === "string") : [];
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Thumbnails of images in a run folder; each opens full size in a new tab. */
|
|
68
|
+
export function renderImageStrip(runId: string, paths: readonly string[], label: string): SafeHtml {
|
|
69
|
+
if (paths.length === 0) return html``;
|
|
70
|
+
return html`<div class="msg msg-images"><div class="msg-label">${label}</div><div class="image-strip">${paths.map((p) => {
|
|
71
|
+
const src = filePath(runId, p);
|
|
72
|
+
return html`<a href="${src}" target="_blank" rel="noopener noreferrer" title="${p}"><img class="thumb" alt="${p}" src="${src}" loading="lazy"></a>`;
|
|
73
|
+
})}</div></div>`;
|
|
74
|
+
}
|
|
59
75
|
|
|
60
76
|
/** Fields, messages, tool input and output, and remaining attributes of one span. */
|
|
61
77
|
export function renderSpanDetail(span: Span, attrsTruncated: boolean, nowMs: number): SafeHtml {
|
|
@@ -87,6 +103,7 @@ export function renderSpanDetail(span: Span, attrsTruncated: boolean, nowMs: num
|
|
|
87
103
|
${a.content !== undefined ? renderContentBlocks(a.content) : ""}
|
|
88
104
|
${a.args !== undefined ? html`<div class="msg msg-tool"><div class="msg-label">input</div><pre>${prettyJson(a.args)}</pre></div>` : ""}
|
|
89
105
|
${typeof a.result === "string" ? html`<div class="msg msg-result"><div class="msg-label">output</div><pre>${a.result}</pre></div>` : ""}
|
|
106
|
+
${renderImageStrip(span.runId, spanImages(span), span.kind === "run" ? "attached images" : "images returned")}
|
|
90
107
|
${span.payloadRef ? html`<p class="note">Full payload stored in ${fileButton(span.runId, span.payloadRef, span.payloadRef)}</p>` : ""}
|
|
91
108
|
${Object.keys(rest).length ? html`<details><summary>attributes</summary><pre>${prettyJson(rest)}</pre></details>` : ""}
|
|
92
109
|
</div>`;
|
package/web/public/pages/run.ts
CHANGED
|
@@ -14,12 +14,14 @@ import { html, type SafeHtml } from "../lib/html.ts";
|
|
|
14
14
|
import { createPoller } from "../lib/poller.ts";
|
|
15
15
|
import { renderChildren, renderFeedbackForm, renderInjectedMemories, renderRunHeader, shouldPoll } from "../lib/run-view.ts";
|
|
16
16
|
import { renderEscalations, renderScores } from "../lib/score-view.ts";
|
|
17
|
-
import { findSpan, renderSpanDetail, renderSpanTree } from "../lib/span-view.ts";
|
|
17
|
+
import { findSpan, renderImageStrip, renderSpanDetail, renderSpanTree, spanImages } from "../lib/span-view.ts";
|
|
18
18
|
import { renderTimeline, timelineRows } from "../lib/timeline.ts";
|
|
19
19
|
import { asErrorBody, el, getJson, getText, postJson } from "./api.ts";
|
|
20
20
|
|
|
21
21
|
const runId = new URLSearchParams(location.search).get("id") ?? "";
|
|
22
22
|
const header = el<HTMLElement>("#header");
|
|
23
|
+
const imagesSection = el<HTMLElement>("#sec-images");
|
|
24
|
+
const inputImages = el<HTMLElement>("#input-images");
|
|
23
25
|
const review = el<HTMLElement>("#review");
|
|
24
26
|
const scores = el<HTMLElement>("#scores");
|
|
25
27
|
const escalations = el<HTMLElement>("#escalations");
|
|
@@ -144,6 +146,11 @@ async function load(): Promise<boolean> {
|
|
|
144
146
|
setHtml(injected, renderInjectedMemories(run.injectedMemories, run.run.id));
|
|
145
147
|
setHtml(children, renderChildren(run.children, now, readOnly ?? null, run.childFeedback));
|
|
146
148
|
roots = spans.roots;
|
|
149
|
+
// The run span holds the images sent with the instruction.
|
|
150
|
+
const attached = roots.find((r) => r.span.kind === "run")?.span;
|
|
151
|
+
const attachedPaths = attached ? spanImages(attached) : [];
|
|
152
|
+
imagesSection.hidden = attachedPaths.length === 0;
|
|
153
|
+
setHtml(inputImages, renderImageStrip(runId, attachedPaths, `${attachedPaths.length} sent with the instruction`));
|
|
147
154
|
setHtml(timeline, renderTimeline(timelineRows(roots, now), selectedSpanId));
|
|
148
155
|
setHtml(tree, html`${spans.truncated ? html`<p class="warn">This run has ${spans.total} spans; only the earliest are shown.</p>` : ""}${renderSpanTree(roots, now, 2, selectedSpanId)}`);
|
|
149
156
|
restoreState(state);
|
package/web/public/run.html
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
<nav class="run-nav" aria-label="Run sections"><a href="#sec-timeline">Timeline</a><a href="#sec-spans">Spans</a><a href="#sec-scores">Scores</a><a href="#sec-memories">Memories</a><a href="#sec-subagents">Subagents</a><a href="#sec-files">Files</a><a href="#sec-review">Review</a></nav>
|
|
18
18
|
<div id="error"></div>
|
|
19
19
|
<div id="header"></div>
|
|
20
|
+
<section id="sec-images" hidden><h2>Attached images</h2><div id="input-images"></div></section>
|
|
20
21
|
<section id="sec-timeline"><h2>Timeline</h2><div id="timeline"></div></section>
|
|
21
22
|
<section class="split" id="sec-spans">
|
|
22
23
|
<div><h2>Spans</h2><div id="tree"></div></div>
|
package/web/public/style.css
CHANGED
|
@@ -203,6 +203,8 @@ pre.diff span { display: block; white-space: pre-wrap; padding: 0 12px; }
|
|
|
203
203
|
.diff-meta { color: var(--muted); }
|
|
204
204
|
.media { display: block; max-width: 100%; max-height: 480px; margin: 4px 0; background: #000; }
|
|
205
205
|
audio.media { background: none; }
|
|
206
|
+
.image-strip { display: flex; flex-wrap: wrap; gap: 8px; margin: 4px 0; }
|
|
207
|
+
.image-strip .thumb { display: block; max-width: 240px; max-height: 180px; border: 1px solid var(--line); border-radius: 2px; background: var(--panel); object-fit: contain; }
|
|
206
208
|
|
|
207
209
|
/* Scores and sources */
|
|
208
210
|
.score-pass, .crit-pass, .esc-applied strong { color: var(--ok); }
|