@hexdecimal16/nexo 0.4.13 → 0.4.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +3 -0
  2. package/cli/src/common.js +1 -1
  3. package/cli/src/main.js +17 -2
  4. package/core/src/engine/context-trim.js +39 -0
  5. package/core/src/engine/errors.js +9 -0
  6. package/core/src/engine/messages.js +14 -1
  7. package/core/src/engine/run.js +39 -8
  8. package/core/src/providers/claude-p.js +75 -19
  9. package/core/src/providers/codex-p.js +16 -3
  10. package/core/src/providers/kiro-p.js +34 -7
  11. package/core/src/session/mentions.js +32 -8
  12. package/core/src/session/session.js +17 -13
  13. package/package.json +1 -1
  14. package/sdk/src/index.js +2 -0
  15. package/store/src/migrations/010_memory_project_scope.sql +27 -0
  16. package/store/src/run-dir.js +10 -0
  17. package/tools/src/builtin.js +39 -8
  18. package/tools/src/images.js +174 -0
  19. package/tools/src/index.js +1 -0
  20. package/tui/src/app.js +51 -2
  21. package/tui/src/commands.js +1 -1
  22. package/tui/src/image-attachments.js +142 -0
  23. package/web/public/area.html +2 -2
  24. package/web/public/cost.html +1 -1
  25. package/web/public/index.html +6 -23
  26. package/web/public/judge.html +1 -1
  27. package/web/public/learning.html +1 -1
  28. package/web/public/lib/common-view.ts +1 -1
  29. package/web/public/lib/run-view.ts +1 -1
  30. package/web/public/lib/runs-view.ts +1 -1
  31. package/web/public/lib/span-view.ts +18 -1
  32. package/web/public/lib/theme.ts +1 -1
  33. package/web/public/optim.html +1 -1
  34. package/web/public/page.html +2 -2
  35. package/web/public/pages/area.ts +3 -3
  36. package/web/public/pages/companion.ts +1 -1
  37. package/web/public/pages/home.ts +104 -21
  38. package/web/public/pages/page.ts +1 -1
  39. package/web/public/pages/run.ts +8 -1
  40. package/web/public/run.html +2 -1
  41. package/web/public/runs.html +40 -0
  42. package/web/public/sessions.html +1 -1
  43. package/web/public/style.css +60 -9
  44. package/web/public/wikis.html +1 -1
  45. package/web/src/companion.js +2 -2
  46. package/web/public/home.html +0 -23
@@ -251,16 +251,17 @@ export class Session {
251
251
  /** Messages steered before the run's agent exists; handed to it once it does. */
252
252
  steerPending = [];
253
253
  steerCtl = null;
254
- /** Sends a message into the running turn, injected before the agent's next model call. False when no turn runs. */
255
- steer(text) {
256
- if (!this.busy || !text.trim())
254
+ /** Sends a message (and any images) into the running turn, injected before the agent's next model call. False when no turn runs. */
255
+ steer(text, images = []) {
256
+ if (!this.busy || (!text.trim() && images.length === 0))
257
257
  return false;
258
258
  // Same `@file` attachment as a typed turn, so the agent gets the contents instead of reading the file itself.
259
- const sent = expandMentions(text, this.state.cwd).text;
259
+ const expanded = expandMentions(text, this.state.cwd);
260
+ const all = [...expanded.images, ...images];
260
261
  if (this.steerCtl)
261
- this.steerCtl.steer(sent);
262
+ this.steerCtl.steer(expanded.text, all);
262
263
  else
263
- this.steerPending.push(sent);
264
+ this.steerPending.push({ text: expanded.text, images: all });
264
265
  return true;
265
266
  }
266
267
  /** Drops steered messages the running turn has not taken yet. */
@@ -274,7 +275,7 @@ export class Session {
274
275
  * when the session cannot be saved.
275
276
  */
276
277
  async turn(instruction, options = {}) {
277
- if (!instruction.trim())
278
+ if (!instruction.trim() && !options.images?.length)
278
279
  throw invalid("The message is empty");
279
280
  if (this.state.status === "closed")
280
281
  throw invalid(`Session ${this.id} is closed`, "Resume it with `nexo --resume`");
@@ -305,12 +306,15 @@ export class Session {
305
306
  // `/impeccable init` pins the skill like `nexo run --skill`, so SDK sessions and the TUI read it the same way.
306
307
  const invoked = options.skill ? null : parseSkillInvocation(instruction, this.deps.skills.skills.keys());
307
308
  const skill = options.skill ?? invoked?.skill;
308
- const sent = withShellNotes(this.state.pendingShell ?? [], expandMentions(invoked?.instruction ?? instruction, this.state.cwd).text);
309
+ const expanded = expandMentions(invoked?.instruction ?? instruction, this.state.cwd);
310
+ const sent = withShellNotes(this.state.pendingShell ?? [], expanded.text);
311
+ const images = [...expanded.images, ...(options.images ?? [])];
309
312
  const workspace = this.state.workspace;
310
313
  const previous = workspace === null ? latestSessionRunWorkspace(this.deps.db, this.id) : undefined;
311
314
  const result = await executeRun(this.env, {
312
315
  request: {
313
316
  instruction: sent,
317
+ ...(images.length > 0 ? { images } : {}),
314
318
  history: buildHistory(this.lines, this.env.now().getTime()),
315
319
  prepareHistory: this.compactionHook(turnIndex, limit.usd),
316
320
  sessionId: this.id,
@@ -328,8 +332,8 @@ export class Session {
328
332
  allowAll: this.allowAll,
329
333
  onSteeringReady: (control) => {
330
334
  this.steerCtl = control;
331
- for (const text of this.steerPending.splice(0))
332
- control.steer(text);
335
+ for (const pending of this.steerPending.splice(0))
336
+ control.steer(pending.text, pending.images);
333
337
  },
334
338
  },
335
339
  budget: new RunBudget(limit.usd),
@@ -344,7 +348,7 @@ export class Session {
344
348
  }),
345
349
  });
346
350
  if (result.trajectoryId !== null)
347
- this.recordTurn(turnIndex, instruction, sent, result);
351
+ this.recordTurn(turnIndex, instruction, sent, result, images);
348
352
  this.learner.kick();
349
353
  return result;
350
354
  }
@@ -552,9 +556,9 @@ export class Session {
552
556
  };
553
557
  }
554
558
  /** `instruction` is what the user typed; `sent` is the message the run started from (with any `!` command notes). */
555
- recordTurn(turnIndex, instruction, sent, result) {
559
+ recordTurn(turnIndex, instruction, sent, result, images = []) {
556
560
  // A run that ended before its first request returns no messages; the message is kept so it is not lost.
557
- const messages = result.messages.length > 0 ? result.messages : [{ role: "user", content: [{ type: "text", text: sent }], timestamp: this.env.now().getTime() }];
561
+ const messages = result.messages.length > 0 ? result.messages : [{ role: "user", content: [{ type: "text", text: sent }, ...images], timestamp: this.env.now().getTime() }];
558
562
  this.appendLines(messages.map((message) => ({ type: "message", turnIndex, message })));
559
563
  this.state.turns = turnIndex;
560
564
  this.state.costUsd += result.costUsd;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hexdecimal16/nexo",
3
- "version": "0.4.13",
3
+ "version": "0.4.15",
4
4
  "description": "nexo agent engine",
5
5
  "license": "PolyForm-Noncommercial-1.0.0",
6
6
  "repository": {
package/sdk/src/index.js CHANGED
@@ -3,5 +3,7 @@ export { SETTING_GROUPS, settingGroupLabel } from "../../core/src/index.js";
3
3
  export { parseSkillInvocation } from "../../skills/src/index.js";
4
4
  export { allowAllPrompter, nonInteractivePrompter } from "../../tools/src/index.js";
5
5
  export { createRuntime, openLogger } from "./runtime.js";
6
+ /** Image helpers for `RunRequest.images`, `session.turn(text, { images })` and `session.steer(text, images)`. */
7
+ export { describeImage, isImageFile, loadImageFile, MAX_IMAGE_BYTES, MAX_IMAGE_EDGE, prepareImage, withAttachedImagesNote } from "../../tools/src/index.js";
6
8
  export { NexoError } from "../../telemetry/src/index.js";
7
9
  export { trackRunTree } from "./run-tree.js";
@@ -0,0 +1,27 @@
1
+ -- nexo:foreign-keys-off
2
+ -- Project-scoped memories (scope 'project:<hash>', from projectScope) were accepted by the store but rejected by the
3
+ -- CHECK from 006, so every curator delta holding one failed as a whole. SQLite cannot widen a CHECK in place, so the
4
+ -- table is rebuilt. Rowids are copied explicitly because memories_fts is keyed by the memories rowid.
5
+ -- memory_embeddings references memories(id), so the runner switches foreign keys off and runs foreign_key_check.
6
+ CREATE TABLE memories_new (
7
+ id TEXT PRIMARY KEY,
8
+ scope TEXT NOT NULL CHECK (scope IN ('user', 'global') OR (scope GLOB 'skill:?*') OR (scope GLOB 'project:?*')),
9
+ type TEXT NOT NULL CHECK (type IN ('fact', 'preference', 'lesson', 'workflow')),
10
+ content TEXT NOT NULL,
11
+ importance REAL NOT NULL CHECK (importance >= 0 AND importance <= 1),
12
+ source TEXT NOT NULL CHECK (source IN ('ace', 'human', 'user')),
13
+ source_run_id TEXT,
14
+ superseded_by TEXT REFERENCES memories(id),
15
+ retired_at TEXT,
16
+ created_at TEXT NOT NULL,
17
+ last_used_at TEXT
18
+ );
19
+ INSERT INTO memories_new (rowid, id, scope, type, content, importance, source, source_run_id, superseded_by, retired_at, created_at, last_used_at)
20
+ SELECT rowid, id, scope, type, content, importance, source, source_run_id, superseded_by, retired_at, created_at, last_used_at FROM memories;
21
+ DROP TABLE memories;
22
+ ALTER TABLE memories_new RENAME TO memories;
23
+ CREATE INDEX memories_live ON memories(scope) WHERE superseded_by IS NULL AND retired_at IS NULL;
24
+ CREATE INDEX memories_source_run ON memories(source_run_id);
25
+ CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories BEGIN
26
+ INSERT INTO memories_fts (rowid, content) VALUES (new.rowid, new.content);
27
+ END;
@@ -102,6 +102,16 @@ export class RunDir {
102
102
  this.writeConfined(this.toolOutputsDir, `${spanId}.txt`, content);
103
103
  return `tool_outputs/${spanId}.txt`;
104
104
  }
105
+ /**
106
+ * Saves a base64 image the run saw (an attached input, a screenshot a tool returned) and returns its path relative to
107
+ * the run directory, e.g. `inputs/image-1.png`. Image bytes are not redacted.
108
+ */
109
+ writeImage(relPath, base64) {
110
+ const target = confine(this.root, relPath);
111
+ mkdirSync(dirname(target), { recursive: true });
112
+ writeFileSync(target, Buffer.from(base64, "base64"));
113
+ return relPath;
114
+ }
105
115
  resolveFile(relPath) {
106
116
  return confine(this.root, relPath);
107
117
  }
@@ -6,6 +6,7 @@ import { NexoError } from "../../telemetry/src/index.js";
6
6
  import { Type } from "typebox";
7
7
  import { canonicalPath, nexoHomeGuard, resolveConfined, SECRET_PATH } from "./confine.js";
8
8
  import { htmlToText, looksLikeHtml } from "./html-text.js";
9
+ import { describeImage, isImageFile, loadImageFile, MAX_IMAGE_EDGE } from "./images.js";
9
10
  export const MAX_BASH_OUTPUT_BYTES = 1_000_000;
10
11
  const MAX_READ_LINES = 2000;
11
12
  const MAX_FETCH_CHARS = 100_000;
@@ -177,39 +178,69 @@ export function createBuiltinTools(ctx) {
177
178
  }
178
179
  throw new Error(`File not found: ${path}`);
179
180
  };
180
- const readOne = (path, offset, limit) => {
181
+ /** A read path made absolute: saved tool outputs, or a file anywhere outside the secret-file rule. */
182
+ const locateRead = (path) => {
181
183
  const outputs = ctx.toolOutputsDir;
182
184
  const savedOutput = outputs !== undefined && path.startsWith(TOOL_OUTPUT_PREFIX);
183
185
  const abs = savedOutput ? resolveSavedOutput(outputs, path) : resolveReadPath(path);
184
186
  guard?.checkRead(abs);
185
187
  if (!existsSync(abs))
186
188
  throw new Error(`File not found: ${path}`);
189
+ return abs;
190
+ };
191
+ const readText = (abs, offset, limit) => {
187
192
  const lines = readFileSync(abs, "utf8").split("\n");
188
193
  const start = (offset ?? 1) - 1;
189
194
  return lines.slice(start, start + (limit ?? MAX_READ_LINES)).join("\n");
190
195
  };
196
+ /** A file's content blocks: its lines as text, or for an image a line describing it and the image itself. */
197
+ const readOne = (path, offset, limit) => {
198
+ const abs = locateRead(path);
199
+ if (isImageFile(abs)) {
200
+ const loaded = loadImageFile(abs);
201
+ return [{ type: "text", text: `Image ${describeImage(path, loaded)}:` }, loaded.image];
202
+ }
203
+ return [{ type: "text", text: readText(abs, offset, limit) }];
204
+ };
191
205
  const read = {
192
206
  name: "read",
193
207
  label: "Read file",
194
- description: `Read text files. Relative paths start at the workspace; absolute paths are read where they are, except key, token and credential files (use bash for those). Pass paths to read several files in one call. Paths starting with ${TOOL_OUTPUT_PREFIX} open saved large tool outputs.`,
208
+ description: `Read text files and images (PNG, JPEG, GIF, WebP; large images are scaled down to ${MAX_IMAGE_EDGE} px). Relative paths start at the workspace; absolute paths are read where they are, except key, token and credential files (use bash for those). Pass paths to read several files in one call. Paths starting with ${TOOL_OUTPUT_PREFIX} open saved large tool outputs.`,
195
209
  parameters: ReadParams,
196
210
  execute: async (_id, params) => {
197
211
  if (params.paths !== undefined) {
198
212
  if (params.path !== undefined)
199
213
  throw new Error("Give either path or paths, not both");
200
- const parts = params.paths.map((p) => {
214
+ const content = [];
215
+ let failures = 0;
216
+ for (const p of params.paths) {
201
217
  try {
202
- return { failed: false, text: `==> ${p.path} <==\n${readOne(p.path, p.offset, p.limit)}` };
218
+ const blocks = readOne(p.path, p.offset, p.limit);
219
+ const first = blocks[0];
220
+ if (first?.type === "text")
221
+ content.push({ type: "text", text: `${content.length > 0 ? "\n" : ""}==> ${p.path} <==\n${first.text}` }, ...blocks.slice(1));
222
+ else
223
+ content.push({ type: "text", text: `==> ${p.path} <==` }, ...blocks);
203
224
  }
204
225
  catch (error) {
205
- return { failed: true, text: `==> ${p.path} <==\n[error] ${error instanceof Error ? error.message : String(error)}` };
226
+ failures++;
227
+ content.push({ type: "text", text: `${content.length > 0 ? "\n" : ""}==> ${p.path} <==\n[error] ${error instanceof Error ? error.message : String(error)}` });
206
228
  }
207
- });
208
- return textResult(parts.map((p) => p.text).join("\n\n"), parts.every((p) => p.failed));
229
+ }
230
+ // Adjacent text blocks are joined so text-only reads keep their single-block shape.
231
+ const merged = [];
232
+ for (const block of content) {
233
+ const last = merged.at(-1);
234
+ if (block.type === "text" && last?.type === "text")
235
+ last.text += `\n${block.text}`;
236
+ else
237
+ merged.push(block.type === "text" ? { ...block } : block);
238
+ }
239
+ return { content: merged, details: undefined, ...(failures === params.paths.length ? { isError: true } : {}) };
209
240
  }
210
241
  if (params.path === undefined)
211
242
  throw new Error("Give path or paths");
212
- return textResult(readOne(params.path, params.offset, params.limit));
243
+ return { content: readOne(params.path, params.offset, params.limit), details: undefined };
213
244
  },
214
245
  };
215
246
  const write = {
@@ -0,0 +1,174 @@
1
+ import { execFileSync } from "node:child_process";
2
+ import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
3
+ import { tmpdir } from "node:os";
4
+ import { basename, join } from "node:path";
5
+ /** File extensions treated as images when a path is dropped or pasted (the bytes still decide). */
6
+ export const IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".webp"];
7
+ /**
8
+ * Long-edge limit images are scaled down to before they reach a model. Claude's high-resolution tier stops at 2,576 px
9
+ * and many-image requests at 3,000 px; 2,000 px keeps text legible while staying under every provider's limit.
10
+ */
11
+ export const MAX_IMAGE_EDGE = 2000;
12
+ /** Largest image sent as is, in raw bytes (about 4.7 MB base64): under Bedrock's 5 MB base64 per-image limit. */
13
+ export const MAX_IMAGE_BYTES = 3_500_000;
14
+ /** Images larger than this are refused outright rather than read into memory. */
15
+ const MAX_SOURCE_BYTES = 50 * 1024 * 1024;
16
+ /** The image type from the file's first bytes, or null when they are not a supported image. */
17
+ export function sniffImageMime(bytes) {
18
+ const b = bytes;
19
+ if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47)
20
+ return "image/png";
21
+ if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff)
22
+ return "image/jpeg";
23
+ if (b.length >= 6 && b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x38)
24
+ return "image/gif";
25
+ if (b.length >= 12 && ascii(b, 0, 4) === "RIFF" && ascii(b, 8, 12) === "WEBP")
26
+ return "image/webp";
27
+ return null;
28
+ }
29
+ function ascii(b, from, to) {
30
+ return String.fromCharCode(...b.subarray(from, to));
31
+ }
32
+ /** Pixel size read from the image header; null when the header is not understood. */
33
+ export function imageDimensions(bytes, mime) {
34
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
35
+ try {
36
+ if (mime === "image/png")
37
+ return { width: view.getUint32(16), height: view.getUint32(20) };
38
+ if (mime === "image/gif")
39
+ return { width: view.getUint16(6, true), height: view.getUint16(8, true) };
40
+ if (mime === "image/webp") {
41
+ const chunk = ascii(bytes, 12, 16);
42
+ if (chunk === "VP8X")
43
+ return { width: 1 + (view.getUint32(24, true) & 0xffffff), height: 1 + (view.getUint32(27, true) & 0xffffff) };
44
+ if (chunk === "VP8L") {
45
+ const bits = view.getUint32(21, true);
46
+ return { width: 1 + (bits & 0x3fff), height: 1 + ((bits >> 14) & 0x3fff) };
47
+ }
48
+ if (chunk === "VP8 ")
49
+ return { width: view.getUint16(26, true) & 0x3fff, height: view.getUint16(28, true) & 0x3fff };
50
+ return null;
51
+ }
52
+ // JPEG: walk the segments to the first start-of-frame marker.
53
+ let offset = 2;
54
+ while (offset + 9 < bytes.length) {
55
+ if (bytes[offset] !== 0xff)
56
+ return null;
57
+ const marker = bytes[offset + 1];
58
+ if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
59
+ return { width: view.getUint16(offset + 7), height: view.getUint16(offset + 5) };
60
+ }
61
+ offset += 2 + view.getUint16(offset + 2);
62
+ }
63
+ return null;
64
+ }
65
+ catch {
66
+ return null;
67
+ }
68
+ }
69
+ function hasCommand(command) {
70
+ try {
71
+ execFileSync(process.platform === "win32" ? "where" : "which", [command], { stdio: "ignore" });
72
+ return true;
73
+ }
74
+ catch {
75
+ return false;
76
+ }
77
+ }
78
+ /**
79
+ * Scales an image down with the platform's own tools (`sips` on macOS, ImageMagick elsewhere) so its long edge is at
80
+ * most `maxEdge`, as JPEG when the source is not a PNG or when PNG stays too large. Null when no tool is available or
81
+ * the tool fails; the caller then sends the original.
82
+ */
83
+ function downscale(bytes, mime, maxEdge, maxBytes) {
84
+ const dir = mkdtempSync(join(tmpdir(), "nexo-image-"));
85
+ try {
86
+ const src = join(dir, `in.${mime.slice(6)}`);
87
+ writeFileSync(src, bytes);
88
+ const attempt = (format) => {
89
+ const out = join(dir, `out.${format === "png" ? "png" : "jpg"}`);
90
+ if (process.platform === "darwin" && hasCommand("sips")) {
91
+ execFileSync("sips", ["-Z", String(maxEdge), "-s", "format", format, ...(format === "jpeg" ? ["-s", "formatOptions", "85"] : []), src, "--out", out], { stdio: "ignore", timeout: 30_000 });
92
+ }
93
+ else if (hasCommand("magick") || hasCommand("convert")) {
94
+ const tool = hasCommand("magick") ? "magick" : "convert";
95
+ execFileSync(tool, [`${src}[0]`, "-resize", `${maxEdge}x${maxEdge}>`, ...(format === "jpeg" ? ["-quality", "85"] : []), out], { stdio: "ignore", timeout: 30_000 });
96
+ }
97
+ else
98
+ return null;
99
+ return readFileSync(out);
100
+ };
101
+ // PNG keeps screenshots and text crisp; fall back to JPEG when PNG is still too large.
102
+ const png = mime === "image/png" ? attempt("png") : null;
103
+ if (png && png.length <= maxBytes)
104
+ return { bytes: png, mime: "image/png" };
105
+ const jpeg = attempt("jpeg");
106
+ return jpeg ? { bytes: jpeg, mime: "image/jpeg" } : null;
107
+ }
108
+ catch {
109
+ return null;
110
+ }
111
+ finally {
112
+ rmSync(dir, { recursive: true, force: true });
113
+ }
114
+ }
115
+ /**
116
+ * Prepares image bytes for a model: checks the type by content, and scales the image down when its long edge or size
117
+ * passes the limits. Throws when the bytes are not a PNG, JPEG, GIF or WebP image.
118
+ */
119
+ export function prepareImage(bytes, options = {}) {
120
+ const mime = sniffImageMime(bytes);
121
+ if (!mime)
122
+ throw new Error("Not a supported image (PNG, JPEG, GIF or WebP)");
123
+ const maxEdge = options.maxEdge ?? MAX_IMAGE_EDGE;
124
+ const maxBytes = options.maxBytes ?? MAX_IMAGE_BYTES;
125
+ const size = imageDimensions(bytes, mime);
126
+ const tooLarge = (size !== null && Math.max(size.width, size.height) > maxEdge) || bytes.length > maxBytes;
127
+ // Animated GIFs would lose their frames; send them as they are.
128
+ const scaled = tooLarge && mime !== "image/gif" ? downscale(bytes, mime, maxEdge, maxBytes) : null;
129
+ if (scaled) {
130
+ const after = imageDimensions(scaled.bytes, scaled.mime);
131
+ return {
132
+ image: { type: "image", data: scaled.bytes.toString("base64"), mimeType: scaled.mime },
133
+ width: after?.width ?? null,
134
+ height: after?.height ?? null,
135
+ ...(size ? { original: { ...size, bytes: bytes.length } } : {}),
136
+ };
137
+ }
138
+ return { image: { type: "image", data: Buffer.from(bytes).toString("base64"), mimeType: mime }, width: size?.width ?? null, height: size?.height ?? null };
139
+ }
140
+ /** True when the file starts with image bytes. False for missing files, directories and anything else. */
141
+ export function isImageFile(path) {
142
+ try {
143
+ if (!statSync(path).isFile())
144
+ return false;
145
+ const head = readFileSync(path).subarray(0, 16);
146
+ return sniffImageMime(head) !== null;
147
+ }
148
+ catch {
149
+ return false;
150
+ }
151
+ }
152
+ /** Reads an image file and prepares it for a model (see prepareImage). Throws for a missing, huge or non-image file. */
153
+ export function loadImageFile(path, options = {}) {
154
+ const stat = statSync(path);
155
+ if (!stat.isFile())
156
+ throw new Error(`${path} is not a file`);
157
+ if (stat.size > MAX_SOURCE_BYTES)
158
+ throw new Error(`${basename(path)} is ${(stat.size / 1024 / 1024).toFixed(1)} MB; images over 50 MB are not read`);
159
+ return prepareImage(readFileSync(path), options);
160
+ }
161
+ /** One-line description of a loaded image for the text next to it, e.g. `shot.png (1456×819 PNG, scaled from 3840×2160)`. */
162
+ export function describeImage(name, loaded) {
163
+ const kind = loaded.image.mimeType.slice(6).toUpperCase();
164
+ const dims = loaded.width !== null && loaded.height !== null ? `${loaded.width}×${loaded.height} ` : "";
165
+ const scaled = loaded.original ? `, scaled from ${loaded.original.width}×${loaded.original.height}` : "";
166
+ return `${name} (${dims}${kind}${scaled})`;
167
+ }
168
+ /**
169
+ * The line added to a message that carries images, naming each in order (e.g. `shot.png (1456×819 PNG)`), so the model
170
+ * can tell which image the user means. Empty labels give the text unchanged.
171
+ */
172
+ export function withAttachedImagesNote(text, labels) {
173
+ return labels.length === 0 ? text : `${text}\n\n[nexo] Attached images, in order: ${labels.join("; ")}.`;
174
+ }
@@ -10,3 +10,4 @@ export * from "./session-rules.js";
10
10
  export * from "./web-search.js";
11
11
  export * from "./wiki-page.js";
12
12
  export * from "./home-folders.js";
13
+ export * from "./images.js";
package/tui/src/app.js CHANGED
@@ -9,6 +9,7 @@ import { createApprovalPrompter } from "./approvals.js";
9
9
  import { helpText, isSkillCommand, parseSlashCommand, SLASH_COMMANDS, skillSlashCommands } from "./commands.js";
10
10
  import { Block, DynamicLine, QueuedList, SafeContainer, SafeMarkdown, SubagentLine, SubagentPanel, Ticker, ToolLine, ViewMode } from "./components.js";
11
11
  import { createAutocompleteProvider } from "./file-autocomplete.js";
12
+ import { ImageAttachments, readClipboardImage, withImageLabels } from "./image-attachments.js";
12
13
  import { createSessionPickerScreen } from "./session-picker.js";
13
14
  import { SettingsScreen } from "./settings-screen.js";
14
15
  import { escapeControl, escapeControlKeepLines, formatError, oneLine, toolSummary, usd } from "./text.js";
@@ -118,6 +119,9 @@ class InteractiveApp {
118
119
  runningShell = false;
119
120
  /** Messages typed while a turn or command runs; sent one at a time once it ends, moved back to the editor on abort or failure. */
120
121
  queued = [];
122
+ /** Images attached to drafts (dropped paths, ctrl+v); messages carry the ones whose `[Image #n]` markers they contain. */
123
+ images = new ImageAttachments();
124
+ replacingPaths = false;
121
125
  queueView = new QueuedList(() => this.queued.map((q) => q.text));
122
126
  turnCost = 0;
123
127
  lastTurnCost = 0;
@@ -157,6 +161,7 @@ class InteractiveApp {
157
161
  this.editor = new Editor(this.tui, editorTheme);
158
162
  this.editor.borderColor = (border) => (this.shellMode() ? shellBorder(border) : editorTheme.borderColor(border));
159
163
  this.editor.onSubmit = (text) => this.onSubmit(text);
164
+ this.editor.onChange = (text) => this.onEditorChange(text);
160
165
  this.prompter = createApprovalPrompter({ show: (box) => this.showApproval(box) }, { now: this.now, redact: (text) => this.settings.redact(text) });
161
166
  const root = new SafeContainer();
162
167
  root.addChild(this.chat);
@@ -394,6 +399,12 @@ class InteractiveApp {
394
399
  this.tui.requestRender(true);
395
400
  return { consume: true };
396
401
  }
402
+ if (matchesKey(data, "ctrl+v") && this.screen === "chat" && this.session) {
403
+ if (isKeyRepeat(data))
404
+ return { consume: true };
405
+ this.pasteClipboardImage();
406
+ return { consume: true };
407
+ }
397
408
  if (matchesKey(data, "ctrl+d") && this.screen === "chat" && this.session && !this.running && this.editor.getText() === "") {
398
409
  this.requestExit();
399
410
  return { consume: true };
@@ -407,6 +418,42 @@ class InteractiveApp {
407
418
  }
408
419
  return undefined;
409
420
  }
421
+ /** A file dragged into the terminal arrives as its path: an image path becomes an `[Image #n]` attachment. */
422
+ onEditorChange(text) {
423
+ if (this.replacingPaths || !/\.(png|jpe?g|gif|webp)\b/i.test(text))
424
+ return;
425
+ // Replace in the expanded text: setText drops the editor's collapsed-paste markers, so their content is kept inline.
426
+ const replaced = this.images.replaceDroppedPaths(this.editor.getExpandedText());
427
+ if (!replaced)
428
+ return;
429
+ if (replaced.attached.length > 0) {
430
+ this.replacingPaths = true;
431
+ try {
432
+ this.editor.setText(replaced.text);
433
+ }
434
+ finally {
435
+ this.replacingPaths = false;
436
+ }
437
+ }
438
+ for (const path of replaced.failed)
439
+ this.say(`could not attach ${path}`);
440
+ this.tui.requestRender();
441
+ }
442
+ /** ctrl+v: attaches the image on the clipboard (a screenshot) at the cursor. */
443
+ pasteClipboardImage() {
444
+ const bytes = readClipboardImage();
445
+ if (!bytes) {
446
+ this.say(process.platform === "darwin" || process.platform === "linux" ? "no image on the clipboard" : "pasting images from the clipboard is not supported here; drag the file in instead");
447
+ return;
448
+ }
449
+ try {
450
+ this.editor.insertTextAtCursor(this.images.attachBytes(bytes, "clipboard image"));
451
+ }
452
+ catch (error) {
453
+ this.say(`could not attach the clipboard image: ${error instanceof Error ? error.message : String(error)}`);
454
+ }
455
+ this.tui.requestRender();
456
+ }
410
457
  onSubmit(raw) {
411
458
  const text = raw.trim();
412
459
  if (!text || !this.session)
@@ -426,7 +473,8 @@ class InteractiveApp {
426
473
  if (this.running) {
427
474
  // Plain text goes into the running turn at its next step; `!` commands and anything typed during a command wait.
428
475
  // A skill command waits for its own turn: steering would drop the skill pin.
429
- const steered = !this.runningShell && !text.startsWith("!") && !text.startsWith("/") && this.session.steer(text);
476
+ const { images, labels } = this.images.resolve(text);
477
+ const steered = !this.runningShell && !text.startsWith("!") && !text.startsWith("/") && this.session.steer(withImageLabels(text, labels), images);
430
478
  this.queued.push({ text, steered });
431
479
  this.say(`queued (${this.queued.length}) · ${steered ? "sent to the agent at its next step" : `sent when the ${this.runningShell ? "command" : "turn"} ends`}`);
432
480
  return;
@@ -526,7 +574,8 @@ class InteractiveApp {
526
574
  this.startTicker();
527
575
  let result = null;
528
576
  try {
529
- result = await session.turn(text, { signal: abort.signal, onEvent: (event) => this.onEvent(event) });
577
+ const { images, labels } = this.images.resolve(text);
578
+ result = await session.turn(withImageLabels(text, labels), { signal: abort.signal, onEvent: (event) => this.onEvent(event), ...(images.length > 0 ? { images } : {}) });
530
579
  }
531
580
  catch (error) {
532
581
  this.sayError(toErrorPayload(error));
@@ -50,6 +50,6 @@ export function helpText() {
50
50
  const width = Math.max(...SLASH_COMMANDS.map((c) => `/${c.name}${c.argumentHint ? ` ${c.argumentHint}` : ""}`.length));
51
51
  const lines = SLASH_COMMANDS.map((c) => `${`/${c.name}${c.argumentHint ? ` ${c.argumentHint}` : ""}`.padEnd(width)} ${c.description ?? ""}`);
52
52
  lines.push("", `${"!<command>".padEnd(width)} Run a shell command; its output goes to the model with your next message`);
53
- lines.push("", "enter send · shift+enter newline · ↑↓ history · ctrl+o expand tool output · esc/ctrl-c stop a turn · esc twice clears the draft (↑ recalls it) · ctrl-c twice or ctrl-d quit");
53
+ lines.push("", "enter send · shift+enter newline · ↑↓ history · ctrl+o expand tool output · drag an image in or ctrl+v to attach it · esc/ctrl-c stop a turn · esc twice clears the draft (↑ recalls it) · ctrl-c twice or ctrl-d quit");
54
54
  return lines.join("\n");
55
55
  }