@hexdecimal16/nexo 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/cli/src/common.js +1 -1
- package/cli/src/main.js +17 -2
- package/core/src/config/schema.js +3 -1
- package/core/src/engine/context-trim.js +39 -0
- package/core/src/engine/errors.js +10 -2
- package/core/src/engine/messages.js +14 -1
- package/core/src/engine/provider-gate.js +5 -3
- package/core/src/engine/run.js +105 -14
- package/core/src/providers/claude-p.js +129 -23
- package/core/src/providers/codex-p.js +16 -3
- package/core/src/providers/kiro-p.js +115 -21
- package/core/src/providers/reply-text.js +110 -0
- package/core/src/session/mentions.js +32 -8
- package/core/src/session/session.js +17 -13
- package/core/src/settings/config-edit.js +6 -0
- package/package.json +1 -1
- package/sdk/src/index.js +2 -0
- package/store/src/run-dir.js +10 -0
- package/tools/src/builtin.js +39 -8
- package/tools/src/images.js +174 -0
- package/tools/src/index.js +1 -0
- package/tui/src/app.js +71 -5
- package/tui/src/commands.js +1 -1
- package/tui/src/components.js +73 -0
- package/tui/src/image-attachments.js +142 -0
- package/web/public/lib/span-view.ts +21 -1
- package/web/public/pages/run.ts +8 -1
- package/web/public/run.html +1 -0
- package/web/public/style.css +2 -0
package/tui/src/app.js
CHANGED
|
@@ -7,8 +7,9 @@ import { toErrorPayload } from "../../telemetry/src/index.js";
|
|
|
7
7
|
import { describeSessionRule } from "../../tools/src/index.js";
|
|
8
8
|
import { createApprovalPrompter } from "./approvals.js";
|
|
9
9
|
import { helpText, isSkillCommand, parseSlashCommand, SLASH_COMMANDS, skillSlashCommands } from "./commands.js";
|
|
10
|
-
import { Block, DynamicLine, QueuedList, SafeContainer, SafeMarkdown, SubagentLine, Ticker, ToolLine, ViewMode } from "./components.js";
|
|
10
|
+
import { Block, DynamicLine, QueuedList, SafeContainer, SafeMarkdown, SubagentLine, SubagentPanel, Ticker, ToolLine, ViewMode } from "./components.js";
|
|
11
11
|
import { createAutocompleteProvider } from "./file-autocomplete.js";
|
|
12
|
+
import { ImageAttachments, readClipboardImage, withImageLabels } from "./image-attachments.js";
|
|
12
13
|
import { createSessionPickerScreen } from "./session-picker.js";
|
|
13
14
|
import { SettingsScreen } from "./settings-screen.js";
|
|
14
15
|
import { escapeControl, escapeControlKeepLines, formatError, oneLine, toolSummary, usd } from "./text.js";
|
|
@@ -102,6 +103,8 @@ class InteractiveApp {
|
|
|
102
103
|
tui;
|
|
103
104
|
chat = new Container();
|
|
104
105
|
status = new Container();
|
|
106
|
+
/** Running subagents, pinned between the chat and the loader so their animation never happens above the viewport. */
|
|
107
|
+
subagentPanel = new SubagentPanel(() => Math.floor(this.terminal.rows / 2));
|
|
105
108
|
slot = new Container();
|
|
106
109
|
editor;
|
|
107
110
|
ticker = new Ticker();
|
|
@@ -116,6 +119,9 @@ class InteractiveApp {
|
|
|
116
119
|
runningShell = false;
|
|
117
120
|
/** Messages typed while a turn or command runs; sent one at a time once it ends, moved back to the editor on abort or failure. */
|
|
118
121
|
queued = [];
|
|
122
|
+
/** Images attached to drafts (dropped paths, ctrl+v); messages carry the ones whose `[Image #n]` markers they contain. */
|
|
123
|
+
images = new ImageAttachments();
|
|
124
|
+
replacingPaths = false;
|
|
119
125
|
queueView = new QueuedList(() => this.queued.map((q) => q.text));
|
|
120
126
|
turnCost = 0;
|
|
121
127
|
lastTurnCost = 0;
|
|
@@ -155,9 +161,11 @@ class InteractiveApp {
|
|
|
155
161
|
this.editor = new Editor(this.tui, editorTheme);
|
|
156
162
|
this.editor.borderColor = (border) => (this.shellMode() ? shellBorder(border) : editorTheme.borderColor(border));
|
|
157
163
|
this.editor.onSubmit = (text) => this.onSubmit(text);
|
|
164
|
+
this.editor.onChange = (text) => this.onEditorChange(text);
|
|
158
165
|
this.prompter = createApprovalPrompter({ show: (box) => this.showApproval(box) }, { now: this.now, redact: (text) => this.settings.redact(text) });
|
|
159
166
|
const root = new SafeContainer();
|
|
160
167
|
root.addChild(this.chat);
|
|
168
|
+
root.addChild(this.subagentPanel);
|
|
161
169
|
root.addChild(this.status);
|
|
162
170
|
root.addChild(this.queueView);
|
|
163
171
|
root.addChild(this.slot);
|
|
@@ -391,6 +399,12 @@ class InteractiveApp {
|
|
|
391
399
|
this.tui.requestRender(true);
|
|
392
400
|
return { consume: true };
|
|
393
401
|
}
|
|
402
|
+
if (matchesKey(data, "ctrl+v") && this.screen === "chat" && this.session) {
|
|
403
|
+
if (isKeyRepeat(data))
|
|
404
|
+
return { consume: true };
|
|
405
|
+
this.pasteClipboardImage();
|
|
406
|
+
return { consume: true };
|
|
407
|
+
}
|
|
394
408
|
if (matchesKey(data, "ctrl+d") && this.screen === "chat" && this.session && !this.running && this.editor.getText() === "") {
|
|
395
409
|
this.requestExit();
|
|
396
410
|
return { consume: true };
|
|
@@ -404,6 +418,42 @@ class InteractiveApp {
|
|
|
404
418
|
}
|
|
405
419
|
return undefined;
|
|
406
420
|
}
|
|
421
|
+
/** A file dragged into the terminal arrives as its path: an image path becomes an `[Image #n]` attachment. */
|
|
422
|
+
onEditorChange(text) {
|
|
423
|
+
if (this.replacingPaths || !/\.(png|jpe?g|gif|webp)\b/i.test(text))
|
|
424
|
+
return;
|
|
425
|
+
// Replace in the expanded text: setText drops the editor's collapsed-paste markers, so their content is kept inline.
|
|
426
|
+
const replaced = this.images.replaceDroppedPaths(this.editor.getExpandedText());
|
|
427
|
+
if (!replaced)
|
|
428
|
+
return;
|
|
429
|
+
if (replaced.attached.length > 0) {
|
|
430
|
+
this.replacingPaths = true;
|
|
431
|
+
try {
|
|
432
|
+
this.editor.setText(replaced.text);
|
|
433
|
+
}
|
|
434
|
+
finally {
|
|
435
|
+
this.replacingPaths = false;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
for (const path of replaced.failed)
|
|
439
|
+
this.say(`could not attach ${path}`);
|
|
440
|
+
this.tui.requestRender();
|
|
441
|
+
}
|
|
442
|
+
/** ctrl+v: attaches the image on the clipboard (a screenshot) at the cursor. */
|
|
443
|
+
pasteClipboardImage() {
|
|
444
|
+
const bytes = readClipboardImage();
|
|
445
|
+
if (!bytes) {
|
|
446
|
+
this.say(process.platform === "darwin" || process.platform === "linux" ? "no image on the clipboard" : "pasting images from the clipboard is not supported here; drag the file in instead");
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
try {
|
|
450
|
+
this.editor.insertTextAtCursor(this.images.attachBytes(bytes, "clipboard image"));
|
|
451
|
+
}
|
|
452
|
+
catch (error) {
|
|
453
|
+
this.say(`could not attach the clipboard image: ${error instanceof Error ? error.message : String(error)}`);
|
|
454
|
+
}
|
|
455
|
+
this.tui.requestRender();
|
|
456
|
+
}
|
|
407
457
|
onSubmit(raw) {
|
|
408
458
|
const text = raw.trim();
|
|
409
459
|
if (!text || !this.session)
|
|
@@ -423,7 +473,8 @@ class InteractiveApp {
|
|
|
423
473
|
if (this.running) {
|
|
424
474
|
// Plain text goes into the running turn at its next step; `!` commands and anything typed during a command wait.
|
|
425
475
|
// A skill command waits for its own turn: steering would drop the skill pin.
|
|
426
|
-
const
|
|
476
|
+
const { images, labels } = this.images.resolve(text);
|
|
477
|
+
const steered = !this.runningShell && !text.startsWith("!") && !text.startsWith("/") && this.session.steer(withImageLabels(text, labels), images);
|
|
427
478
|
this.queued.push({ text, steered });
|
|
428
479
|
this.say(`queued (${this.queued.length}) · ${steered ? "sent to the agent at its next step" : `sent when the ${this.runningShell ? "command" : "turn"} ends`}`);
|
|
429
480
|
return;
|
|
@@ -523,7 +574,8 @@ class InteractiveApp {
|
|
|
523
574
|
this.startTicker();
|
|
524
575
|
let result = null;
|
|
525
576
|
try {
|
|
526
|
-
|
|
577
|
+
const { images, labels } = this.images.resolve(text);
|
|
578
|
+
result = await session.turn(withImageLabels(text, labels), { signal: abort.signal, onEvent: (event) => this.onEvent(event), ...(images.length > 0 ? { images } : {}) });
|
|
527
579
|
}
|
|
528
580
|
catch (error) {
|
|
529
581
|
this.sayError(toErrorPayload(error));
|
|
@@ -537,6 +589,7 @@ class InteractiveApp {
|
|
|
537
589
|
line.interrupt();
|
|
538
590
|
for (const line of this.subagents.values())
|
|
539
591
|
line.interrupt();
|
|
592
|
+
this.flushSubagents();
|
|
540
593
|
this.currentText = null;
|
|
541
594
|
}
|
|
542
595
|
if (result) {
|
|
@@ -636,7 +689,7 @@ class InteractiveApp {
|
|
|
636
689
|
}
|
|
637
690
|
const line = new SubagentLine(escapeControl(oneLine(event.instruction, 60)), event.depth, this.ticker, escapeControl(`${event.taskId ?? event.runId.slice(-6)} · ${event.model}`));
|
|
638
691
|
this.subagents.set(event.runId, line);
|
|
639
|
-
this.
|
|
692
|
+
this.subagentPanel.add(line);
|
|
640
693
|
break;
|
|
641
694
|
}
|
|
642
695
|
case "run_end": {
|
|
@@ -644,6 +697,8 @@ class InteractiveApp {
|
|
|
644
697
|
return;
|
|
645
698
|
this.subagents.get(event.runId)?.finish(event.status, event.costUsd, event.error ? escapeControl(`${event.error.code}: ${oneLine(event.error.message, 120)}`) : "");
|
|
646
699
|
this.subagents.delete(event.runId);
|
|
700
|
+
if (!this.subagentPanel.anyRunning)
|
|
701
|
+
this.flushSubagents();
|
|
647
702
|
break;
|
|
648
703
|
}
|
|
649
704
|
case "llm_end":
|
|
@@ -662,7 +717,12 @@ class InteractiveApp {
|
|
|
662
717
|
break;
|
|
663
718
|
}
|
|
664
719
|
case "approval": {
|
|
665
|
-
|
|
720
|
+
// A subagent's approvals update its own row (latest reason only) so parallel subagents don't flood the chat.
|
|
721
|
+
if (event.depth > 0) {
|
|
722
|
+
this.subagents.get(event.runId)?.approved(escapeControl(oneLine(event.reason, 160)));
|
|
723
|
+
break;
|
|
724
|
+
}
|
|
725
|
+
this.chat.addChild(new Block(`auto-allowed: ${escapeControl(oneLine(event.reason, 160))}`, style.gray, " "));
|
|
666
726
|
break;
|
|
667
727
|
}
|
|
668
728
|
case "escalation_failed": {
|
|
@@ -705,6 +765,11 @@ class InteractiveApp {
|
|
|
705
765
|
}
|
|
706
766
|
this.tui.requestRender();
|
|
707
767
|
}
|
|
768
|
+
/** Moves the finished fan-out from the live panel into the chat, where its rows stay frozen. */
|
|
769
|
+
flushSubagents() {
|
|
770
|
+
for (const line of this.subagentPanel.takeFinished())
|
|
771
|
+
this.chat.addChild(line);
|
|
772
|
+
}
|
|
708
773
|
startTicker() {
|
|
709
774
|
this.stopTicker();
|
|
710
775
|
this.loader = new Loader(this.tui, style.cyan, style.gray, "working… (esc to stop)");
|
|
@@ -947,6 +1012,7 @@ class InteractiveApp {
|
|
|
947
1012
|
this.liveModel = null;
|
|
948
1013
|
this.tools.clear();
|
|
949
1014
|
this.subagents.clear();
|
|
1015
|
+
this.subagentPanel.clear();
|
|
950
1016
|
this.editor.setAutocompleteProvider(createAutocompleteProvider([...SLASH_COMMANDS, ...skillSlashCommands(next.skills())], next.cwd));
|
|
951
1017
|
this.chat.clear();
|
|
952
1018
|
this.renderHeader(next);
|
package/tui/src/commands.js
CHANGED
|
@@ -50,6 +50,6 @@ export function helpText() {
|
|
|
50
50
|
const width = Math.max(...SLASH_COMMANDS.map((c) => `/${c.name}${c.argumentHint ? ` ${c.argumentHint}` : ""}`.length));
|
|
51
51
|
const lines = SLASH_COMMANDS.map((c) => `${`/${c.name}${c.argumentHint ? ` ${c.argumentHint}` : ""}`.padEnd(width)} ${c.description ?? ""}`);
|
|
52
52
|
lines.push("", `${"!<command>".padEnd(width)} Run a shell command; its output goes to the model with your next message`);
|
|
53
|
-
lines.push("", "enter send · shift+enter newline · ↑↓ history · ctrl+o expand tool output · esc/ctrl-c stop a turn · esc twice clears the draft (↑ recalls it) · ctrl-c twice or ctrl-d quit");
|
|
53
|
+
lines.push("", "enter send · shift+enter newline · ↑↓ history · ctrl+o expand tool output · drag an image in or ctrl+v to attach it · esc/ctrl-c stop a turn · esc twice clears the draft (↑ recalls it) · ctrl-c twice or ctrl-d quit");
|
|
54
54
|
return lines.join("\n");
|
|
55
55
|
}
|
package/tui/src/components.js
CHANGED
|
@@ -147,6 +147,7 @@ export class SubagentLine {
|
|
|
147
147
|
status = "";
|
|
148
148
|
error = "";
|
|
149
149
|
tool = "";
|
|
150
|
+
approval = "";
|
|
150
151
|
toolCalls = 0;
|
|
151
152
|
endedAt = null;
|
|
152
153
|
startedAt = Date.now();
|
|
@@ -154,6 +155,8 @@ export class SubagentLine {
|
|
|
154
155
|
ticker;
|
|
155
156
|
depth;
|
|
156
157
|
label;
|
|
158
|
+
/** One line only (no current-tool, approval or error line), for when the live panel is short on rows. */
|
|
159
|
+
compact = false;
|
|
157
160
|
constructor(summary, depth, ticker, label) {
|
|
158
161
|
this.summary = summary;
|
|
159
162
|
this.depth = depth;
|
|
@@ -167,19 +170,28 @@ export class SubagentLine {
|
|
|
167
170
|
toolEnded() {
|
|
168
171
|
this.tool = "";
|
|
169
172
|
}
|
|
173
|
+
/** Keeps only the latest auto-approval reason; it replaces the previous one instead of piling up in the chat. */
|
|
174
|
+
approved(reason) {
|
|
175
|
+
this.approval = reason;
|
|
176
|
+
}
|
|
170
177
|
finish(status, costUsd, error = "") {
|
|
171
178
|
this.state = status === "succeeded" ? "ok" : "error";
|
|
172
179
|
this.status = status;
|
|
173
180
|
this.costUsd = costUsd;
|
|
174
181
|
this.error = error;
|
|
175
182
|
this.tool = "";
|
|
183
|
+
this.approval = "";
|
|
176
184
|
this.endedAt = Date.now();
|
|
177
185
|
}
|
|
186
|
+
get running() {
|
|
187
|
+
return this.state === "running";
|
|
188
|
+
}
|
|
178
189
|
interrupt() {
|
|
179
190
|
if (this.state === "running") {
|
|
180
191
|
this.state = "error";
|
|
181
192
|
this.status = "interrupted";
|
|
182
193
|
this.tool = "";
|
|
194
|
+
this.approval = "";
|
|
183
195
|
this.endedAt = Date.now();
|
|
184
196
|
}
|
|
185
197
|
}
|
|
@@ -190,14 +202,75 @@ export class SubagentLine {
|
|
|
190
202
|
const calls = this.toolCalls > 0 ? ` · ${this.toolCalls} tool${this.toolCalls === 1 ? "" : "s"}` : "";
|
|
191
203
|
const tail = this.state === "running" ? style.gray(` ${elapsed}${calls}`) : style.gray(` ${this.status} · ${elapsed}${calls} · ${usd(this.costUsd)}`);
|
|
192
204
|
const lines = [truncateToWidth(`${indent} ↳ ${mark} ${style.bold(this.label)} ${this.summary}${tail}`, width)];
|
|
205
|
+
if (this.compact)
|
|
206
|
+
return lines;
|
|
193
207
|
if (this.state === "running" && this.tool)
|
|
194
208
|
lines.push(truncateToWidth(style.gray(`${indent} ${this.tool}`), width));
|
|
209
|
+
if (this.state === "running" && this.approval)
|
|
210
|
+
lines.push(truncateToWidth(style.gray(`${indent} auto-allowed: ${this.approval}`), width));
|
|
195
211
|
if (this.state === "error" && this.error)
|
|
196
212
|
lines.push(truncateToWidth(style.red(`${indent} ${this.error}`), width));
|
|
197
213
|
return lines;
|
|
198
214
|
}
|
|
199
215
|
invalidate() { }
|
|
200
216
|
}
|
|
217
|
+
/**
|
|
218
|
+
* The subagents of the running fan-out, pinned below the chat. Their rows animate (spinner, elapsed time), and the
|
|
219
|
+
* main-screen renderer clears scrollback and redraws everything whenever a line above the viewport changes, so live rows
|
|
220
|
+
* must never sit in the chat where new output can push them off screen. Finished rows wait here until the whole wave is
|
|
221
|
+
* done and are then moved into the chat in one go (see `takeFinished`), where they never change again. The panel never
|
|
222
|
+
* renders more than `maxRows()` lines: first finished rows collapse into one line, then running rows go one line each,
|
|
223
|
+
* then the rest are counted.
|
|
224
|
+
*/
|
|
225
|
+
export class SubagentPanel {
|
|
226
|
+
lines = [];
|
|
227
|
+
maxRows;
|
|
228
|
+
constructor(maxRows) {
|
|
229
|
+
this.maxRows = maxRows;
|
|
230
|
+
}
|
|
231
|
+
add(line) {
|
|
232
|
+
this.lines.push(line);
|
|
233
|
+
}
|
|
234
|
+
get anyRunning() {
|
|
235
|
+
return this.lines.some((line) => line.running);
|
|
236
|
+
}
|
|
237
|
+
/** Removes and returns every row, in start order, expanded for the chat. */
|
|
238
|
+
takeFinished() {
|
|
239
|
+
const taken = this.lines.splice(0);
|
|
240
|
+
for (const line of taken)
|
|
241
|
+
line.compact = false;
|
|
242
|
+
return taken;
|
|
243
|
+
}
|
|
244
|
+
clear() {
|
|
245
|
+
this.lines.length = 0;
|
|
246
|
+
}
|
|
247
|
+
render(width) {
|
|
248
|
+
if (this.lines.length === 0)
|
|
249
|
+
return [];
|
|
250
|
+
const budget = Math.max(3, this.maxRows());
|
|
251
|
+
for (const line of this.lines)
|
|
252
|
+
line.compact = false;
|
|
253
|
+
const full = [""].concat(...this.lines.map((line) => line.render(width)));
|
|
254
|
+
if (full.length <= budget)
|
|
255
|
+
return full;
|
|
256
|
+
const running = this.lines.filter((line) => line.running);
|
|
257
|
+
const done = this.lines.length - running.length;
|
|
258
|
+
const out = [""];
|
|
259
|
+
if (done > 0)
|
|
260
|
+
out.push(truncateToWidth(style.gray(` ↳ ${done} subagent${done === 1 ? "" : "s"} finished`), width));
|
|
261
|
+
for (const line of running)
|
|
262
|
+
line.compact = true;
|
|
263
|
+
const room = budget - out.length;
|
|
264
|
+
const shown = running.length <= room ? running : running.slice(0, Math.max(0, room - 1));
|
|
265
|
+
for (const line of shown)
|
|
266
|
+
out.push(...line.render(width));
|
|
267
|
+
const hidden = running.length - shown.length;
|
|
268
|
+
if (hidden > 0)
|
|
269
|
+
out.push(truncateToWidth(style.gray(` ↳ … +${hidden} more running`), width));
|
|
270
|
+
return out;
|
|
271
|
+
}
|
|
272
|
+
invalidate() { }
|
|
273
|
+
}
|
|
201
274
|
/**
|
|
202
275
|
* Single-line secret entry. Shows one bullet per character and nothing else; the value is handed to `onSubmit` and
|
|
203
276
|
* dropped. It keeps no history, undo or kill ring.
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { basename, extname } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { describeImage, IMAGE_EXTENSIONS, withAttachedImagesNote, isImageFile, loadImageFile, prepareImage } from "../../tools/src/index.js";
|
|
6
|
+
/**
|
|
7
|
+
* A path token starting at a word boundary: single- or double-quoted, or unquoted with backslash-escaped spaces, the
|
|
8
|
+
* forms terminals use when a file is dragged in (iTerm and Terminal escape, Ghostty and VS Code quote).
|
|
9
|
+
*/
|
|
10
|
+
const PATH_TOKEN = /(^|\s)('[^'\n]+'|"[^"\n]+"|(?:\\.|[^\s\\'"])+)/g;
|
|
11
|
+
function unquote(token) {
|
|
12
|
+
if ((token.startsWith("'") && token.endsWith("'")) || (token.startsWith('"') && token.endsWith('"')))
|
|
13
|
+
return token.slice(1, -1);
|
|
14
|
+
return token.replace(/\\(.)/g, "$1");
|
|
15
|
+
}
|
|
16
|
+
/** The absolute file path a token names, or null: only absolute, `~/` and `file://` paths count, so typed words never do. */
|
|
17
|
+
function tokenPath(token) {
|
|
18
|
+
const raw = unquote(token);
|
|
19
|
+
let path;
|
|
20
|
+
if (raw.startsWith("file://")) {
|
|
21
|
+
try {
|
|
22
|
+
path = fileURLToPath(raw);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
else if (raw.startsWith("~/"))
|
|
29
|
+
path = `${homedir()}${raw.slice(1)}`;
|
|
30
|
+
else if (raw.startsWith("/") || /^[A-Za-z]:[\\/]/.test(raw))
|
|
31
|
+
path = raw;
|
|
32
|
+
else
|
|
33
|
+
return null;
|
|
34
|
+
return IMAGE_EXTENSIONS.includes(extname(path).toLowerCase()) ? path : null;
|
|
35
|
+
}
|
|
36
|
+
/** Image files named by dropped or pasted paths in `text`, in order. Paths to missing or non-image files are left out. */
|
|
37
|
+
export function findDroppedImagePaths(text, isImage = isImageFile) {
|
|
38
|
+
const out = [];
|
|
39
|
+
for (const match of text.matchAll(PATH_TOKEN)) {
|
|
40
|
+
const token = match[2];
|
|
41
|
+
const path = tokenPath(token);
|
|
42
|
+
if (!path || !isImage(path))
|
|
43
|
+
continue;
|
|
44
|
+
const start = (match.index ?? 0) + match[1].length;
|
|
45
|
+
out.push({ start, end: start + token.length, path });
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
const MARKER = /\[Image #(\d+)\]/g;
|
|
50
|
+
export const imageMarker = (n) => `[Image #${n}]`;
|
|
51
|
+
/**
|
|
52
|
+
* Images attached to the draft. Each one gets a `[Image #n]` marker in the editor; a submitted message carries the
|
|
53
|
+
* images whose markers it still contains, in the order they appear, so deleting a marker drops its image. Attachments
|
|
54
|
+
* live as long as the TUI, so a message recalled from history or put back from the queue keeps its images.
|
|
55
|
+
*/
|
|
56
|
+
export class ImageAttachments {
|
|
57
|
+
byId = new Map();
|
|
58
|
+
next = 1;
|
|
59
|
+
add(loaded, name) {
|
|
60
|
+
const id = this.next++;
|
|
61
|
+
this.byId.set(id, { image: loaded.image, label: describeImage(name, loaded) });
|
|
62
|
+
return imageMarker(id);
|
|
63
|
+
}
|
|
64
|
+
/** Attaches an image file and returns its marker. Throws for an unreadable or non-image file. */
|
|
65
|
+
attachFile(path) {
|
|
66
|
+
return this.add(loadImageFile(path), basename(path));
|
|
67
|
+
}
|
|
68
|
+
/** Attaches image bytes (a clipboard image) and returns its marker. */
|
|
69
|
+
attachBytes(bytes, name) {
|
|
70
|
+
return this.add(prepareImage(bytes), name);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Replaces every dropped image path in `text` with its marker. Returns null when there is none, so the editor is
|
|
74
|
+
* left alone; `failed` lists paths that looked like images but could not be read.
|
|
75
|
+
*/
|
|
76
|
+
replaceDroppedPaths(text) {
|
|
77
|
+
const dropped = findDroppedImagePaths(text);
|
|
78
|
+
if (dropped.length === 0)
|
|
79
|
+
return null;
|
|
80
|
+
let out = "";
|
|
81
|
+
let at = 0;
|
|
82
|
+
const attached = [];
|
|
83
|
+
const failed = [];
|
|
84
|
+
for (const d of dropped) {
|
|
85
|
+
out += text.slice(at, d.start);
|
|
86
|
+
try {
|
|
87
|
+
const marker = this.attachFile(d.path);
|
|
88
|
+
out += marker;
|
|
89
|
+
attached.push(marker);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
out += text.slice(d.start, d.end);
|
|
93
|
+
failed.push(d.path);
|
|
94
|
+
}
|
|
95
|
+
at = d.end;
|
|
96
|
+
}
|
|
97
|
+
out += text.slice(at);
|
|
98
|
+
return attached.length > 0 || failed.length > 0 ? { text: out, attached, failed } : null;
|
|
99
|
+
}
|
|
100
|
+
/** The images a message's markers refer to, in order (a repeated marker counts once), with a line naming each. */
|
|
101
|
+
resolve(text) {
|
|
102
|
+
const images = [];
|
|
103
|
+
const labels = [];
|
|
104
|
+
const seen = new Set();
|
|
105
|
+
for (const match of text.matchAll(MARKER)) {
|
|
106
|
+
const id = Number(match[1]);
|
|
107
|
+
const attachment = this.byId.get(id);
|
|
108
|
+
if (!attachment || seen.has(id))
|
|
109
|
+
continue;
|
|
110
|
+
seen.add(id);
|
|
111
|
+
images.push(attachment.image);
|
|
112
|
+
labels.push(`${imageMarker(id)} ${attachment.label}`);
|
|
113
|
+
}
|
|
114
|
+
return { images, labels };
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/** The message sent for a draft with images: the text as typed, plus a line saying which image each marker is. */
|
|
118
|
+
export const withImageLabels = withAttachedImagesNote;
|
|
119
|
+
/**
|
|
120
|
+
* The image on the system clipboard as PNG bytes, or null when there is none or no clipboard tool is available:
|
|
121
|
+
* AppleScript on macOS, wl-paste or xclip on Linux.
|
|
122
|
+
*/
|
|
123
|
+
export function readClipboardImage() {
|
|
124
|
+
const run = (command, args) => {
|
|
125
|
+
try {
|
|
126
|
+
const out = execFileSync(command, args, { stdio: ["ignore", "pipe", "ignore"], maxBuffer: 200 * 1024 * 1024, timeout: 10_000 });
|
|
127
|
+
return out.length > 0 ? out : null;
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
if (process.platform === "darwin") {
|
|
134
|
+
const out = run("osascript", ["-e", "try", "-e", "the clipboard as «class PNGf»", "-e", "end try"]);
|
|
135
|
+
const hex = out?.toString("utf8").match(/«data PNGf([0-9A-Fa-f]+)»/)?.[1];
|
|
136
|
+
return hex ? Buffer.from(hex, "hex") : null;
|
|
137
|
+
}
|
|
138
|
+
if (process.platform === "linux") {
|
|
139
|
+
return run("wl-paste", ["--no-newline", "--type", "image/png"]) ?? run("xclip", ["-selection", "clipboard", "-t", "image/png", "-o"]);
|
|
140
|
+
}
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Span, SpanNode } from "./api-types.ts";
|
|
2
2
|
import { statusBadge } from "./common-view.ts";
|
|
3
3
|
import { durationMs, formatDuration, formatUsd, shortId } from "./format.ts";
|
|
4
|
+
import { filePath } from "./files-view.ts";
|
|
4
5
|
import { html, prettyJson, type SafeHtml } from "./html.ts";
|
|
5
6
|
|
|
6
7
|
export function findSpan(roots: SpanNode[], id: string): SpanNode | undefined {
|
|
@@ -55,7 +56,22 @@ function renderContentBlocks(content: unknown): SafeHtml {
|
|
|
55
56
|
})}`;
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
const SHOWN_ATTRS = new Set(["content", "args", "result", "reason", "trigger", "instruction", "errorMessage"]);
|
|
59
|
+
const SHOWN_ATTRS = new Set(["content", "args", "result", "reason", "trigger", "instruction", "errorMessage", "firstChunkMs", "firstTextMs", "images"]);
|
|
60
|
+
|
|
61
|
+
/** Paths of the images a span saved in its run folder (`attrs.images`): a run's attached inputs or a tool's returned images. */
|
|
62
|
+
export function spanImages(span: Span): string[] {
|
|
63
|
+
const images = span.attrs.images;
|
|
64
|
+
return Array.isArray(images) ? images.filter((p): p is string => typeof p === "string") : [];
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Thumbnails of images in a run folder; each opens full size in a new tab. */
|
|
68
|
+
export function renderImageStrip(runId: string, paths: readonly string[], label: string): SafeHtml {
|
|
69
|
+
if (paths.length === 0) return html``;
|
|
70
|
+
return html`<div class="msg msg-images"><div class="msg-label">${label}</div><div class="image-strip">${paths.map((p) => {
|
|
71
|
+
const src = filePath(runId, p);
|
|
72
|
+
return html`<a href="${src}" target="_blank" rel="noopener noreferrer" title="${p}"><img class="thumb" alt="${p}" src="${src}" loading="lazy"></a>`;
|
|
73
|
+
})}</div></div>`;
|
|
74
|
+
}
|
|
59
75
|
|
|
60
76
|
/** Fields, messages, tool input and output, and remaining attributes of one span. */
|
|
61
77
|
export function renderSpanDetail(span: Span, attrsTruncated: boolean, nowMs: number): SafeHtml {
|
|
@@ -67,6 +83,9 @@ export function renderSpanDetail(span: Span, attrsTruncated: boolean, nowMs: num
|
|
|
67
83
|
["started", span.startedAt],
|
|
68
84
|
["duration", formatDuration(durationMs(span.startedAt, span.endedAt, nowMs)) + (span.endedAt === null ? " (running)" : "")],
|
|
69
85
|
["queued", span.queuedMs === null ? null : formatDuration(span.queuedMs)],
|
|
86
|
+
// A model call's wait for its first streamed output (thinking included) and for the first text of the answer.
|
|
87
|
+
["first output", typeof a.firstChunkMs === "number" ? formatDuration(a.firstChunkMs) : null],
|
|
88
|
+
["first text", typeof a.firstTextMs === "number" ? formatDuration(a.firstTextMs) : null],
|
|
70
89
|
["model", span.model ? `${span.provider ?? "?"}/${span.model}` : null],
|
|
71
90
|
["tier", span.tier],
|
|
72
91
|
["effort", span.effort],
|
|
@@ -84,6 +103,7 @@ export function renderSpanDetail(span: Span, attrsTruncated: boolean, nowMs: num
|
|
|
84
103
|
${a.content !== undefined ? renderContentBlocks(a.content) : ""}
|
|
85
104
|
${a.args !== undefined ? html`<div class="msg msg-tool"><div class="msg-label">input</div><pre>${prettyJson(a.args)}</pre></div>` : ""}
|
|
86
105
|
${typeof a.result === "string" ? html`<div class="msg msg-result"><div class="msg-label">output</div><pre>${a.result}</pre></div>` : ""}
|
|
106
|
+
${renderImageStrip(span.runId, spanImages(span), span.kind === "run" ? "attached images" : "images returned")}
|
|
87
107
|
${span.payloadRef ? html`<p class="note">Full payload stored in ${fileButton(span.runId, span.payloadRef, span.payloadRef)}</p>` : ""}
|
|
88
108
|
${Object.keys(rest).length ? html`<details><summary>attributes</summary><pre>${prettyJson(rest)}</pre></details>` : ""}
|
|
89
109
|
</div>`;
|
package/web/public/pages/run.ts
CHANGED
|
@@ -14,12 +14,14 @@ import { html, type SafeHtml } from "../lib/html.ts";
|
|
|
14
14
|
import { createPoller } from "../lib/poller.ts";
|
|
15
15
|
import { renderChildren, renderFeedbackForm, renderInjectedMemories, renderRunHeader, shouldPoll } from "../lib/run-view.ts";
|
|
16
16
|
import { renderEscalations, renderScores } from "../lib/score-view.ts";
|
|
17
|
-
import { findSpan, renderSpanDetail, renderSpanTree } from "../lib/span-view.ts";
|
|
17
|
+
import { findSpan, renderImageStrip, renderSpanDetail, renderSpanTree, spanImages } from "../lib/span-view.ts";
|
|
18
18
|
import { renderTimeline, timelineRows } from "../lib/timeline.ts";
|
|
19
19
|
import { asErrorBody, el, getJson, getText, postJson } from "./api.ts";
|
|
20
20
|
|
|
21
21
|
const runId = new URLSearchParams(location.search).get("id") ?? "";
|
|
22
22
|
const header = el<HTMLElement>("#header");
|
|
23
|
+
const imagesSection = el<HTMLElement>("#sec-images");
|
|
24
|
+
const inputImages = el<HTMLElement>("#input-images");
|
|
23
25
|
const review = el<HTMLElement>("#review");
|
|
24
26
|
const scores = el<HTMLElement>("#scores");
|
|
25
27
|
const escalations = el<HTMLElement>("#escalations");
|
|
@@ -144,6 +146,11 @@ async function load(): Promise<boolean> {
|
|
|
144
146
|
setHtml(injected, renderInjectedMemories(run.injectedMemories, run.run.id));
|
|
145
147
|
setHtml(children, renderChildren(run.children, now, readOnly ?? null, run.childFeedback));
|
|
146
148
|
roots = spans.roots;
|
|
149
|
+
// The run span holds the images sent with the instruction.
|
|
150
|
+
const attached = roots.find((r) => r.span.kind === "run")?.span;
|
|
151
|
+
const attachedPaths = attached ? spanImages(attached) : [];
|
|
152
|
+
imagesSection.hidden = attachedPaths.length === 0;
|
|
153
|
+
setHtml(inputImages, renderImageStrip(runId, attachedPaths, `${attachedPaths.length} sent with the instruction`));
|
|
147
154
|
setHtml(timeline, renderTimeline(timelineRows(roots, now), selectedSpanId));
|
|
148
155
|
setHtml(tree, html`${spans.truncated ? html`<p class="warn">This run has ${spans.total} spans; only the earliest are shown.</p>` : ""}${renderSpanTree(roots, now, 2, selectedSpanId)}`);
|
|
149
156
|
restoreState(state);
|
package/web/public/run.html
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
<nav class="run-nav" aria-label="Run sections"><a href="#sec-timeline">Timeline</a><a href="#sec-spans">Spans</a><a href="#sec-scores">Scores</a><a href="#sec-memories">Memories</a><a href="#sec-subagents">Subagents</a><a href="#sec-files">Files</a><a href="#sec-review">Review</a></nav>
|
|
18
18
|
<div id="error"></div>
|
|
19
19
|
<div id="header"></div>
|
|
20
|
+
<section id="sec-images" hidden><h2>Attached images</h2><div id="input-images"></div></section>
|
|
20
21
|
<section id="sec-timeline"><h2>Timeline</h2><div id="timeline"></div></section>
|
|
21
22
|
<section class="split" id="sec-spans">
|
|
22
23
|
<div><h2>Spans</h2><div id="tree"></div></div>
|
package/web/public/style.css
CHANGED
|
@@ -203,6 +203,8 @@ pre.diff span { display: block; white-space: pre-wrap; padding: 0 12px; }
|
|
|
203
203
|
.diff-meta { color: var(--muted); }
|
|
204
204
|
.media { display: block; max-width: 100%; max-height: 480px; margin: 4px 0; background: #000; }
|
|
205
205
|
audio.media { background: none; }
|
|
206
|
+
.image-strip { display: flex; flex-wrap: wrap; gap: 8px; margin: 4px 0; }
|
|
207
|
+
.image-strip .thumb { display: block; max-width: 240px; max-height: 180px; border: 1px solid var(--line); border-radius: 2px; background: var(--panel); object-fit: contain; }
|
|
206
208
|
|
|
207
209
|
/* Scores and sources */
|
|
208
210
|
.score-pass, .crit-pass, .esc-applied strong { color: var(--ok); }
|