@bismawy/pi-vision-watcher 1.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1161 @@
1
+ /**
2
+ * pi-vision-watcher — give text-only models vision by proxying image input
3
+ * through a vision-capable model of your choice.
4
+ *
5
+ * Extracted from the GLM 5.1 vision-handoff pipeline in pi-umans-provider and
6
+ * generalized: instead of a hardcoded describer, the user picks any
7
+ * vision-capable model from the registry via an interactive picker, and the
8
+ * choice is persisted to ~/.pi/agent/extensions/pi-vision-watcher.json.
9
+ *
10
+ * Pipeline (provider-agnostic via @earendil-works/pi-ai's complete()):
11
+ * before_agent_start → warm the description cache for attached images AND
12
+ * pasted clipboard image file paths found in the prompt text (pre-warm at
13
+ * paste-enter, concurrent with the agent's first response).
14
+ * optional async clipboard fallback → race matching read tool calls against a
15
+ * non-blocking steering-message injection; matching reads cancel delivery.
16
+ * tool_result (read) → loadDescription() each read-tool image and AWAIT the
17
+ * shared batch, so N parallel reads coalesce into ONE vision call. The
18
+ * descriptions land in the tool results before the agent's next turn.
19
+ * context → swap any remaining image blocks for their (now cached) text
20
+ * description on the cloned LLM-bound payload.
21
+ *
22
+ * This file is the wiring layer: pi event handlers + the /vision-watcher
23
+ * command. The dataloader (`src/dataloader.ts`), describer (`src/describer.ts`),
24
+ * image IO (`src/image.ts`), resource guards (`src/dispose.ts`), config/types
25
+ * (`src/index.ts`), and usage/energy (`src/usage.ts`) live in `src/`.
26
+ *
27
+ * Image blocks are detected by shape across the four formats pi uses — see
28
+ * `extractImageFromBlock` in `src/index.ts`. Descriptions are cached per image
29
+ * hash (LRU, size = config.cacheMax) so the swap is instant by the time
30
+ * `context` fires.
31
+ */
32
+
33
+ import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, ModelRegistry } from "@earendil-works/pi-coding-agent";
34
+ import type { Api, ImageContent, Model, TextContent } from "@earendil-works/pi-ai";
35
+ import { isAbsolute, resolve } from "node:path";
36
+ import {
37
+ DESCRIBE_TIMEOUT_MS,
38
+ extractImageFromBlock,
39
+ formatModelRef,
40
+ isThinkingLevel,
41
+ isVisionModel,
42
+ NON_VISION_IMAGE_NOTE,
43
+ parseModelRef,
44
+ readConfig,
45
+ stripNonVisionImageNote,
46
+ writeConfig,
47
+ HANDOFF_COMMAND_DESCRIPTION,
48
+ type ExtractedImage,
49
+ type VisionHandoffConfig,
50
+ } from "./src/index.js";
51
+ import {
52
+ USAGE_ENTRY_TYPE,
53
+ USAGE_EVENT_CHANNEL,
54
+ type VisionHandoffUsageRecord,
55
+ } from "./src/usage.js";
56
+ import { DescriptionLoader, UNAVAILABLE, type LoaderDeps } from "./src/dataloader.js";
57
+ import { imageHash, findPastedImagePaths, readImageBuffer, readImageBufferBounded, resolvePrewarmImage, isOmittedImageNote } from "./src/image.js";
58
+ import { appendVisionError } from "./src/error-log.js";
59
+ import { resizeImage } from "@earendil-works/pi-coding-agent";
60
+ import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
61
+ import { PrewarmEditor } from "./src/prewarm-editor.js";
62
+ import { Text } from "@earendil-works/pi-tui";
63
+
64
+ let config: VisionHandoffConfig = readConfig();
65
+
66
+ /** Most recent describer failure message (auth error, network error, abort,
67
+ * empty response, etc.). Set by the describer via the loader deps; surfaced
68
+ * to the user by the `context`/`tool_result` handler via ctx.ui.notify so a
69
+ * broken vision model stops looking like a silent "extension doesn't work".
70
+ * Cleared at the start of each describer attempt. */
71
+ let lastDescriberError: string | null = null;
72
+
73
+ /** Image hashes we've already warned the user about this session. Prevents the
74
+ * `context` hook (which fires before every LLM turn) from re-warning on the
75
+ * same failing images every turn — describer failures aren't cached, so
76
+ * without this guard a broken vision model would spam a warning per turn.
77
+ * Cleared on `session_start`. */
78
+ const warnedHashes = new Set<string>();
79
+
80
+ /** Current model, tracked so the paste-time prewarm gate can skip prewarming
81
+ * when the active model is vision-capable (handoff won't run → a prewarm
82
+ * would be a wasted vision call). Updated in session_start and model_select. */
83
+ let currentModel: Model<Api> | undefined | null;
84
+
85
+ /** Whether the paste-time prewarm editor is installed this session. False in
86
+ * non-TUI modes or when another extension already replaced the editor. */
87
+ let editorInstalled = false;
88
+
89
+ let visionModelCache: { ref: string; model: Model<Api> } | null = null;
90
+ let visionModelUnresolvedRef: string | null = null;
91
+
92
+ interface PendingAsyncClipboardHandoff {
93
+ token: symbol;
94
+ paths: Set<string>;
95
+ cancelled: boolean;
96
+ }
97
+
98
+ interface PreparedClipboardImage {
99
+ path: string;
100
+ image: ExtractedImage;
101
+ description: Promise<string>;
102
+ }
103
+
104
+ let pendingAsyncClipboardHandoff: PendingAsyncClipboardHandoff | null = null;
105
+
106
+ function cancelAsyncClipboardHandoff(): void {
107
+ if (!pendingAsyncClipboardHandoff) return;
108
+ pendingAsyncClipboardHandoff.cancelled = true;
109
+ pendingAsyncClipboardHandoff = null;
110
+ }
111
+
112
+ function isPendingClipboardRead(input: unknown, cwd: string): boolean {
113
+ const pending = pendingAsyncClipboardHandoff;
114
+ if (!pending || !input || typeof input !== "object") return false;
115
+ const raw = (input as { path?: unknown }).path;
116
+ if (typeof raw !== "string") return false;
117
+ const path = raw.startsWith("@") ? raw.slice(1) : raw;
118
+ const absolute = isAbsolute(path) ? path : resolve(cwd, path);
119
+ return pending.paths.has(absolute);
120
+ }
121
+
122
+ function scheduleAsyncClipboardInjection(
123
+ pi: ExtensionAPI,
124
+ ctx: ExtensionContext,
125
+ paths: string[],
126
+ prepared: Promise<PreparedClipboardImage | null>[],
127
+ ): void {
128
+ cancelAsyncClipboardHandoff();
129
+ const pending: PendingAsyncClipboardHandoff = {
130
+ token: Symbol("async-clipboard-handoff"),
131
+ paths: new Set(paths),
132
+ cancelled: false,
133
+ };
134
+ pendingAsyncClipboardHandoff = pending;
135
+
136
+ void (async () => {
137
+ const entries = (await Promise.all(prepared)).filter(
138
+ (entry): entry is PreparedClipboardImage => entry !== null,
139
+ );
140
+ if (entries.length === 0) {
141
+ if (pendingAsyncClipboardHandoff?.token === pending.token) {
142
+ pendingAsyncClipboardHandoff = null;
143
+ }
144
+ return;
145
+ }
146
+ const descriptions = await Promise.all(entries.map((entry) => entry.description));
147
+
148
+ // Always yield out of before_agent_start, even when paste-time prewarm made
149
+ // every description an immediate cache hit. This lets Pi enter the active
150
+ // agent run before sendMessage queues the fallback as a steering message.
151
+ await new Promise<void>((resolveImmediate) => setImmediate(resolveImmediate));
152
+ if (
153
+ pending.cancelled ||
154
+ pendingAsyncClipboardHandoff?.token !== pending.token ||
155
+ !config.asyncClipboardHandoff ||
156
+ !isConfigured(config)
157
+ ) {
158
+ return;
159
+ }
160
+
161
+ warnFailedImages(
162
+ ctx,
163
+ entries.map((entry) => entry.image),
164
+ descriptions,
165
+ lastDescriberError ?? "unknown error",
166
+ );
167
+ const content = [
168
+ "Asynchronous vision handoff for pasted image path(s):",
169
+ ...entries.map(
170
+ (entry, index) => `${entry.path}\n${descriptions[index] ?? UNAVAILABLE}`,
171
+ ),
172
+ ].join("\n\n");
173
+
174
+ pendingAsyncClipboardHandoff = null;
175
+ try {
176
+ pi.sendMessage(
177
+ {
178
+ customType: "vision-watcher-async",
179
+ content,
180
+ display: true,
181
+ details: { imageCount: entries.length },
182
+ },
183
+ { deliverAs: "steer", triggerTurn: true },
184
+ );
185
+ } catch {
186
+ // The session may have been replaced/reloaded while the description was
187
+ // in flight. The stale extension instance must not affect the new one.
188
+ }
189
+ })().catch(() => {
190
+ if (pendingAsyncClipboardHandoff?.token === pending.token) {
191
+ pendingAsyncClipboardHandoff = null;
192
+ }
193
+ });
194
+ }
195
+
196
+ // Usage reporter; wired to pi.appendEntry + pi.events.emit in the default
197
+ // export. No-op until then so the describer is safe to call before wiring.
198
+ let reportUsage: (record: VisionHandoffUsageRecord) => void = () => {};
199
+
200
+ function resolveVisionModel(modelRegistry: ModelRegistry, ref: string): Model<Api> | null {
201
+ if (visionModelCache && visionModelCache.ref === ref) return visionModelCache.model;
202
+ const parsed = parseModelRef(ref);
203
+ if (!parsed) return null;
204
+ const model = modelRegistry.find(parsed.provider, parsed.id);
205
+ if (!model) return null;
206
+ visionModelCache = { ref, model };
207
+ return model;
208
+ }
209
+
210
+ /** True when the primary vision model resolves, or — failing that — at least
211
+ * one configured fallback resolves. A broken primary ref (typo, removed
212
+ * model) must not short-circuit the whole pipeline to "unresolved" warnings
213
+ * when a fallback could still describe images; the loader's failover picks
214
+ * the resolvable fallback up in bindTurnContext. */
215
+ function isAnyVisionModelResolvable(modelRegistry: ModelRegistry, config: VisionHandoffConfig): boolean {
216
+ if (resolveVisionModel(modelRegistry, config.visionModel!)) return true;
217
+ return config.fallbackModels.some((ref) => resolveVisionModel(modelRegistry, ref));
218
+ }
219
+
220
+ const loaderDeps: LoaderDeps = {
221
+ getConfig: () => config,
222
+ resolveVisionModel,
223
+ reportUsage: (record) => reportUsage(record),
224
+ setLastError: (msg) => {
225
+ lastDescriberError = msg;
226
+ },
227
+ };
228
+ const loader = new DescriptionLoader(loaderDeps);
229
+
230
+ function isConfigured(cfg: VisionHandoffConfig): boolean {
231
+ return cfg.enabled && !!cfg.visionModel;
232
+ }
233
+
234
+ function isHandoffTarget(
235
+ model: { provider?: string; id?: string; input?: ("text" | "image")[] } | undefined | null,
236
+ cfg: VisionHandoffConfig,
237
+ ): boolean {
238
+ if (!model || !model.provider || !model.id) return false;
239
+ const ref = formatModelRef(model.provider, model.id);
240
+ if (cfg.handoffModels.includes(ref)) return true;
241
+ if (cfg.autoHandoff && !isVisionModel(model)) return true;
242
+ return false;
243
+ }
244
+
245
+ /** Whether paste-time prewarm should fire for a text change right now: the
246
+ * opt-in flag is on, handoff is configured, and the active model is a handoff
247
+ * target (so the prewarmed description will actually be consumed — a
248
+ * vision-capable model needs no handoff, so prewarming would waste a call). */
249
+ function shouldPrewarmPaste(): boolean {
250
+ return config.prewarmPastedImages && isConfigured(config) && isHandoffTarget(currentModel, config);
251
+ }
252
+
253
+ /** Prewarm one pasted clipboard image path through the dataloader at paste
254
+ * time. Mirrors the before_agent_start clipboard prewarm, but binds only the
255
+ * model registry (no turn signal — the agent is idle at paste time) and
256
+ * resets the turn prompt to "" so a stale previous-turn prompt can't leak
257
+ * into the description. The user's question isn't typed yet at paste time, so
258
+ * the description is generated without question context (the documented
259
+ * tradeoff of the opt-in). If the user submits before this dispatch fires,
260
+ * before_agent_start overwrites the prompt — a benign bonus, not a bug. */
261
+ function prewarmClipboardPath(path: string, modelRegistry: ModelRegistry): void {
262
+ const read = readImageBuffer(path);
263
+ if (!read) return;
264
+ resolvePrewarmImage(read.buf, read.mimeType, resizeImage)
265
+ .then((img) => {
266
+ if (!img) return;
267
+ // Paste happens while the agent is idle (no run → no live signal).
268
+ // Reset the turn-abort controller so a previous turn's ESC doesn't leave
269
+ // it aborted and short-circuit this prewarm; the next submit's
270
+ // `before_agent_start` resets again. The prewarm batch uses the loader's
271
+ // controller and becomes abortable once a run starts and binds a live
272
+ // signal.
273
+ loader.resetTurnAbort();
274
+ loader.setPendingTurnPrompt("");
275
+ loader.bindTurnContext({ modelRegistry });
276
+ loader.loadDescription(img).catch(() => {});
277
+ })
278
+ .catch(() => {});
279
+ }
280
+
281
+ /** Install the paste-time prewarm editor wrapper for this session. TUI only,
282
+ * and only when no other extension has replaced the editor — installing over
283
+ * a custom editor would clobber its input handling and break clipboard paste
284
+ * (pi wires paste-image to the outermost editor only). When a custom editor
285
+ * is present, paste-time prewarm is unavailable; submit-time prewarm
286
+ * (before_agent_start) still covers clipboard paths. */
287
+ function installPrewarmEditor(ctx: ExtensionContext): void {
288
+ if (ctx.mode !== "tui") {
289
+ editorInstalled = false;
290
+ return;
291
+ }
292
+ if (ctx.ui.getEditorComponent()) {
293
+ editorInstalled = false;
294
+ return;
295
+ }
296
+ editorInstalled = true;
297
+ ctx.ui.setEditorComponent((_tui, theme, keybindings) =>
298
+ new PrewarmEditor(_tui, theme, keybindings, {
299
+ modelRegistry: ctx.modelRegistry,
300
+ shouldPrewarm: shouldPrewarmPaste,
301
+ prewarmPath: prewarmClipboardPath,
302
+ }),
303
+ );
304
+ }
305
+
306
+ function notifyUnresolvedVisionModel(ctx: ExtensionContext, ref: string): void {
307
+ if (visionModelUnresolvedRef === ref) return;
308
+ visionModelUnresolvedRef = ref;
309
+ if (ctx.hasUI) {
310
+ ctx.ui.notify(
311
+ `pi-vision-watcher: configured vision model "${ref}" was not found in the registry — run /vision-watcher to pick a model.`,
312
+ "warning",
313
+ );
314
+ }
315
+ }
316
+
317
+ /** Notify once per failing image per session (dedup via warnedHashes), and log
318
+ * EVERY failure (even in headless/SDK mode with no UI) to the error log so the
319
+ * user can troubleshoot. The log entry's `phase: "warn"` carries the failing
320
+ * image hashes and the surfaced reason — when the reason is "unknown error",
321
+ * the real cause lives in the matching `batch`/`single` entry for those hashes. */
322
+ function warnFailedImages(
323
+ ctx: ExtensionContext,
324
+ imgs: ExtractedImage[],
325
+ descs: string[],
326
+ reason: string,
327
+ ): void {
328
+ const newlyFailed: string[] = [];
329
+ for (let i = 0; i < imgs.length; i++) {
330
+ if (descs[i] === UNAVAILABLE) {
331
+ const h = imageHash(imgs[i].mimeType, imgs[i].data);
332
+ if (!warnedHashes.has(h)) newlyFailed.push(h);
333
+ }
334
+ }
335
+ if (newlyFailed.length === 0) return;
336
+ for (const h of newlyFailed) warnedHashes.add(h);
337
+ // Always log: troubleshooting must work in headless/SDK mode too, where the
338
+ // notify below never fires.
339
+ appendVisionError({
340
+ phase: "warn",
341
+ reason,
342
+ visionModel: config.visionModel,
343
+ imageHashes: newlyFailed,
344
+ imageCount: newlyFailed.length,
345
+ activeModel: ctx.model ? formatModelRef(ctx.model.provider, ctx.model.id) : undefined,
346
+ });
347
+ if (!ctx.hasUI) return;
348
+ ctx.ui.notify(
349
+ `pi-vision-watcher: image description failed — ${reason}. Vision model: ${config.visionModel}`,
350
+ "warning",
351
+ );
352
+ }
353
+
354
+ export default function (pi: ExtensionAPI) {
355
+ config = readConfig();
356
+
357
+ pi.registerMessageRenderer("vision-watcher-async", (message, { expanded }, theme) => {
358
+ const details = message.details as { imageCount?: number } | undefined;
359
+ const count = details?.imageCount ?? 1;
360
+ const label = `Vision handoff · ${count} pasted image${count === 1 ? "" : "s"}`;
361
+ const hint = expanded ? "Ctrl+O to collapse" : "Ctrl+O to expand";
362
+ const summary = theme.fg("dim", `👀 ${label} · ${hint}`);
363
+ if (!expanded) return new Text(summary, 0, 0);
364
+ // Apply the dim (grey) style per line: the TUI appends a full SGR reset at
365
+ // the end of each rendered line, so a single style wrapper would only tint
366
+ // the first line. Text wraps with wrapTextWithAnsi (ANSI-preserving), and
367
+ // each line here carries its own dim code so wrapped continuations stay grey.
368
+ const contentStr = typeof message.content === "string" ? message.content : "";
369
+ const body = contentStr.length > 0
370
+ ? contentStr.split("\n").map((line) => theme.fg("dim", line)).join("\n")
371
+ : "";
372
+ return new Text(body ? `${summary}\n${body}` : summary, 0, 0);
373
+ });
374
+
375
+ // Wire the usage reporter to pi's persistence + event bus. appendEntry
376
+ // persists the record so it replays on session resume/branch, and the event
377
+ // lets live consumers filter on one channel for tokens AND energy. Each call
378
+ // is independently guarded so a persistence/emit failure never breaks a
379
+ // describer turn. Re-assigned every factory invocation (pi re-runs the
380
+ // factory on /new, /resume, fork, /reload) so the closure always references
381
+ // the live pi.
382
+ reportUsage = (record: VisionHandoffUsageRecord) => {
383
+ try {
384
+ pi.appendEntry(USAGE_ENTRY_TYPE, record);
385
+ } catch {
386
+ // never break the describer on persistence failure
387
+ }
388
+ try {
389
+ pi.events?.emit(USAGE_EVENT_CHANNEL, record);
390
+ } catch {
391
+ // never break the describer on emit failure
392
+ }
393
+ };
394
+
395
+ pi.on("session_start", async (_event, ctx) => {
396
+ cancelAsyncClipboardHandoff();
397
+ // Reload in case the user edited the config on disk from another session.
398
+ config = readConfig();
399
+ visionModelCache = null;
400
+ visionModelUnresolvedRef = null;
401
+ warnedHashes.clear();
402
+ loader.reset();
403
+ currentModel = ctx.model;
404
+ installPrewarmEditor(ctx);
405
+ });
406
+
407
+ pi.on("before_agent_start", async (event, ctx) => {
408
+ cancelAsyncClipboardHandoff();
409
+ if (!isConfigured(config)) return;
410
+ if (!isHandoffTarget(ctx.model, config)) return;
411
+
412
+ // Fresh turn → fresh turn-abort controller. `before_agent_start` fires
413
+ // BEFORE the agent run starts, so `ctx.signal` is undefined here (the run's
414
+ // abort signal doesn't exist yet — it's created in `agent.prompt()` →
415
+ // `runWithLifecycle`, which runs AFTER this event). The prewarm below
416
+ // dispatches describer batches now, and they must be abortable once the
417
+ // run's live signal arrives — so reset the loader's turn-abort controller
418
+ // here, and let the later `tool_result`/`context` binds forward the live
419
+ // signal into it. Without this reset, a previous turn's ESC would leave
420
+ // the controller aborted and every dispatch would short-circuit to
421
+ // UNAVAILABLE.
422
+ loader.resetTurnAbort();
423
+
424
+ // Capture this turn's user prompt so every image in the turn — attached or
425
+ // read via the read tool — is described in the same request context.
426
+ loader.setPendingTurnPrompt(event.prompt || "");
427
+ loader.bindTurnContext(ctx);
428
+
429
+ if (!isAnyVisionModelResolvable(ctx.modelRegistry, config)) {
430
+ notifyUnresolvedVisionModel(ctx, config.visionModel!);
431
+ return;
432
+ }
433
+
434
+ // PRE-WARM at paste-enter via the DataLoader. Two image sources land here:
435
+ //
436
+ // 1. Attached image blocks (event.images) — vision-capable targets where
437
+ // the user message itself carries image blocks (e.g. `pi --image`).
438
+ //
439
+ // 2. Pasted image FILE PATHS in the prompt text — the common non-vision
440
+ // flow. pi's `handleClipboardImagePaste` (and other paste mechanisms like
441
+ // localterm-paste) write each pasted image to a temp file and insert the
442
+ // PATH as
443
+ // text at the cursor; on a non-vision model these arrive as path tokens
444
+ // in `event.prompt`, NOT as `event.images`. We scan the prompt for those
445
+ // temp paths, read the files, and `loadDescription()` them so the ONE
446
+ // batched vision call starts the instant you press enter — CONCURRENT
447
+ // with the agent's first response generation — instead of waiting for
448
+ // the agent to `read` the files. By the time the agent's `read` tool
449
+ // results fire, `tool_result`'s `loadDescription()` is a cache hit.
450
+ //
451
+ // Both sources flow through the same loader: `load()` is synchronous and
452
+ // memoized, so all images in this frame (attached + clipboard-path)
453
+ // coalesce into ONE batch dispatched via `setImmediate` after the
454
+ // microtask cascade settles.
455
+ for (const image of event.images ?? []) {
456
+ if (!image || image.type !== "image" || !image.data) continue;
457
+ loader.loadDescription({ data: image.data, mimeType: image.mimeType || "image/png" }).catch(() => {});
458
+ }
459
+
460
+ // Pasted clipboard image paths in the prompt text — resolve each to the
461
+ // SAME ExtractedImage pi's `read` tool will emit, then warm the loader so
462
+ // the `tool_result`'s `loadDescription()` is a cache hit (no wasted vision
463
+ // call). pi's read tool resizes images by default: for a small image it
464
+ // returns the raw bytes unchanged (our raw read matches), but for an
465
+ // oversized image it RE-ENCODES via Photon — so we run the same
466
+ // `resizeImage` pipeline to produce a matching key. The resize branch is
467
+ // async (worker thread) and fire-and-forget; its `loadDescription()` lands
468
+ // in a later batch than the no-resize images, so a mixed small+oversized
469
+ // paste may split into two vision calls (still no WASTED call — each
470
+ // describes an image the agent will see). A file that can't be read, isn't
471
+ // a supported image, or fails to resize is skipped (the agent's `read`
472
+ // will still describe it via `tool_result` if it emits an image block).
473
+ const clipboardPaths = findPastedImagePaths(event.prompt || "");
474
+ const preparedClipboardImages = clipboardPaths.map(
475
+ async (path): Promise<PreparedClipboardImage | null> => {
476
+ try {
477
+ const read = readImageBuffer(path);
478
+ if (!read) return null;
479
+ const image = await resolvePrewarmImage(read.buf, read.mimeType, resizeImage);
480
+ if (!image) return null;
481
+ const description = loader.loadDescription(image);
482
+ description.catch(() => {});
483
+ return { path, image, description };
484
+ } catch {
485
+ return null;
486
+ }
487
+ },
488
+ );
489
+
490
+ if (config.asyncClipboardHandoff && clipboardPaths.length > 0) {
491
+ scheduleAsyncClipboardInjection(pi, ctx, clipboardPaths, preparedClipboardImages);
492
+ } else {
493
+ // Start every preparation task even when no consumer awaits it. Each task
494
+ // calls loadDescription as soon as resize completes, preserving the
495
+ // original fire-and-forget submit-time prewarm behavior.
496
+ for (const prepared of preparedClipboardImages) prepared.catch(() => {});
497
+ }
498
+ });
499
+
500
+ // A direct read and a nested pi.read both emit a read tool_call. If it targets
501
+ // one of this turn's pasted clipboard paths before the async fallback is
502
+ // injected, the normal tool_result/context path wins and the queued custom
503
+ // message is cancelled. The in-flight description is deliberately retained:
504
+ // the read result reuses it as a cache/in-flight hit.
505
+ pi.on("tool_call", (event, ctx) => {
506
+ if (event.toolName !== "read" && event.toolName !== "pi.read") return;
507
+ if (isPendingClipboardRead(event.input, ctx.cwd)) {
508
+ cancelAsyncClipboardHandoff();
509
+ }
510
+ });
511
+
512
+ pi.on("session_shutdown", () => {
513
+ cancelAsyncClipboardHandoff();
514
+ });
515
+
516
+ // The PRIMARY injection point: the `read` tool's `tool_result` handler.
517
+ // When the agent reads image files, this fires for each read result. It
518
+ // calls the loader's `loadDescription(img)` for every image block and
519
+ // AWAITS the shared batch — so N parallel reads (pi runs `read` via
520
+ // Promise.all) coalesce into ONE batched vision call: pi fires each read's
521
+ // `tool_result` as its I/O completes (poll phase), and the loader's
522
+ // `setImmediate` dispatch defers to the check phase AFTER the whole poll
523
+ // iteration, so reads completing together land in ONE batch and all resolve
524
+ // together.
525
+ //
526
+ // Both a direct agent `read` and a NESTED `pi.read` inside fabric_exec reach
527
+ // this handler and take the SAME path: WARM the cache (during the free
528
+ // tool-result phase), strip pi's misleading non-vision note, and KEEP the
529
+ // image block so kitty renders it inline and /resume retains it. The
530
+ // `context` hook swaps image→description on the LLM-bound clone before the
531
+ // next turn, so the text-only agent receives the description as text. This
532
+ // holds for fabric too: pi-fabric re-attaches the nested read's image to the
533
+ // fabric_exec tool-result content, which IS the agent's message context — so
534
+ // the `context` hook sees and swaps it, exactly like a direct read.
535
+ pi.on("tool_result", async (event, ctx) => {
536
+ if (!isConfigured(config)) return;
537
+ if (event.toolName !== "read") return;
538
+ const content = event.content;
539
+ if (!Array.isArray(content)) return;
540
+
541
+ // Collect image blocks in this read result. (The read tool emits image
542
+ // blocks even for non-vision models — they reach here untouched.)
543
+ if (!isHandoffTarget(ctx.model, config)) return;
544
+ const imgs: ExtractedImage[] = [];
545
+ let hasImageBlock = false;
546
+ for (const block of content) {
547
+ const img = extractImageFromBlock(block);
548
+ if (img) {
549
+ imgs.push(img);
550
+ hasImageBlock = true;
551
+ }
552
+ }
553
+
554
+ // Fallback: `read` detected an image but `processImage` failed (Photon
555
+ // unavailable / decode fail / convert fail / couldn't resize below the
556
+ // inline limit), so it emitted a "[Image omitted: …]" text note with NO
557
+ // image block. The image-block path above never sees it, so the image
558
+ // would go undescribed and the model would be told the image was "omitted".
559
+ // Re-read the raw file and describe its bytes directly — the vision model
560
+ // decodes them itself (no Photon needed). This recovers the Photon-
561
+ // unavailable and under-vision-model-limit cases; APNG/unsupported are
562
+ // rejected by the sniff (the vision model can't decode them either).
563
+ let omittedNoteIndices: number[] = [];
564
+ if (!hasImageBlock) {
565
+ for (let i = 0; i < content.length; i++) {
566
+ const block = content[i];
567
+ if (
568
+ block &&
569
+ typeof block === "object" &&
570
+ (block as { type: string }).type === "text" &&
571
+ typeof (block as { text: string }).text === "string" &&
572
+ isOmittedImageNote((block as { text: string }).text)
573
+ ) {
574
+ omittedNoteIndices.push(i);
575
+ }
576
+ }
577
+ if (omittedNoteIndices.length > 0) {
578
+ const inputPath = event.input.path;
579
+ if (typeof inputPath === "string") {
580
+ const resolved = isAbsolute(inputPath) ? inputPath : resolve(ctx.cwd, inputPath);
581
+ const read = readImageBufferBounded(resolved);
582
+ if (read) imgs.push({ data: read.buf.toString("base64"), mimeType: read.mimeType });
583
+ }
584
+ }
585
+ }
586
+
587
+ if (imgs.length === 0) return;
588
+ loader.bindTurnContext(ctx);
589
+ if (!isAnyVisionModelResolvable(ctx.modelRegistry, config)) {
590
+ notifyUnresolvedVisionModel(ctx, config.visionModel!);
591
+ return;
592
+ }
593
+
594
+ // load() each image — synchronous calls that push into the current batch
595
+ // and return memoized promises — then await them all. pi fires each read's
596
+ // `tool_result` event as that read's I/O completes (poll phase); the
597
+ // loader's `setImmediate` dispatch defers to the check phase, AFTER the
598
+ // whole poll iteration, so reads completing together (the common case for
599
+ // cached local files) land in ONE batch — ONE vision call for the whole
600
+ // read set, not N. Awaiting here runs the describer during the tool-result
601
+ // phase (free time), so the batch is COMPLETE before `context` fires,
602
+ // making `context` a non-blocking cache hit instead of a cold miss.
603
+ const descs = await Promise.all(imgs.map((img) => loader.loadDescription(img)));
604
+
605
+ // On user abort, leave the result untouched — pi is tearing the turn
606
+ // down and the LLM-bound content won't be sent.
607
+ if (ctx.signal?.aborted) return;
608
+
609
+ warnFailedImages(ctx, imgs, descs, lastDescriberError ?? "unknown error");
610
+
611
+ // Recovery path: no image block was emitted, so there's nothing for the
612
+ // `context` hook to swap. Replace each "[Image omitted: …]" note directly
613
+ // with its description (the recovered raw-bytes image is the only img here,
614
+ // so descs aligns 1:1 with omittedNoteIndices).
615
+ if (!hasImageBlock) {
616
+ const recovered = content.slice();
617
+ for (let i = 0; i < omittedNoteIndices.length && i < descs.length; i++) {
618
+ recovered[omittedNoteIndices[i]] = { type: "text", text: descs[i] };
619
+ }
620
+ return { content: recovered as (TextContent | ImageContent)[] };
621
+ }
622
+
623
+ // Warm the cache (done above) and strip pi's
624
+ // `[Current model does not support images…]` note from text blocks —
625
+ // since the handoff replaces the image with a description in the
626
+ // `context` hook, that note is misleading (the agent WILL receive the
627
+ // image's content, as text). Keep the image block itself so kitty still
628
+ // renders it inline and `/resume` retains it; the `context` hook swaps
629
+ // the image for its description in the LLM-bound clone before the next turn.
630
+ let stripped = false;
631
+ const next = content.slice();
632
+ for (let i = 0; i < next.length; i++) {
633
+ const block = next[i];
634
+ if (block && typeof block === "object" && (block as { type: string }).type === "text") {
635
+ const text = (block as { text: string }).text;
636
+ if (typeof text === "string" && text.includes(NON_VISION_IMAGE_NOTE)) {
637
+ const cleaned = stripNonVisionImageNote(text);
638
+ if (cleaned !== text) {
639
+ next[i] = { type: "text", text: cleaned };
640
+ stripped = true;
641
+ }
642
+ }
643
+ }
644
+ }
645
+ if (stripped) return { content: next as (TextContent | ImageContent)[] };
646
+ });
647
+
648
+ // The FALLBACK injection point: the `context` event fires as the agent's
649
+ // `transformContext`, BEFORE pi-ai's `downgradeUnsupportedImages` strips
650
+ // image blocks and BEFORE `convertToLlm`. It catches any image blocks that
651
+ // didn't go through the `read` tool's `tool_result` handler — user-attached
652
+ // images (for vision-capable handoff targets), custom extension-injected
653
+ // messages, or reads that somehow bypassed the handler. `emitContext` does a
654
+ // `structuredClone`, so swapping here touches only the LLM-bound payload.
655
+ //
656
+ // Read images are already text by this point (the `tool_result` handler
657
+ // replaced them), so this is usually a no-op for the common paste-and-read
658
+ // flow. For the images it does find, `loadDescription()` is a cache hit
659
+ // (warmed by `before_agent_start`) or queues into the loader's current batch.
660
+ pi.on("context", async (event, ctx) => {
661
+ if (!isConfigured(config)) return;
662
+
663
+ const messages = event.messages as unknown as Array<Record<string, unknown>>;
664
+ if (!Array.isArray(messages)) return;
665
+
666
+ const byHash = new Map<string, ExtractedImage>();
667
+ let anyImage = false;
668
+ for (const msg of messages) {
669
+ const content = msg.content;
670
+ if (!Array.isArray(content)) continue;
671
+ for (const block of content) {
672
+ const img = extractImageFromBlock(block);
673
+ if (!img) continue;
674
+ anyImage = true;
675
+ byHash.set(imageHash(img.mimeType, img.data), img);
676
+ }
677
+ }
678
+ if (!anyImage) return;
679
+ if (!isHandoffTarget(ctx.model, config)) return;
680
+ loader.bindTurnContext(ctx);
681
+ if (!isAnyVisionModelResolvable(ctx.modelRegistry, config)) {
682
+ notifyUnresolvedVisionModel(ctx, config.visionModel!);
683
+ return;
684
+ }
685
+
686
+ // Cache hits (warmed by before_agent_start / tool_result) resolve
687
+ // instantly; any remaining misses queue into the loader's current batch.
688
+ const imgs = [...byHash.values()];
689
+ const descArr = await Promise.all(imgs.map((img) => loader.loadDescription(img)));
690
+ const descs = new Map<string, string>();
691
+ for (let i = 0; i < imgs.length; i++) {
692
+ descs.set(imageHash(imgs[i].mimeType, imgs[i].data), descArr[i]);
693
+ }
694
+
695
+ if (ctx.signal?.aborted) return;
696
+
697
+ warnFailedImages(ctx, imgs, descArr, lastDescriberError ?? "unknown error");
698
+
699
+ let changed = false;
700
+ for (const msg of messages) {
701
+ const content = msg.content;
702
+ if (!Array.isArray(content)) continue;
703
+ let touched = false;
704
+ const next: unknown[] = [];
705
+ for (const block of content) {
706
+ const img = extractImageFromBlock(block);
707
+ if (img) {
708
+ next.push({ type: "text", text: descs.get(imageHash(img.mimeType, img.data)) ?? UNAVAILABLE });
709
+ touched = true;
710
+ } else {
711
+ next.push(block);
712
+ }
713
+ }
714
+ if (touched) {
715
+ msg.content = next;
716
+ changed = true;
717
+ }
718
+ }
719
+ if (changed) return { messages: event.messages };
720
+ });
721
+
722
+ pi.on("model_select", (event, ctx) => {
723
+ currentModel = event.model;
724
+ if (!ctx.hasUI) return;
725
+ if (!isConfigured(config)) return;
726
+ const model = event.model;
727
+ if (!model) return;
728
+ if (isHandoffTarget(model, config) && !isVisionModel(model)) {
729
+ ctx.ui.notify(
730
+ `pi-vision-watcher: active — images will be described by ${config.visionModel}`,
731
+ "info",
732
+ );
733
+ }
734
+ });
735
+
736
+ pi.registerCommand("vision-watcher", {
737
+ description: HANDOFF_COMMAND_DESCRIPTION,
738
+ getArgumentCompletions(prefix: string) {
739
+ const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "fallback", "timeout", "add", "remove", "clear", "help"];
740
+ const matches = subcommands.filter((s) => s.startsWith(prefix));
741
+ return matches.length > 0 ? matches.map((s) => ({ value: s, label: s })) : null;
742
+ },
743
+ handler: async (args, ctx) => {
744
+ await handleHandoffCommand(ctx, args.trim());
745
+ },
746
+ });
747
+ }
748
+
749
+ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string): Promise<void> {
750
+ const parts = args.split(/\s+/);
751
+ const subcommand = parts[0]?.toLowerCase() ?? "";
752
+ const rest = parts.slice(1).join(" ");
753
+
754
+ // /vision-watcher (no args) or /vision-watcher select — interactive picker
755
+ if (!subcommand || subcommand === "select") {
756
+ await showSelector(ctx);
757
+ return;
758
+ }
759
+
760
+ if (subcommand === "help") {
761
+ ctx.ui.notify(
762
+ [
763
+ "pi-vision-watcher commands:",
764
+ " /vision-watcher Open interactive picker to choose the vision model",
765
+ " /vision-watcher select Same as /vision-watcher",
766
+ " /vision-watcher model <p/id> Set the vision model directly",
767
+ " /vision-watcher status Show current config and active state",
768
+ " /vision-watcher enable Enable vision handoff",
769
+ " /vision-watcher disable Disable vision handoff (keeps configured model)",
770
+ " /vision-watcher auto <on|off> Toggle automatic handoff for all non-vision models",
771
+ " /vision-watcher thinking <off|minimal|low|medium|high|xhigh|max>",
772
+ " Set the vision describer's thinking effort (off = disabled)",
773
+ " /vision-watcher prewarm <on|off>",
774
+ " Toggle describing pasted images at paste-time (opt-in, off by default)",
775
+ " /vision-watcher fallback <on|off>",
776
+ " Inject pasted-image descriptions asynchronously when no matching read wins",
777
+ " /vision-watcher timeout <ms> Set the per-image description timeout (default 45000)",
778
+ " /vision-watcher add <p/id> Force handoff for an extra model",
779
+ " /vision-watcher remove <p/id> Stop forcing handoff for a model",
780
+ " /vision-watcher clear Clear the configured vision model",
781
+ " /vision-watcher help This message",
782
+ "",
783
+ "Config: ~/.pi/agent/extensions/pi-vision-watcher.json",
784
+ "Mechanism: before_agent_start warms a description cache; tool_result loads",
785
+ " read images through a dataloader (one batched vision call); context swaps",
786
+ " image blocks in the payload for the cached text description.",
787
+ " prewarm on wraps the editor to describe pasted images at paste-time.",
788
+ " fallback on asynchronously injects a collapsed description unless a matching read wins.",
789
+ ].join("\n"),
790
+ "info",
791
+ );
792
+ return;
793
+ }
794
+
795
+ if (subcommand === "status") {
796
+ showStatus(ctx);
797
+ return;
798
+ }
799
+
800
+ if (subcommand === "enable") {
801
+ updateConfig(ctx, (c) => ({ ...c, enabled: true }), "Vision handoff enabled.");
802
+ return;
803
+ }
804
+
805
+ if (subcommand === "disable") {
806
+ updateConfig(ctx, (c) => ({ ...c, enabled: false }), "Vision handoff disabled.");
807
+ return;
808
+ }
809
+
810
+ if (subcommand === "auto") {
811
+ const value = rest.toLowerCase();
812
+ if (value !== "on" && value !== "off") {
813
+ ctx.ui.notify("Usage: /vision-watcher auto <on|off>", "warning");
814
+ return;
815
+ }
816
+ const on = value === "on";
817
+ updateConfig(
818
+ ctx,
819
+ (c) => ({ ...c, autoHandoff: on }),
820
+ `Automatic handoff for non-vision models ${on ? "on" : "off"}.`,
821
+ );
822
+ return;
823
+ }
824
+
825
+ if (subcommand === "thinking") {
826
+ handleThinkingSubcommand(ctx, rest);
827
+ return;
828
+ }
829
+
830
+ if (subcommand === "prewarm") {
831
+ handlePrewarmSubcommand(ctx, rest);
832
+ return;
833
+ }
834
+
835
+ if (subcommand === "fallback") {
836
+ handleFallbackSubcommand(ctx, rest);
837
+ return;
838
+ }
839
+
840
+ if (subcommand === "timeout") {
841
+ handleTimeoutSubcommand(ctx, rest);
842
+ return;
843
+ }
844
+
845
+ if (subcommand === "clear") {
846
+ updateConfig(
847
+ ctx,
848
+ (c) => ({ ...c, visionModel: null }),
849
+ "Vision model cleared — handoff inactive until you pick a model.",
850
+ );
851
+ return;
852
+ }
853
+
854
+ if (subcommand === "model") {
855
+ if (!rest) {
856
+ ctx.ui.notify("Usage: /vision-watcher model <provider/id>", "warning");
857
+ return;
858
+ }
859
+ const parsed = parseModelRef(rest);
860
+ if (!parsed) {
861
+ ctx.ui.notify(`Invalid model reference: "${rest}". Use "provider/id".`, "error");
862
+ return;
863
+ }
864
+ const model = ctx.modelRegistry.find(parsed.provider, parsed.id);
865
+ if (!model) {
866
+ ctx.ui.notify(`Model not found: ${rest}. Use /vision-watcher to pick from the list.`, "error");
867
+ return;
868
+ }
869
+ const ref = formatModelRef(parsed.provider, parsed.id);
870
+ updateConfig(ctx, (c) => ({ ...c, visionModel: ref }), `Vision model set to ${ref}.`);
871
+ if (!isVisionModel(model)) {
872
+ ctx.ui.notify(
873
+ `Note: ${ref} does not declare image input — it may not describe images well.`,
874
+ "warning",
875
+ );
876
+ }
877
+ return;
878
+ }
879
+
880
+ if (subcommand === "add") {
881
+ if (!rest) {
882
+ ctx.ui.notify("Usage: /vision-watcher add <provider/id>", "warning");
883
+ return;
884
+ }
885
+ const parsed = parseModelRef(rest);
886
+ if (!parsed) {
887
+ ctx.ui.notify(`Invalid model reference: "${rest}". Use "provider/id".`, "error");
888
+ return;
889
+ }
890
+ const ref = formatModelRef(parsed.provider, parsed.id);
891
+ updateConfig(
892
+ ctx,
893
+ (c) => ({ ...c, handoffModels: Array.from(new Set([...c.handoffModels, ref])) }),
894
+ `Added ${ref} to handoff targets.`,
895
+ );
896
+ return;
897
+ }
898
+
899
+ if (subcommand === "remove") {
900
+ if (!rest) {
901
+ ctx.ui.notify("Usage: /vision-watcher remove <provider/id>", "warning");
902
+ return;
903
+ }
904
+ const parsed = parseModelRef(rest);
905
+ if (!parsed) {
906
+ ctx.ui.notify(`Invalid model reference: "${rest}". Use "provider/id".`, "error");
907
+ return;
908
+ }
909
+ const ref = formatModelRef(parsed.provider, parsed.id);
910
+ const before = config.handoffModels.length;
911
+ updateConfig(
912
+ ctx,
913
+ (c) => ({ ...c, handoffModels: c.handoffModels.filter((m) => m !== ref) }),
914
+ `Removed ${ref} from handoff targets.`,
915
+ );
916
+ if (config.handoffModels.length === before) {
917
+ ctx.ui.notify(`Note: ${ref} was not in the handoff list.`, "info");
918
+ }
919
+ return;
920
+ }
921
+
922
+ ctx.ui.notify(`Unknown subcommand: "${subcommand}". Use /vision-watcher help for usage.`, "warning");
923
+ }
924
+
925
+ function updateConfig(
926
+ ctx: ExtensionCommandContext,
927
+ transform: (c: VisionHandoffConfig) => VisionHandoffConfig,
928
+ message: string,
929
+ ): void {
930
+ const next = transform(config);
931
+ if (!next.asyncClipboardHandoff) cancelAsyncClipboardHandoff();
932
+ const path = writeConfig(next);
933
+ config = next;
934
+ visionModelCache = null;
935
+ visionModelUnresolvedRef = null;
936
+ ctx.ui.notify(`${message} (config: ${path})`, "info");
937
+ }
938
+
939
+ /** Resolve a `/vision-watcher thinking <level>` argument into a
940
+ * `(thinking, thinkingLevel)` pair. `off` disables thinking; any of the
941
+ * {@link THINKING_LEVELS} enables it at that effort. */
942
+ function handleThinkingSubcommand(ctx: ExtensionCommandContext, rest: string): void {
943
+ const arg = rest.trim().toLowerCase();
944
+ if (!arg) {
945
+ ctx.ui.notify(
946
+ `Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}.\n` +
947
+ `Usage: /vision-watcher thinking <off|minimal|low|medium|high|xhigh|max>`,
948
+ "info",
949
+ );
950
+ return;
951
+ }
952
+ if (arg === "off") {
953
+ updateConfig(ctx, (c) => ({ ...c, thinking: false }), "Vision describer thinking off.");
954
+ return;
955
+ }
956
+ if (!isThinkingLevel(arg)) {
957
+ ctx.ui.notify(
958
+ `Unknown thinking level: "${arg}". Use off, minimal, low, medium, high, xhigh, or max.`,
959
+ "error",
960
+ );
961
+ return;
962
+ }
963
+ const level = arg;
964
+ updateConfig(
965
+ ctx,
966
+ (c) => ({ ...c, thinking: true, thinkingLevel: level }),
967
+ `Vision describer thinking on (${level}).`,
968
+ );
969
+ }
970
+
971
+ /** Handle `/vision-watcher prewarm <on|off>` — toggle paste-time prewarm. */
972
+ function handlePrewarmSubcommand(ctx: ExtensionCommandContext, rest: string): void {
973
+ const value = rest.trim().toLowerCase();
974
+ if (!value) {
975
+ ctx.ui.notify(
976
+ `Paste-time prewarm: ${config.prewarmPastedImages ? "on" : "off"}.\n` +
977
+ `Usage: /vision-watcher prewarm <on|off>`,
978
+ "info",
979
+ );
980
+ return;
981
+ }
982
+ if (value !== "on" && value !== "off") {
983
+ ctx.ui.notify("Usage: /vision-watcher prewarm <on|off>", "warning");
984
+ return;
985
+ }
986
+ const on = value === "on";
987
+ const note = on
988
+ ? editorInstalled
989
+ ? "Paste-time prewarm on — pasted images are described the instant their path lands in the prompt (before submit)."
990
+ : "Paste-time prewarm on — but another custom editor extension is active, so it's unavailable. Submit-time prewarm still works; disable the other editor extension (or /vision-watcher prewarm off) to silence this."
991
+ : "Paste-time prewarm off — images are described at submit time (default).";
992
+ updateConfig(ctx, (c) => ({ ...c, prewarmPastedImages: on }), note);
993
+ }
994
+
995
+ /** Handle /vision-watcher fallback <on|off>. */
996
+ function handleFallbackSubcommand(ctx: ExtensionCommandContext, rest: string): void {
997
+ const value = rest.trim().toLowerCase();
998
+ if (!value) {
999
+ ctx.ui.notify(
1000
+ `Async pasted-path fallback: ${config.asyncClipboardHandoff ? "on" : "off"}.\n` +
1001
+ "Usage: /vision-watcher fallback <on|off>",
1002
+ "info",
1003
+ );
1004
+ return;
1005
+ }
1006
+ if (value !== "on" && value !== "off") {
1007
+ ctx.ui.notify("Usage: /vision-watcher fallback <on|off>", "warning");
1008
+ return;
1009
+ }
1010
+ const on = value === "on";
1011
+ updateConfig(
1012
+ ctx,
1013
+ (c) => ({ ...c, asyncClipboardHandoff: on }),
1014
+ on
1015
+ ? "Async pasted-path fallback on — a matching read wins the race; otherwise the description is injected as a collapsed message."
1016
+ : "Async pasted-path fallback off.",
1017
+ );
1018
+ }
1019
+
1020
+ /** Handle `/vision-watcher timeout <ms>` — set the base per-image description
1021
+ * timeout (controls how long a slow vision model runs before failing over to
1022
+ * the configured fallback models). */
1023
+ function handleTimeoutSubcommand(ctx: ExtensionCommandContext, rest: string): void {
1024
+ const arg = rest.trim();
1025
+ if (!arg) {
1026
+ ctx.ui.notify(
1027
+ `Per-image description timeout: ${config.describeTimeoutMs} ms ` +
1028
+ `(default ${DESCRIBE_TIMEOUT_MS} ms).\n` +
1029
+ "Usage: /vision-watcher timeout <ms>",
1030
+ "info",
1031
+ );
1032
+ return;
1033
+ }
1034
+ const ms = Number(arg);
1035
+ if (!Number.isFinite(ms) || ms <= 0 || !Number.isInteger(ms)) {
1036
+ ctx.ui.notify(
1037
+ `Invalid timeout: "${arg}". Use a positive whole number of milliseconds (default ${DESCRIBE_TIMEOUT_MS}).`,
1038
+ "error",
1039
+ );
1040
+ return;
1041
+ }
1042
+ updateConfig(
1043
+ ctx,
1044
+ (c) => ({ ...c, describeTimeoutMs: ms }),
1045
+ `Per-image description timeout set to ${ms} ms.`,
1046
+ );
1047
+ }
1048
+
1049
+ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
1050
+ if (!ctx.hasUI) {
1051
+ ctx.ui.notify("/vision-watcher requires interactive mode.", "error");
1052
+ return;
1053
+ }
1054
+
1055
+ // Only list models that are actually connected (configured auth via /login,
1056
+ // /better-custom, a models.json apiKey, or env/command keys) — the same set
1057
+ // the built-in /model picker shows. getAvailable() excludes the whole
1058
+ // catalogue of unauthenticated models.
1059
+ // Only vision-capable models can describe images — text-only models are hidden.
1060
+ const availableModels = ctx.modelRegistry
1061
+ .getAvailable()
1062
+ .map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input, reasoning: m.reasoning }))
1063
+ .filter((m) => isVisionModel(m));
1064
+
1065
+ if (ctx.mode !== "tui") {
1066
+ const modelItems = ["None", ...availableModels.map((m) => `${m.provider}/${m.id}`)];
1067
+ const modelPick = await ctx.ui.select("Vision model", modelItems);
1068
+ if (modelPick === undefined) return;
1069
+ const ref = modelPick === "None" ? null : modelPick;
1070
+ const thinkingPick = await ctx.ui.select("Thinking", ["on", "off"]);
1071
+ if (thinkingPick === undefined) return;
1072
+ const thinking = thinkingPick === "on";
1073
+ const thinkingLevel = config.thinkingLevel;
1074
+ const fallbackPick = await ctx.ui.select("Async pasted-path fallback", ["on", "off"]);
1075
+ if (fallbackPick === undefined) return;
1076
+ const asyncClipboardHandoff = fallbackPick === "on";
1077
+ const thinkingNote = thinking
1078
+ ? `thinking on (${thinkingLevel})${ref ? " \u2014 applies only if the vision model supports reasoning" : ""}`
1079
+ : "thinking off";
1080
+ updateConfig(
1081
+ ctx,
1082
+ (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel, asyncClipboardHandoff }),
1083
+ ref ? `Vision model set to ${ref} \u00b7 ${thinkingNote}` : `Vision model cleared \u00b7 ${thinkingNote}`,
1084
+ );
1085
+ if (!ref) {
1086
+ ctx.ui.notify("Handoff is inactive until you pick a vision model.", "warning");
1087
+ }
1088
+ return;
1089
+ }
1090
+
1091
+ const result = await ctx.ui.custom<VisionModelSelectorResult>((tui, theme, _kb, done) => {
1092
+ const selector = new VisionModelSelectorComponent(
1093
+ theme,
1094
+ availableModels,
1095
+ config.visionModel,
1096
+ config.thinking,
1097
+ config.thinkingLevel,
1098
+ config.asyncClipboardHandoff,
1099
+ (r) => done(r),
1100
+ );
1101
+ return {
1102
+ render(width: number) {
1103
+ return selector.render(width);
1104
+ },
1105
+ invalidate() {
1106
+ selector.invalidate();
1107
+ },
1108
+ handleInput(data: string) {
1109
+ selector.handleInput(data);
1110
+ tui.requestRender();
1111
+ },
1112
+ };
1113
+ });
1114
+
1115
+ if (!result || result.cancelled) {
1116
+ ctx.ui.notify("Vision handoff picker cancelled.", "info");
1117
+ return;
1118
+ }
1119
+
1120
+ const ref = result.ref;
1121
+ const thinking = result.thinking;
1122
+ const thinkingLevel = result.thinkingLevel;
1123
+ const asyncClipboardHandoff = result.asyncClipboardHandoff;
1124
+ // Fold the thinking state into the single updateConfig notify so the
1125
+ // "thinking off" message can't overwrite the model-change message.
1126
+ const thinkingNote = thinking
1127
+ ? `thinking on (${thinkingLevel})${ref ? " — applies only if the vision model supports reasoning" : ""}`
1128
+ : "thinking off";
1129
+ updateConfig(
1130
+ ctx,
1131
+ (c) => ({ ...c, visionModel: ref, thinking, thinkingLevel, asyncClipboardHandoff }),
1132
+ ref ? `Vision model set to ${ref} · ${thinkingNote}` : `Vision model cleared · ${thinkingNote}`,
1133
+ );
1134
+ if (!ref) {
1135
+ ctx.ui.notify("Handoff is inactive until you pick a vision model.", "warning");
1136
+ }
1137
+ }
1138
+
1139
+ function showStatus(ctx: ExtensionCommandContext): void {
1140
+ const lines: string[] = [];
1141
+ lines.push(`Vision handoff: ${config.enabled ? "enabled" : "disabled"}`);
1142
+ lines.push(`Vision model: ${config.visionModel ?? "(none — pick one with /vision-watcher)"}`);
1143
+ lines.push(`Auto handoff (non-vision models): ${config.autoHandoff ? "on" : "off"}`);
1144
+ lines.push(`Handoff targets (explicit): ${config.handoffModels.length ? config.handoffModels.join(", ") : "(none)"}`);
1145
+ lines.push(`Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}`);
1146
+ lines.push(`Paste-time prewarm: ${config.prewarmPastedImages ? `on${editorInstalled ? "" : " (inactive — another custom editor is active)"}` : "off"}`);
1147
+ lines.push(`Async pasted-path fallback: ${config.asyncClipboardHandoff ? "on" : "off"}`);
1148
+ lines.push(`Timeout (per image): ${config.describeTimeoutMs} ms`);
1149
+ lines.push(`maxTokens: ${config.maxTokens ?? "unbounded"} · cacheMax: ${config.cacheMax} · maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
1150
+
1151
+ const model = ctx.model;
1152
+ let active = false;
1153
+ if (isConfigured(config) && model) {
1154
+ active = isHandoffTarget(model, config);
1155
+ }
1156
+ lines.push(
1157
+ `Active for current model (${model ? formatModelRef(model.provider, model.id) : "none"}): ${active ? "yes" : "no"}`,
1158
+ );
1159
+
1160
+ ctx.ui.notify(lines.join("\n"), "info");
1161
+ }