@bismawy/pi-vision-watcher 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,583 +1,605 @@
1
- /**
2
- * Shared constants, types, and utilities for pi-vision-watcher.
3
- *
4
- * Config lives at ~/.pi/agent/extensions/pi-vision-watcher.json — the same
5
- * convention pi-model-sort uses for picker-backed extensions.
6
- */
7
-
8
- import type { ThinkingLevel } from "@earendil-works/pi-ai";
9
- import { getAgentDir } from "@earendil-works/pi-coding-agent";
10
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
11
- import { join } from "node:path";
12
-
13
- export * from "./usage.js";
14
-
15
- /** Subdirectory under the pi agent dir where picker extensions store config. */
16
- const CONFIG_SUBDIR = "extensions";
17
-
18
- /** Config file name. */
19
- export const CONFIG_FILENAME = "pi-vision-watcher.json";
20
-
21
- /** Full config path: ~/.pi/agent/extensions/pi-vision-watcher.json */
22
- export function getConfigPath(): string {
23
- return join(getAgentDir(), CONFIG_SUBDIR, CONFIG_FILENAME);
24
- }
25
-
26
- /** Description shown in the / commands list. */
27
- export const HANDOFF_COMMAND_DESCRIPTION =
28
- "Configure vision watcher — pick a vision model to describe images for text-only models";
29
-
30
- /** Default system prompt for the vision describer. Mirrors the pi-umans-provider pipeline. */
31
- export const DEFAULT_VISION_PROMPT =
32
- "You are a vision assistant for a coding agent. Describe this image exhaustively. Cover: all visible text (verbatim if possible), code snippets, UI layout and widgets, diagrams and flow arrows, error messages and stack traces, file trees, terminal output, color and style details, spatial relationships between elements, and anything else a developer would need to act on this image. Do not summarize — be exhaustive.";
33
-
34
- /** Prefix prepended to the user's original prompt when describing an image. */
35
- export const DEFAULT_USER_PROMPT_PREFIX = "The user's request about this image: ";
36
-
37
- /** Placeholder text block injected in place of an image block. */
38
- export const IMAGE_PLACEHOLDER_PREFIX = "[Image: ";
39
- export const IMAGE_PLACEHOLDER_SUFFIX = "]";
40
-
41
- /** Marker appended to a description whose `completeSimple()` call ended with
42
- * `stopReason: "length"` — i.e. the vision model hit a token limit (either the
43
- * configured `maxTokens` or the provider's hard output cap) before finishing.
44
- * A truncated description is still useful, but the agent must not mistake it
45
- * for the complete picture (it would reason as if the image contained only
46
- * what made it into the text). The marker makes the cut-off visible to both
47
- * the agent (it reads the marker in the tool result / context) and the user
48
- * (it renders inline). */
49
- export const DESCRIPTION_TRUNCATED_MARKER =
50
- "\n\n[... description truncated — the vision model reached its output token limit. The above may be incomplete; raise maxTokens or use a higher-output vision model for the full description.]";
51
-
52
- /** Append the {@link DESCRIPTION_TRUNCATED_MARKER} to a raw description. */
53
- export function markDescriptionTruncated(text: string): string {
54
- return text + DESCRIPTION_TRUNCATED_MARKER;
55
- }
56
-
57
- /** Default vision cache size (number of described images kept in memory per session). */
58
- export const DEFAULT_CACHE_MAX = 50;
59
-
60
- /** The pi thinking levels, ordered from lightest to deepest reasoning.
61
- * Mirrors `@earendil-works/pi-ai`'s `ThinkingLevel`; "off" is handled
62
- * separately via {@link VisionHandoffConfig.thinking} (a boolean on/off
63
- * switch) so the level persists across toggles. */
64
- export const THINKING_LEVELS: readonly ThinkingLevel[] = [
65
- "minimal",
66
- "low",
67
- "medium",
68
- "high",
69
- "xhigh",
70
- "max",
71
- ];
72
-
73
- /** Default thinking effort for the vision describer when thinking is enabled. */
74
- export const DEFAULT_THINKING_LEVEL: ThinkingLevel = "medium";
75
-
76
- /** Whether `level` is one of the supported {@link ThinkingLevel} values. */
77
- export function isThinkingLevel(level: unknown): level is ThinkingLevel {
78
- return typeof level === "string" && (THINKING_LEVELS as readonly string[]).includes(level);
79
- }
80
-
81
- /** Default per-image description timeout (ms). Kept modest so a slow vision
82
- * provider aborts early and fails over to {@link VisionHandoffConfig.fallbackModels}
83
- * instead of blocking the agent turn for minutes on a free-tier queue. A single
84
- * image commonly takes ~20s; raise via `describeTimeoutMs` in config only if
85
- * a specific model genuinely needs it. {@link describeTimeoutMs} scales this
86
- * per batch size. */
87
- export const DESCRIBE_TIMEOUT_MS = 45_000;
88
-
89
- /** Per-batch timeout: `baseTimeout` is the per-image budget, so the wall-clock
90
- * budget scales with the number of images in the call (a 5-image batch isn't
91
- * held to a single image's budget — the describer generates exhaustive prose
92
- * per image and websocket transport adds latency). `baseTimeout` defaults to
93
- * {@link DESCRIBE_TIMEOUT_MS} and is overridden by the configured
94
- * `describeTimeoutMs` field, so lowering it fails over sooner for the *whole*
95
- * batch. */
96
- export function describeTimeoutMs(imageCount: number, baseTimeout: number = DESCRIBE_TIMEOUT_MS): number {
97
- return baseTimeout * Math.max(1, imageCount);
98
- }
99
-
100
- /**
101
- * Default cap on the number of lines kept from a description before truncation.
102
- *
103
- * Mirrors pi core's read-result truncation: tool output is bounded for both the
104
- * TUI render and the model context. The stored `result.content` is the single
105
- * source for both surfaces (no decouple point exists at `tool_result` — it's the
106
- * only hook that still has the raw image before pi-ai strips it), so the cap
107
- * applies uniformly. 0 = unbounded (default): the read tool's native collapse
108
- * handles compactness and `ctrl+o` expands to the full description.
109
- */
110
- export const DEFAULT_MAX_DESCRIPTION_LINES = 0;
111
-
112
- /**
113
- * The note pi core's `read` tool appends to image-result text blocks when the
114
- * active model lacks image input (see `getNonVisionImageNote` in pi core).
115
- *
116
- * Once this extension inserts a description, the note becomes self-
117
- * contradicting ("image will be omitted" vs. the vivid description that
118
- * follows) and confuses the model. {@link stripNonVisionImageNote} removes it.
119
- */
120
- export const NON_VISION_IMAGE_NOTE =
121
- "[Current model does not support images. The image will be omitted from this request.]";
122
-
123
- /**
124
- * Remove {@link NON_VISION_IMAGE_NOTE} from a text block.
125
- *
126
- * The read tool appends the note as `\n${NOTE}` (always the trailing segment of
127
- * the metadata text block), so this also collapses the orphaned newline it
128
- * leaves behind. Safe to call on text that does not contain the note (no-op).
129
- */
130
- export function stripNonVisionImageNote(text: string): string {
131
- if (!text.includes(NON_VISION_IMAGE_NOTE)) return text;
132
- return text.split(NON_VISION_IMAGE_NOTE).join("").replace(/\n+$/, "");
133
- }
134
-
135
- export interface VisionHandoffConfig {
136
- /** Master switch. When false, no handoff occurs even if a vision model is configured. */
137
- enabled: boolean;
138
- /** The vision-capable model that describes images, as "provider/id". null = not configured. */
139
- visionModel: string | null;
140
- /** Optional fallback describers, tried in order when the primary vision
141
- * model's batched call fails entirely (auth, rate limit, timeout, network,
142
- * empty/error stopReason) and the same-model retry didn't recover.
143
- * Each entry is a "provider/id" ref; providers usually differ from the
144
- * primary so their rate limits / auth / availability are independent.
145
- * Empty (default) = no failover. */
146
- fallbackModels: string[];
147
- /** When true (default), handoff is applied to every model whose input does not include "image". */
148
- autoHandoff: boolean;
149
- /** Extra "provider/id" refs that should ALSO receive handoff (e.g. weak vision models). */
150
- handoffModels: string[];
151
- /** Opt-in: when true, the editor is wrapped at session start to describe
152
- * pasted clipboard images the instant their temp-file path lands in the
153
- * prompt (pre-submit), instead of waiting for submit. Trades a bit of
154
- * description quality (the description is generated without the user's
155
- * typed question as context, since the question usually isn't entered yet
156
- * at paste time) and a speculative vision call on paste-then-abandon, for
157
- * earlier prewarm. Off by default — submit-time prewarm
158
- * (before_agent_start) still covers this case when off. TUI mode only, and
159
- * inactive when another extension has replaced the editor (installing over
160
- * a custom editor would break clipboard paste, since pi wires paste-image
161
- * to the outermost editor only). */
162
- prewarmPastedImages: boolean;
163
- /** Opt-in: asynchronously inject descriptions for pasted clipboard paths when
164
- * the target model does not read them. The normal read/tool_result path wins
165
- * the race: a matching read cancels the queued fallback injection while
166
- * reusing the same in-flight description. */
167
- asyncClipboardHandoff: boolean;
168
- /** Max output tokens for a single description. `undefined` (default) = use the
169
- * vision model's declared max output (`model.maxTokens`) as the cap — the
170
- * highest the model supports — so the "be exhaustive" prompt isn't defeated
171
- * by a provider's small omitted-default. Set a number only to cap lower.
172
- * A truncation is always surfaced via {@link DESCRIPTION_TRUNCATED_MARKER}
173
- * when the model hits the limit. */
174
- maxTokens?: number;
175
- /** Base per-image description timeout in ms (defaults to
176
- * {@link DESCRIBE_TIMEOUT_MS}). The per-batch timeout scales from this base
177
- * by image count. Lower it to fail over to {@link fallbackModels} sooner on a
178
- * slow primary; raise it for a high-latency vision model that times out
179
- * spuriously. */
180
- describeTimeoutMs: number;
181
- /** Max images kept in the in-memory description cache. */
182
- cacheMax: number;
183
- /**
184
- * Max lines kept from a description before truncation (0 = unbounded).
185
- * Bounds both the TUI render and the model context, like pi core's tool-output
186
- * truncation.
187
- */
188
- maxDescriptionLines: number;
189
- /** Override the describer system prompt (defaults to DEFAULT_VISION_PROMPT). */
190
- prompt?: string;
191
- /** Override the user-prompt prefix (defaults to DEFAULT_USER_PROMPT_PREFIX). */
192
- userPromptPrefix?: string;
193
- /** Whether the vision describer should reason (think) before describing.
194
- * Off by default — describing images is a perception task, not a reasoning
195
- * one, and thinking adds latency + cost. When on, the level below is sent
196
- * to the vision model via pi-ai's `reasoning` option (only honoured when
197
- * the configured vision model declares `reasoning: true`). */
198
- thinking: boolean;
199
- /** Thinking effort for the describer when {@link thinking} is on. One of
200
- * {@link THINKING_LEVELS}. Ignored when {@link thinking} is off or the
201
- * vision model lacks reasoning support. */
202
- thinkingLevel: ThinkingLevel;
203
- }
204
-
205
- export const DEFAULT_CONFIG: VisionHandoffConfig = {
206
- enabled: true,
207
- visionModel: null,
208
- fallbackModels: [],
209
- autoHandoff: true,
210
- handoffModels: [],
211
- prewarmPastedImages: false,
212
- asyncClipboardHandoff: false,
213
- maxTokens: undefined,
214
- describeTimeoutMs: DESCRIBE_TIMEOUT_MS,
215
- cacheMax: DEFAULT_CACHE_MAX,
216
- maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
217
- thinking: false,
218
- thinkingLevel: DEFAULT_THINKING_LEVEL,
219
- };
220
-
221
- /** Parse a "provider/id" reference. Returns null if malformed. */
222
- export function parseModelRef(ref: string): { provider: string; id: string } | null {
223
- const trimmed = ref.trim();
224
- if (!trimmed) return null;
225
- const slashIndex = trimmed.indexOf("/");
226
- if (slashIndex <= 0) return null; // no slash, or empty provider
227
- const provider = trimmed.slice(0, slashIndex);
228
- const id = trimmed.slice(slashIndex + 1);
229
- if (!provider || !id) return null;
230
- return { provider, id };
231
- }
232
-
233
- /** Format a provider/id reference string. */
234
- export function formatModelRef(provider: string, id: string): string {
235
- return `${provider}/${id}`;
236
- }
237
-
238
- /** Match provider errors meaning "this model can't take image input" — e.g.
239
- * AgentRouter's 400 `"This model does not support image"`, OpenRouter's
240
- * "does not support image inputs". Used by the message_end auto-recovery:
241
- * a model whose registry entry FALSELY declares `input: ["text","image"]`
242
- * passes the autoHandoff vision check, so the raw image reaches the provider
243
- * and 400s — we learn from that error and force handoff for the model. */
244
- export function isImageNotSupportedError(text: string): boolean {
245
- return /not support (?:the )?image|image[s]? (?:input[s]? )?(?:is |are )?not supported|does not accept image|unsupported (?:image|multimodal)|only (?:supports|accepts) text|unsupported content type[\s\S]{0,40}image/i.test(
246
- text,
247
- );
248
- }
249
-
250
- /** Models whose registry entries commonly declare image input while the
251
- * backend rejects images (DeepSeek V4, GLM 4/5 non-V via aggregator
252
- * proxies). Conservative: VL / Vision / Janus / *V variants keep image
253
- * input. Used so autoHandoff covers them on the FIRST send — waiting for a
254
- * 400 is too late. */
255
- export function isKnownTextOnlyFalselyVision(id: string | undefined | null): boolean {
256
- if (!id) return false;
257
- const n = id.toLowerCase();
258
- if (n.includes("vl") || n.includes("vision") || n.includes("janus")) return false;
259
- if (/\dv(?=[-_./\s]|$)/.test(n)) return false; // glm-4v, glm-4.5v — vision variants
260
- if (/deepseek[-_./]*v4/.test(n)) return true;
261
- if (/glm[-_./]*[45]/.test(n)) return true;
262
- return false;
263
- }
264
-
265
- /** Extract the provider error string from a finalized assistant message.
266
- * Pi stores it on `errorMessage`; some providers also put the 400 body in a
267
- * text content block. */
268
- export function assistantErrorText(msg: {
269
- errorMessage?: unknown;
270
- content?: unknown;
271
- } | null | undefined): string {
272
- if (!msg) return "";
273
- if (typeof msg.errorMessage === "string" && msg.errorMessage) return msg.errorMessage;
274
- if (typeof msg.content === "string") return msg.content;
275
- if (!Array.isArray(msg.content)) return "";
276
- return msg.content
277
- .filter((b): b is { type: string; text: string } => !!b && typeof b === "object" && typeof (b as { text?: unknown }).text === "string")
278
- .map((b) => b.text)
279
- .join("\n");
280
- }
281
-
282
- /** Set (or add) a `modelOverrides` entry forcing a model's input to
283
- * `["text"]` in a PARSED models.json config object. Pure: returns a new
284
- * object, never mutates the input. Preserves every other field (credentials,
285
- * compat, sibling models) — only the one model's `input` override is touched.
286
- * Used by the message_end auto-recovery to fix models whose registry entry
287
- * falsely declares image input the backend rejects with a 400. */
288
- export function setTextOnlyInputOverride(
289
- cfg: unknown,
290
- providerId: string,
291
- modelId: string,
292
- ): Record<string, unknown> {
293
- const root =
294
- cfg && typeof cfg === "object" && !Array.isArray(cfg)
295
- ? { ...(cfg as Record<string, unknown>) }
296
- : {};
297
- const providers =
298
- root.providers && typeof root.providers === "object" && !Array.isArray(root.providers)
299
- ? { ...(root.providers as Record<string, unknown>) }
300
- : {};
301
- const provider =
302
- providers[providerId] && typeof providers[providerId] === "object" && !Array.isArray(providers[providerId])
303
- ? { ...(providers[providerId] as Record<string, unknown>) }
304
- : {};
305
- const overrides =
306
- provider.modelOverrides && typeof provider.modelOverrides === "object" && !Array.isArray(provider.modelOverrides)
307
- ? { ...(provider.modelOverrides as Record<string, unknown>) }
308
- : {};
309
- const existing =
310
- overrides[modelId] && typeof overrides[modelId] === "object" && !Array.isArray(overrides[modelId])
311
- ? { ...(overrides[modelId] as Record<string, unknown>) }
312
- : {};
313
- overrides[modelId] = { ...existing, input: ["text"] };
314
- provider.modelOverrides = overrides;
315
- providers[providerId] = provider;
316
- root.providers = providers;
317
- return root;
318
- }
319
-
320
- /** Whether a model declares image input. */
321
- export function isVisionModel(model: { input?: ("text" | "image")[] } | undefined | null): boolean {
322
- return !!model && Array.isArray(model.input) && model.input.includes("image");
323
- }
324
-
325
- /** Merge a parsed config object onto defaults, tolerating missing/invalid fields. */
326
- export function normalizeConfig(raw: unknown): VisionHandoffConfig {
327
- const base: VisionHandoffConfig = { ...DEFAULT_CONFIG };
328
- if (!raw || typeof raw !== "object") return base;
329
- const obj = raw as Record<string, unknown>;
330
-
331
- if (typeof obj.enabled === "boolean") base.enabled = obj.enabled;
332
- if (typeof obj.visionModel === "string" && obj.visionModel.trim()) {
333
- base.visionModel = parseModelRef(obj.visionModel) ? obj.visionModel.trim() : null;
334
- } else if (obj.visionModel === null) {
335
- base.visionModel = null;
336
- }
337
- if (Array.isArray(obj.fallbackModels)) {
338
- base.fallbackModels = obj.fallbackModels
339
- .filter((m): m is string => typeof m === "string")
340
- .map((m) => m.trim())
341
- .filter((m) => m && parseModelRef(m));
342
- }
343
- if (typeof obj.autoHandoff === "boolean") base.autoHandoff = obj.autoHandoff;
344
-
345
- if (Array.isArray(obj.handoffModels)) {
346
- base.handoffModels = obj.handoffModels
347
- .filter((m): m is string => typeof m === "string")
348
- .map((m) => m.trim())
349
- .filter((m) => m && parseModelRef(m));
350
- }
351
- if (typeof obj.prewarmPastedImages === "boolean") base.prewarmPastedImages = obj.prewarmPastedImages;
352
- if (typeof obj.asyncClipboardHandoff === "boolean") {
353
- base.asyncClipboardHandoff = obj.asyncClipboardHandoff;
354
- }
355
- // maxTokens: optional. undefined (default) = no artificial cap. Only set when
356
- // a valid positive finite number is given; any other value leaves it unset.
357
- if (typeof obj.maxTokens === "number" && Number.isFinite(obj.maxTokens) && obj.maxTokens > 0) {
358
- base.maxTokens = Math.floor(obj.maxTokens);
359
- }
360
- if (
361
- typeof obj.describeTimeoutMs === "number" &&
362
- Number.isFinite(obj.describeTimeoutMs) &&
363
- obj.describeTimeoutMs > 0
364
- ) {
365
- base.describeTimeoutMs = Math.floor(obj.describeTimeoutMs);
366
- }
367
- if (typeof obj.cacheMax === "number" && Number.isFinite(obj.cacheMax) && obj.cacheMax > 0) {
368
- base.cacheMax = Math.floor(obj.cacheMax);
369
- }
370
- // maxDescriptionLines: any non-negative finite integer. 0 = unbounded.
371
- if (
372
- typeof obj.maxDescriptionLines === "number" &&
373
- Number.isFinite(obj.maxDescriptionLines) &&
374
- obj.maxDescriptionLines >= 0
375
- ) {
376
- base.maxDescriptionLines = Math.floor(obj.maxDescriptionLines);
377
- }
378
- if (typeof obj.prompt === "string" && obj.prompt.trim()) base.prompt = obj.prompt;
379
- if (typeof obj.userPromptPrefix === "string") base.userPromptPrefix = obj.userPromptPrefix;
380
-
381
- if (typeof obj.thinking === "boolean") base.thinking = obj.thinking;
382
- if (isThinkingLevel(obj.thinkingLevel)) base.thinkingLevel = obj.thinkingLevel;
383
-
384
- return base;
385
- }
386
-
387
- /** Read config from disk (falls back to defaults on missing/corrupt file). */
388
- export function readConfig(): VisionHandoffConfig {
389
- const path = getConfigPath();
390
- if (!existsSync(path)) return { ...DEFAULT_CONFIG };
391
- try {
392
- const raw = readFileSync(path, "utf8");
393
- return normalizeConfig(JSON.parse(raw));
394
- } catch {
395
- return { ...DEFAULT_CONFIG };
396
- }
397
- }
398
-
399
- /** Write config to disk. Creates the directory if needed. Returns the path written. */
400
- export function writeConfig(config: VisionHandoffConfig): string {
401
- const path = getConfigPath();
402
- const dir = join(getAgentDir(), CONFIG_SUBDIR);
403
- if (!existsSync(dir)) {
404
- mkdirSync(dir, { recursive: true });
405
- }
406
- writeFileSync(path, JSON.stringify(config, null, 2) + "\n", "utf8");
407
- return path;
408
- }
409
-
410
- export interface ExtractedImage {
411
- data: string;
412
- mimeType: string;
413
- }
414
-
415
- /** Parse a `data:<mime>;base64,<data>` URL into raw base64 + mime. Returns null if not a data URL. */
416
- export function parseDataUrl(url: string): ExtractedImage | null {
417
- const match = /^data:([^;,]+)?(?:;base64)?,(.*)$/s.exec(url);
418
- if (!match) return null;
419
- return { mimeType: match[1] || "image/png", data: match[2] };
420
- }
421
-
422
- /**
423
- * Detect an image block by shape across the three request formats pi uses:
424
- * openai-completions: { type: "image_url", image_url: { url: "data:..." } }
425
- * openai-responses: { type: "input_image", image_url: "data:..." | { url } }
426
- * anthropic-messages: { type: "image", source: { type: "base64", media_type, data } }
427
- */
428
- export function extractImageFromBlock(block: unknown): ExtractedImage | null {
429
- if (!block || typeof block !== "object") return null;
430
- const b = block as Record<string, any>;
431
-
432
- if (b.type === "image_url" && typeof b.image_url?.url === "string") {
433
- return parseDataUrl(b.image_url.url);
434
- }
435
-
436
- if (b.type === "input_image") {
437
- const url = typeof b.image_url === "string" ? b.image_url : b.image_url?.url;
438
- if (typeof url === "string") return parseDataUrl(url);
439
- return null;
440
- }
441
-
442
- // anthropic-messages image block (wrapped in `source`).
443
- if (b.type === "image" && b.source?.type === "base64" && typeof b.source.data === "string") {
444
- return { data: b.source.data, mimeType: b.source.media_type || "image/png" };
445
- }
446
-
447
- // pi-ai internal image block — emitted by the `read` tool and carried by
448
- // ToolResultEvent.content / user-message content blocks (regardless of whether
449
- // the active model declares image input):
450
- // { type: "image", data: "<base64>", mimeType }
451
- // Distinct from the anthropic shape above: no `source` wrapper, direct fields.
452
- if (b.type === "image" && typeof b.data === "string") {
453
- return { data: b.data, mimeType: b.mimeType || "image/png" };
454
- }
455
-
456
- return null;
457
- }
458
-
459
- /** Outcome of {@link truncateDescription}. */
460
- export interface TruncatedDescription {
461
- text: string;
462
- /** True iff lines were dropped. */
463
- truncated: boolean;
464
- /** Number of trailing lines removed. */
465
- hidden: number;
466
- }
467
-
468
- /**
469
- * Head-truncate a description to its first `maxLines` lines with a
470
- * "... (N more lines)" footer, mirroring pi core's read-result truncation
471
- * notice (which itself surfaces "... (N more lines, ctrl+o to expand)").
472
- *
473
- * Returns the text unchanged when it is at or below the limit, or when
474
- * `maxLines` is non-positive (0 = unbounded). Head-truncation keeps the leading
475
- * layout / text-content / structure sections, which are typically the most
476
- * actionable for a coding agent.
477
- */
478
- export function truncateDescription(text: string, maxLines: number): TruncatedDescription {
479
- if (!maxLines || maxLines <= 0) {
480
- return { text, truncated: false, hidden: 0 };
481
- }
482
- const lines = text.split("\n");
483
- if (lines.length <= maxLines) {
484
- return { text, truncated: false, hidden: 0 };
485
- }
486
- const hidden = lines.length - maxLines;
487
- const kept = lines.slice(0, maxLines).join("\n");
488
- return { text: `${kept}\n... (${hidden} more lines)`, truncated: true, hidden };
489
- }
490
-
491
- /** Start marker for the k-th image section in a batched describer response. */
492
- export const BATCH_IMAGE_MARKER_START = "<<<IMAGE";
493
- /** End marker closing a single image section in a batched describer response. */
494
- export const BATCH_IMAGE_MARKER_END = "<<<END>>>";
495
-
496
- /** Escape a literal string for use in a RegExp. */
497
- function escapeRe(s: string): string {
498
- return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
499
- }
500
-
501
- // Section regexes built from the marker constants so the producer
502
- // (batchUserPrompt) and consumer (parseBatchedDescriptions) share one source of
503
- // truth for the delimiter format. A lone `<<<IMAGE k>>> … <<<END>>>` wrapper
504
- // for a single image; a global scan for the multi-image case that stops at the
505
- // next start marker, the end marker, or end-of-text.
506
- const SECTION_SINGLE_RE = new RegExp(
507
- `${escapeRe(BATCH_IMAGE_MARKER_START)}\\s+1>>>\\s*([\\s\\S]*?)${escapeRe(BATCH_IMAGE_MARKER_END)}`,
508
- );
509
- const SECTION_MULTI_RE = new RegExp(
510
- `${escapeRe(BATCH_IMAGE_MARKER_START)}\\s+(\\d+)>>>\\s*([\\s\\S]*?)(?=${escapeRe(
511
- BATCH_IMAGE_MARKER_START,
512
- )}\\s+\\d+>>>|${escapeRe(BATCH_IMAGE_MARKER_END)}|$)`,
513
- "g",
514
- );
515
- const SECTION_END_STRIP_RE = new RegExp(`${escapeRe(BATCH_IMAGE_MARKER_END)}\\s*$`);
516
-
517
- /** Build the user-message text block for a (possibly batched) describer call.
518
- *
519
- * For a single image (count <= 1) this returns the same simple prompt the
520
- * original per-image pipeline used (no markers), so single-image behaviour is
521
- * unchanged. For count > 1 it instructs the vision model to emit one delimited
522
- * `<<<IMAGE k>>> … <<<END>>>` section per image, in order, so the response can
523
- * be split back into per-image descriptions by {@link parseBatchedDescriptions}.
524
- *
525
- * The user's turn prompt (if any) is folded in via {@link DEFAULT_USER_PROMPT_PREFIX}
526
- * so every image in the batch is described in the context of the same request —
527
- * one describer call covers the whole prompt's image set, dataloader-style,
528
- * rather than spinning up one agent call per image. */
529
- export function batchUserPrompt(count: number, userPrompt: string, prefix: string): string {
530
- const userLine = userPrompt && userPrompt.trim() ? `${prefix}${userPrompt}` : "";
531
- if (count <= 1) {
532
- return userLine || "Describe this image.";
533
- }
534
- return [
535
- `Describe each of the ${count} images below exhaustively, in order. For image k (1 through ${count}), emit exactly:`,
536
- "",
537
- `${BATCH_IMAGE_MARKER_START} k>>>`,
538
- "<your exhaustive description>",
539
- BATCH_IMAGE_MARKER_END,
540
- ...(userLine ? ["", userLine] : []),
541
- ].join("\n");
542
- }
543
-
544
- /** Parse a batched vision-model response into one description per image, in order.
545
- *
546
- * Returns an array of length `count`; each slot is the trimmed description for
547
- * image k (1-indexed → array index k-1), or `null` when that image's section is
548
- * missing, empty, or unparseable. Callers treat `null` as "description
549
- * unavailable" (graceful degradation) — one image failing to parse must not
550
- * void the rest of the batch.
551
- *
552
- * For `count <= 1` the whole response is treated as the single image's
553
- * description (a lone `<<<IMAGE 1>>> … <<<END>>>` wrapper is stripped if the
554
- * model emitted one anyway), preserving the original single-image behaviour. */
555
- export function parseBatchedDescriptions(text: string, count: number): (string | null)[] {
556
- const trimmed = (text ?? "").trim();
557
- if (count <= 1) {
558
- const m = SECTION_SINGLE_RE.exec(trimmed);
559
- const body = (m ? m[1] : trimmed).trim();
560
- return [body || null];
561
- }
562
- const out: (string | null)[] = new Array(count).fill(null);
563
- let m: RegExpExecArray | null;
564
- SECTION_MULTI_RE.lastIndex = 0;
565
- while ((m = SECTION_MULTI_RE.exec(trimmed)) !== null) {
566
- const idx = parseInt(m[1], 10) - 1;
567
- const body = m[2].replace(SECTION_END_STRIP_RE, "").trim();
568
- if (idx >= 0 && idx < count && !out[idx]) out[idx] = body || null;
569
- }
570
- return out;
571
- }
572
-
573
- /** Truncate (if configured) and wrap a raw description in the `[Image: …]`
574
- * placeholder envelope used by every insertion path. Centralises the wrap+
575
- * truncate logic shared by the read-tool and context injection paths so both
576
- * surfaces stay identical for the same image hash. */
577
- export function wrapDescription(description: string, cfg: VisionHandoffConfig): string {
578
- const { text: final } =
579
- cfg.maxDescriptionLines && cfg.maxDescriptionLines > 0
580
- ? truncateDescription(description, cfg.maxDescriptionLines)
581
- : { text: description };
582
- return `${IMAGE_PLACEHOLDER_PREFIX}${final}${IMAGE_PLACEHOLDER_SUFFIX}`;
583
- }
1
+ /**
2
+ * Shared constants, types, and utilities for pi-vision-watcher.
3
+ *
4
+ * Config lives at ~/.pi/agent/extensions/pi-vision-watcher.json — the same
5
+ * convention pi-model-sort uses for picker-backed extensions.
6
+ */
7
+
8
+ import type { ThinkingLevel } from "@earendil-works/pi-ai";
9
+ import { getAgentDir } from "@earendil-works/pi-coding-agent";
10
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
11
+ import { join } from "node:path";
12
+
13
+ export * from "./usage.js";
14
+
15
+ /** Subdirectory under the pi agent dir where picker extensions store config. */
16
+ const CONFIG_SUBDIR = "extensions";
17
+
18
+ /** Config file name. */
19
+ export const CONFIG_FILENAME = "pi-vision-watcher.json";
20
+
21
+ /** Full config path: ~/.pi/agent/extensions/pi-vision-watcher.json */
22
+ export function getConfigPath(): string {
23
+ return join(getAgentDir(), CONFIG_SUBDIR, CONFIG_FILENAME);
24
+ }
25
+
26
+ /** Description shown in the / commands list. */
27
+ export const HANDOFF_COMMAND_DESCRIPTION =
28
+ "Configure vision watcher — pick a vision model to describe images for text-only models";
29
+
30
+ /** Default system prompt for the vision describer. Mirrors the pi-umans-provider pipeline. */
31
+ export const DEFAULT_VISION_PROMPT =
32
+ "You are a vision assistant for a coding agent. Describe this image exhaustively. Cover: all visible text (verbatim if possible), code snippets, UI layout and widgets, diagrams and flow arrows, error messages and stack traces, file trees, terminal output, color and style details, spatial relationships between elements, and anything else a developer would need to act on this image. Do not summarize — be exhaustive.";
33
+
34
+ /** Prefix prepended to the user's original prompt when describing an image. */
35
+ export const DEFAULT_USER_PROMPT_PREFIX = "The user's request about this image: ";
36
+
37
+ /** Placeholder text block injected in place of an image block. */
38
+ export const IMAGE_PLACEHOLDER_PREFIX = "[Image: ";
39
+ export const IMAGE_PLACEHOLDER_SUFFIX = "]";
40
+
41
+ /** Marker appended to a description whose `completeSimple()` call ended with
42
+ * `stopReason: "length"` — i.e. the vision model hit a token limit (either the
43
+ * configured `maxTokens` or the provider's hard output cap) before finishing.
44
+ * A truncated description is still useful, but the agent must not mistake it
45
+ * for the complete picture (it would reason as if the image contained only
46
+ * what made it into the text). The marker makes the cut-off visible to both
47
+ * the agent (it reads the marker in the tool result / context) and the user
48
+ * (it renders inline). */
49
+ export const DESCRIPTION_TRUNCATED_MARKER =
50
+ "\n\n[... description truncated — the vision model reached its output token limit. The above may be incomplete; raise maxTokens or use a higher-output vision model for the full description.]";
51
+
52
+ /** Append the {@link DESCRIPTION_TRUNCATED_MARKER} to a raw description. */
53
+ export function markDescriptionTruncated(text: string): string {
54
+ return text + DESCRIPTION_TRUNCATED_MARKER;
55
+ }
56
+
57
+ /** Default vision cache size (number of described images kept in memory per session). */
58
+ export const DEFAULT_CACHE_MAX = 50;
59
+
60
+ /** The pi thinking levels, ordered from lightest to deepest reasoning.
61
+ * Mirrors `@earendil-works/pi-ai`'s `ThinkingLevel`; "off" is handled
62
+ * separately via {@link VisionHandoffConfig.thinking} (a boolean on/off
63
+ * switch) so the level persists across toggles. */
64
+ export const THINKING_LEVELS: readonly ThinkingLevel[] = [
65
+ "minimal",
66
+ "low",
67
+ "medium",
68
+ "high",
69
+ "xhigh",
70
+ "max",
71
+ ];
72
+
73
+ /** Default thinking effort for the vision describer when thinking is enabled. */
74
+ export const DEFAULT_THINKING_LEVEL: ThinkingLevel = "medium";
75
+
76
+ /** Whether `level` is one of the supported {@link ThinkingLevel} values. */
77
+ export function isThinkingLevel(level: unknown): level is ThinkingLevel {
78
+ return typeof level === "string" && (THINKING_LEVELS as readonly string[]).includes(level);
79
+ }
80
+
81
+ /** Default per-image description timeout (ms). Kept modest so a slow vision
82
+ * provider aborts early and fails over to {@link VisionHandoffConfig.fallbackModels}
83
+ * instead of blocking the agent turn for minutes on a free-tier queue. A single
84
+ * image commonly takes ~20s; raise via `describeTimeoutMs` in config only if
85
+ * a specific model genuinely needs it. {@link describeTimeoutMs} scales this
86
+ * per batch size. */
87
+ export const DESCRIBE_TIMEOUT_MS = 45_000;
88
+
89
+ /** Per-batch timeout: `baseTimeout` is the per-image budget, so the wall-clock
90
+ * budget scales with the number of images in the call (a 5-image batch isn't
91
+ * held to a single image's budget — the describer generates exhaustive prose
92
+ * per image and websocket transport adds latency). `baseTimeout` defaults to
93
+ * {@link DESCRIBE_TIMEOUT_MS} and is overridden by the configured
94
+ * `describeTimeoutMs` field, so lowering it fails over sooner for the *whole*
95
+ * batch. */
96
+ export function describeTimeoutMs(imageCount: number, baseTimeout: number = DESCRIBE_TIMEOUT_MS): number {
97
+ return baseTimeout * Math.max(1, imageCount);
98
+ }
99
+
100
+ /** Default byte budget for ONE describer request body, images included.
101
+ *
102
+ * Aggregator proxies cap the whole JSON body (text plus base64 images), not the
103
+ * model context: Remote Magpie answers a 1.1 MB body with HTTP 413. A base64
104
+ * image is 4/3 its raw size, so one 2000px screenshot can exceed that cap alone
105
+ * — and a batched call multiplies it by the image count. The describer splits a
106
+ * batch into groups that each fit this budget, so a multi-image read degrades
107
+ * into a few calls instead of one 413 that loses every description. Override it
108
+ * with `maxRequestBytes` when the provider takes a different limit; `0` keeps
109
+ * the old always-one-call behaviour. */
110
+ export const DEFAULT_MAX_REQUEST_BYTES = 700_000;
111
+
112
+ /**
113
+ * Default cap on the number of lines kept from a description before truncation.
114
+ *
115
+ * Mirrors pi core's read-result truncation: tool output is bounded for both the
116
+ * TUI render and the model context. The stored `result.content` is the single
117
+ * source for both surfaces (no decouple point exists at `tool_result` — it's the
118
+ * only hook that still has the raw image before pi-ai strips it), so the cap
119
+ * applies uniformly. 0 = unbounded (default): the read tool's native collapse
120
+ * handles compactness and `ctrl+o` expands to the full description.
121
+ */
122
+ export const DEFAULT_MAX_DESCRIPTION_LINES = 0;
123
+
124
+ /**
125
+ * The note pi core's `read` tool appends to image-result text blocks when the
126
+ * active model lacks image input (see `getNonVisionImageNote` in pi core).
127
+ *
128
+ * Once this extension inserts a description, the note becomes self-
129
+ * contradicting ("image will be omitted" vs. the vivid description that
130
+ * follows) and confuses the model. {@link stripNonVisionImageNote} removes it.
131
+ */
132
+ export const NON_VISION_IMAGE_NOTE =
133
+ "[Current model does not support images. The image will be omitted from this request.]";
134
+
135
+ /**
136
+ * Remove {@link NON_VISION_IMAGE_NOTE} from a text block.
137
+ *
138
+ * The read tool appends the note as `\n${NOTE}` (always the trailing segment of
139
+ * the metadata text block), so this also collapses the orphaned newline it
140
+ * leaves behind. Safe to call on text that does not contain the note (no-op).
141
+ */
142
+ export function stripNonVisionImageNote(text: string): string {
143
+ if (!text.includes(NON_VISION_IMAGE_NOTE)) return text;
144
+ return text.split(NON_VISION_IMAGE_NOTE).join("").replace(/\n+$/, "");
145
+ }
146
+
147
+ export interface VisionHandoffConfig {
148
+ /** Master switch. When false, no handoff occurs even if a vision model is configured. */
149
+ enabled: boolean;
150
+ /** The vision-capable model that describes images, as "provider/id". null = not configured. */
151
+ visionModel: string | null;
152
+ /** Optional fallback describers, tried in order when the primary vision
153
+ * model's batched call fails entirely (auth, rate limit, timeout, network,
154
+ * empty/error stopReason) and the same-model retry didn't recover.
155
+ * Each entry is a "provider/id" ref; providers usually differ from the
156
+ * primary so their rate limits / auth / availability are independent.
157
+ * Empty (default) = no failover. */
158
+ fallbackModels: string[];
159
+ /** When true (default), handoff is applied to every model whose input does not include "image". */
160
+ autoHandoff: boolean;
161
+ /** Extra "provider/id" refs that should ALSO receive handoff (e.g. weak vision models). */
162
+ handoffModels: string[];
163
+ /** Opt-in: when true, the editor is wrapped at session start to describe
164
+ * pasted clipboard images the instant their temp-file path lands in the
165
+ * prompt (pre-submit), instead of waiting for submit. Trades a bit of
166
+ * description quality (the description is generated without the user's
167
+ * typed question as context, since the question usually isn't entered yet
168
+ * at paste time) and a speculative vision call on paste-then-abandon, for
169
+ * earlier prewarm. Off by default — submit-time prewarm
170
+ * (before_agent_start) still covers this case when off. TUI mode only, and
171
+ * inactive when another extension has replaced the editor (installing over
172
+ * a custom editor would break clipboard paste, since pi wires paste-image
173
+ * to the outermost editor only). */
174
+ prewarmPastedImages: boolean;
175
+ /** Opt-in: asynchronously inject descriptions for pasted clipboard paths when
176
+ * the target model does not read them. The normal read/tool_result path wins
177
+ * the race: a matching read cancels the queued fallback injection while
178
+ * reusing the same in-flight description. */
179
+ asyncClipboardHandoff: boolean;
180
+ /** Max output tokens for a single description. `undefined` (default) = use the
181
+ * vision model's declared max output (`model.maxTokens`) as the cap — the
182
+ * highest the model supports — so the "be exhaustive" prompt isn't defeated
183
+ * by a provider's small omitted-default. Set a number only to cap lower.
184
+ * A truncation is always surfaced via {@link DESCRIPTION_TRUNCATED_MARKER}
185
+ * when the model hits the limit. */
186
+ maxTokens?: number;
187
+ /** Base per-image description timeout in ms (defaults to
188
+ * {@link DESCRIBE_TIMEOUT_MS}). The per-batch timeout scales from this base
189
+ * by image count. Lower it to fail over to {@link fallbackModels} sooner on a
190
+ * slow primary; raise it for a high-latency vision model that times out
191
+ * spuriously. */
192
+ describeTimeoutMs: number;
193
+ /** Byte budget for ONE describer request body (prompt text + base64 images).
194
+ * Images are grouped into as few calls as fit this budget; a single image
195
+ * above it still gets its own call. `0` disables splitting. Defaults to
196
+ * {@link DEFAULT_MAX_REQUEST_BYTES}. */
197
+ maxRequestBytes: number;
198
+ /** Max images kept in the in-memory description cache. */
199
+ cacheMax: number;
200
+ /**
201
+ * Max lines kept from a description before truncation (0 = unbounded).
202
+ * Bounds both the TUI render and the model context, like pi core's tool-output
203
+ * truncation.
204
+ */
205
+ maxDescriptionLines: number;
206
+ /** Override the describer system prompt (defaults to DEFAULT_VISION_PROMPT). */
207
+ prompt?: string;
208
+ /** Override the user-prompt prefix (defaults to DEFAULT_USER_PROMPT_PREFIX). */
209
+ userPromptPrefix?: string;
210
+ /** Whether the vision describer should reason (think) before describing.
211
+ * Off by default — describing images is a perception task, not a reasoning
212
+ * one, and thinking adds latency + cost. When on, the level below is sent
213
+ * to the vision model via pi-ai's `reasoning` option (only honoured when
214
+ * the configured vision model declares `reasoning: true`). */
215
+ thinking: boolean;
216
+ /** Thinking effort for the describer when {@link thinking} is on. One of
217
+ * {@link THINKING_LEVELS}. Ignored when {@link thinking} is off or the
218
+ * vision model lacks reasoning support. */
219
+ thinkingLevel: ThinkingLevel;
220
+ }
221
+
222
+ export const DEFAULT_CONFIG: VisionHandoffConfig = {
223
+ enabled: true,
224
+ visionModel: null,
225
+ fallbackModels: [],
226
+ autoHandoff: true,
227
+ handoffModels: [],
228
+ prewarmPastedImages: false,
229
+ asyncClipboardHandoff: false,
230
+ maxTokens: undefined,
231
+ describeTimeoutMs: DESCRIBE_TIMEOUT_MS,
232
+ maxRequestBytes: DEFAULT_MAX_REQUEST_BYTES,
233
+ cacheMax: DEFAULT_CACHE_MAX,
234
+ maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
235
+ thinking: false,
236
+ thinkingLevel: DEFAULT_THINKING_LEVEL,
237
+ };
238
+
239
+ /** Parse a "provider/id" reference. Returns null if malformed. */
240
+ export function parseModelRef(ref: string): { provider: string; id: string } | null {
241
+ const trimmed = ref.trim();
242
+ if (!trimmed) return null;
243
+ const slashIndex = trimmed.indexOf("/");
244
+ if (slashIndex <= 0) return null; // no slash, or empty provider
245
+ const provider = trimmed.slice(0, slashIndex);
246
+ const id = trimmed.slice(slashIndex + 1);
247
+ if (!provider || !id) return null;
248
+ return { provider, id };
249
+ }
250
+
251
+ /** Format a provider/id reference string. */
252
+ export function formatModelRef(provider: string, id: string): string {
253
+ return `${provider}/${id}`;
254
+ }
255
+
256
+ /** Match provider errors meaning "this model can't take image input" — e.g.
257
+ * AgentRouter's 400 `"This model does not support image"`, OpenRouter's
258
+ * "does not support image inputs". Used by the message_end auto-recovery:
259
+ * a model whose registry entry FALSELY declares `input: ["text","image"]`
260
+ * passes the autoHandoff vision check, so the raw image reaches the provider
261
+ * and 400s — we learn from that error and force handoff for the model. */
262
+ export function isImageNotSupportedError(text: string): boolean {
263
+ return /not support (?:the )?image|image[s]? (?:input[s]? )?(?:is |are )?not supported|does not accept image|unsupported (?:image|multimodal)|only (?:supports|accepts) text|unsupported content type[\s\S]{0,40}image|can(?:not|'t) see (?:the )?images?/i.test(
264
+ text,
265
+ );
266
+ }
267
+
268
+ /** Models whose registry entries commonly declare image input while the
269
+ * backend rejects images (DeepSeek V4, GLM 4/5 non-V via aggregator
270
+ * proxies). Conservative: VL / Vision / Janus / *V variants keep image
271
+ * input. Used so autoHandoff covers them on the FIRST send — waiting for a
272
+ * 400 is too late. */
273
+ export function isKnownTextOnlyFalselyVision(id: string | undefined | null): boolean {
274
+ if (!id) return false;
275
+ const n = id.toLowerCase();
276
+ if (n.includes("vl") || n.includes("vision") || n.includes("janus")) return false;
277
+ if (/\dv(?=[-_./\s]|$)/.test(n)) return false; // glm-4v, glm-4.5v — vision variants
278
+ if (/deepseek[-_./]*v4/.test(n)) return true;
279
+ if (/glm[-_./]*[45]/.test(n)) return true;
280
+ return false;
281
+ }
282
+
283
+ /** Extract the provider error string from a finalized assistant message.
284
+ * Pi stores it on `errorMessage`; some providers also put the 400 body in a
285
+ * text content block. */
286
+ export function assistantErrorText(msg: {
287
+ errorMessage?: unknown;
288
+ content?: unknown;
289
+ } | null | undefined): string {
290
+ if (!msg) return "";
291
+ if (typeof msg.errorMessage === "string" && msg.errorMessage) return msg.errorMessage;
292
+ if (typeof msg.content === "string") return msg.content;
293
+ if (!Array.isArray(msg.content)) return "";
294
+ return msg.content
295
+ .filter((b): b is { type: string; text: string } => !!b && typeof b === "object" && typeof (b as { text?: unknown }).text === "string")
296
+ .map((b) => b.text)
297
+ .join("\n");
298
+ }
299
+
300
+ /** Set (or add) a `modelOverrides` entry forcing a model's input to
301
+ * `["text"]` in a PARSED models.json config object. Pure: returns a new
302
+ * object, never mutates the input. Preserves every other field (credentials,
303
+ * compat, sibling models) — only the one model's `input` override is touched.
304
+ * Used by the message_end auto-recovery to fix models whose registry entry
305
+ * falsely declares image input the backend rejects with a 400. */
306
+ export function setTextOnlyInputOverride(
307
+ cfg: unknown,
308
+ providerId: string,
309
+ modelId: string,
310
+ ): Record<string, unknown> {
311
+ const root =
312
+ cfg && typeof cfg === "object" && !Array.isArray(cfg)
313
+ ? { ...(cfg as Record<string, unknown>) }
314
+ : {};
315
+ const providers =
316
+ root.providers && typeof root.providers === "object" && !Array.isArray(root.providers)
317
+ ? { ...(root.providers as Record<string, unknown>) }
318
+ : {};
319
+ const provider =
320
+ providers[providerId] && typeof providers[providerId] === "object" && !Array.isArray(providers[providerId])
321
+ ? { ...(providers[providerId] as Record<string, unknown>) }
322
+ : {};
323
+ const overrides =
324
+ provider.modelOverrides && typeof provider.modelOverrides === "object" && !Array.isArray(provider.modelOverrides)
325
+ ? { ...(provider.modelOverrides as Record<string, unknown>) }
326
+ : {};
327
+ const existing =
328
+ overrides[modelId] && typeof overrides[modelId] === "object" && !Array.isArray(overrides[modelId])
329
+ ? { ...(overrides[modelId] as Record<string, unknown>) }
330
+ : {};
331
+ overrides[modelId] = { ...existing, input: ["text"] };
332
+ provider.modelOverrides = overrides;
333
+ providers[providerId] = provider;
334
+ root.providers = providers;
335
+ return root;
336
+ }
337
+
338
+ /** Whether a model declares image input. */
339
+ export function isVisionModel(model: { input?: ("text" | "image")[] } | undefined | null): boolean {
340
+ return !!model && Array.isArray(model.input) && model.input.includes("image");
341
+ }
342
+
343
+ /** Merge a parsed config object onto defaults, tolerating missing/invalid fields. */
344
+ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
345
+ const base: VisionHandoffConfig = { ...DEFAULT_CONFIG };
346
+ if (!raw || typeof raw !== "object") return base;
347
+ const obj = raw as Record<string, unknown>;
348
+
349
+ if (typeof obj.enabled === "boolean") base.enabled = obj.enabled;
350
+ if (typeof obj.visionModel === "string" && obj.visionModel.trim()) {
351
+ base.visionModel = parseModelRef(obj.visionModel) ? obj.visionModel.trim() : null;
352
+ } else if (obj.visionModel === null) {
353
+ base.visionModel = null;
354
+ }
355
+ if (Array.isArray(obj.fallbackModels)) {
356
+ base.fallbackModels = obj.fallbackModels
357
+ .filter((m): m is string => typeof m === "string")
358
+ .map((m) => m.trim())
359
+ .filter((m) => m && parseModelRef(m));
360
+ }
361
+ if (typeof obj.autoHandoff === "boolean") base.autoHandoff = obj.autoHandoff;
362
+
363
+ if (Array.isArray(obj.handoffModels)) {
364
+ base.handoffModels = obj.handoffModels
365
+ .filter((m): m is string => typeof m === "string")
366
+ .map((m) => m.trim())
367
+ .filter((m) => m && parseModelRef(m));
368
+ }
369
+ if (typeof obj.prewarmPastedImages === "boolean") base.prewarmPastedImages = obj.prewarmPastedImages;
370
+ if (typeof obj.asyncClipboardHandoff === "boolean") {
371
+ base.asyncClipboardHandoff = obj.asyncClipboardHandoff;
372
+ }
373
+ // maxTokens: optional. undefined (default) = no artificial cap. Only set when
374
+ // a valid positive finite number is given; any other value leaves it unset.
375
+ if (typeof obj.maxTokens === "number" && Number.isFinite(obj.maxTokens) && obj.maxTokens > 0) {
376
+ base.maxTokens = Math.floor(obj.maxTokens);
377
+ }
378
+ if (
379
+ typeof obj.describeTimeoutMs === "number" &&
380
+ Number.isFinite(obj.describeTimeoutMs) &&
381
+ obj.describeTimeoutMs > 0
382
+ ) {
383
+ base.describeTimeoutMs = Math.floor(obj.describeTimeoutMs);
384
+ }
385
+ if (typeof obj.cacheMax === "number" && Number.isFinite(obj.cacheMax) && obj.cacheMax > 0) {
386
+ base.cacheMax = Math.floor(obj.cacheMax);
387
+ }
388
+ // maxDescriptionLines: any non-negative finite integer. 0 = unbounded.
389
+ if (
390
+ typeof obj.maxDescriptionLines === "number" &&
391
+ Number.isFinite(obj.maxDescriptionLines) &&
392
+ obj.maxDescriptionLines >= 0
393
+ ) {
394
+ base.maxDescriptionLines = Math.floor(obj.maxDescriptionLines);
395
+ }
396
+ // maxRequestBytes: non-negative finite integer. 0 = never split a batch.
397
+ if (typeof obj.maxRequestBytes === "number" && Number.isFinite(obj.maxRequestBytes) && obj.maxRequestBytes >= 0) {
398
+ base.maxRequestBytes = Math.floor(obj.maxRequestBytes);
399
+ }
400
+ if (typeof obj.prompt === "string" && obj.prompt.trim()) base.prompt = obj.prompt;
401
+ if (typeof obj.userPromptPrefix === "string") base.userPromptPrefix = obj.userPromptPrefix;
402
+
403
+ if (typeof obj.thinking === "boolean") base.thinking = obj.thinking;
404
+ if (isThinkingLevel(obj.thinkingLevel)) base.thinkingLevel = obj.thinkingLevel;
405
+
406
+ return base;
407
+ }
408
+
409
+ /** Read config from disk (falls back to defaults on missing/corrupt file). */
410
+ export function readConfig(): VisionHandoffConfig {
411
+ const path = getConfigPath();
412
+ if (!existsSync(path)) return { ...DEFAULT_CONFIG };
413
+ try {
414
+ const raw = readFileSync(path, "utf8");
415
+ return normalizeConfig(JSON.parse(raw));
416
+ } catch {
417
+ return { ...DEFAULT_CONFIG };
418
+ }
419
+ }
420
+
421
+ /** Write config to disk. Creates the directory if needed. Returns the path written. */
422
+ export function writeConfig(config: VisionHandoffConfig): string {
423
+ const path = getConfigPath();
424
+ const dir = join(getAgentDir(), CONFIG_SUBDIR);
425
+ if (!existsSync(dir)) {
426
+ mkdirSync(dir, { recursive: true });
427
+ }
428
+ writeFileSync(path, JSON.stringify(config, null, 2) + "\n", "utf8");
429
+ return path;
430
+ }
431
+
432
+ export interface ExtractedImage {
433
+ data: string;
434
+ mimeType: string;
435
+ }
436
+
437
+ /** Parse a `data:<mime>;base64,<data>` URL into raw base64 + mime. Returns null if not a data URL. */
438
+ export function parseDataUrl(url: string): ExtractedImage | null {
439
+ const match = /^data:([^;,]+)?(?:;base64)?,(.*)$/s.exec(url);
440
+ if (!match) return null;
441
+ return { mimeType: match[1] || "image/png", data: match[2] };
442
+ }
443
+
444
+ /**
445
+ * Detect an image block by shape across the three request formats pi uses:
446
+ * openai-completions: { type: "image_url", image_url: { url: "data:..." } }
447
+ * openai-responses: { type: "input_image", image_url: "data:..." | { url } }
448
+ * anthropic-messages: { type: "image", source: { type: "base64", media_type, data } }
449
+ */
450
+ export function extractImageFromBlock(block: unknown): ExtractedImage | null {
451
+ if (!block || typeof block !== "object") return null;
452
+ const b = block as Record<string, any>;
453
+
454
+ if (b.type === "image_url" && typeof b.image_url?.url === "string") {
455
+ return parseDataUrl(b.image_url.url);
456
+ }
457
+
458
+ if (b.type === "input_image") {
459
+ const url = typeof b.image_url === "string" ? b.image_url : b.image_url?.url;
460
+ if (typeof url === "string") return parseDataUrl(url);
461
+ return null;
462
+ }
463
+
464
+ // anthropic-messages image block (wrapped in `source`).
465
+ if (b.type === "image" && b.source?.type === "base64" && typeof b.source.data === "string") {
466
+ return { data: b.source.data, mimeType: b.source.media_type || "image/png" };
467
+ }
468
+
469
+ // pi-ai internal image block — emitted by the `read` tool and carried by
470
+ // ToolResultEvent.content / user-message content blocks (regardless of whether
471
+ // the active model declares image input):
472
+ // { type: "image", data: "<base64>", mimeType }
473
+ // Distinct from the anthropic shape above: no `source` wrapper, direct fields.
474
+ if (b.type === "image" && typeof b.data === "string") {
475
+ return { data: b.data, mimeType: b.mimeType || "image/png" };
476
+ }
477
+
478
+ return null;
479
+ }
480
+
481
+ /** Outcome of {@link truncateDescription}. */
482
+ export interface TruncatedDescription {
483
+ text: string;
484
+ /** True iff lines were dropped. */
485
+ truncated: boolean;
486
+ /** Number of trailing lines removed. */
487
+ hidden: number;
488
+ }
489
+
490
+ /**
491
+ * Head-truncate a description to its first `maxLines` lines with a
492
+ * "... (N more lines)" footer, mirroring pi core's read-result truncation
493
+ * notice (which itself surfaces "... (N more lines, ctrl+o to expand)").
494
+ *
495
+ * Returns the text unchanged when it is at or below the limit, or when
496
+ * `maxLines` is non-positive (0 = unbounded). Head-truncation keeps the leading
497
+ * layout / text-content / structure sections, which are typically the most
498
+ * actionable for a coding agent.
499
+ */
500
+ export function truncateDescription(text: string, maxLines: number): TruncatedDescription {
501
+ if (!maxLines || maxLines <= 0) {
502
+ return { text, truncated: false, hidden: 0 };
503
+ }
504
+ const lines = text.split("\n");
505
+ if (lines.length <= maxLines) {
506
+ return { text, truncated: false, hidden: 0 };
507
+ }
508
+ const hidden = lines.length - maxLines;
509
+ const kept = lines.slice(0, maxLines).join("\n");
510
+ return { text: `${kept}\n... (${hidden} more lines)`, truncated: true, hidden };
511
+ }
512
+
513
+ /** Start marker for the k-th image section in a batched describer response. */
514
+ export const BATCH_IMAGE_MARKER_START = "<<<IMAGE";
515
+ /** End marker closing a single image section in a batched describer response. */
516
+ export const BATCH_IMAGE_MARKER_END = "<<<END>>>";
517
+
518
+ /** Escape a literal string for use in a RegExp. */
519
+ function escapeRe(s: string): string {
520
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
521
+ }
522
+
523
+ // Section regexes built from the marker constants so the producer
524
+ // (batchUserPrompt) and consumer (parseBatchedDescriptions) share one source of
525
+ // truth for the delimiter format. A lone `<<<IMAGE k>>> … <<<END>>>` wrapper
526
+ // for a single image; a global scan for the multi-image case that stops at the
527
+ // next start marker, the end marker, or end-of-text.
528
+ const SECTION_SINGLE_RE = new RegExp(
529
+ `${escapeRe(BATCH_IMAGE_MARKER_START)}\\s+1>>>\\s*([\\s\\S]*?)${escapeRe(BATCH_IMAGE_MARKER_END)}`,
530
+ );
531
+ const SECTION_MULTI_RE = new RegExp(
532
+ `${escapeRe(BATCH_IMAGE_MARKER_START)}\\s+(\\d+)>>>\\s*([\\s\\S]*?)(?=${escapeRe(
533
+ BATCH_IMAGE_MARKER_START,
534
+ )}\\s+\\d+>>>|${escapeRe(BATCH_IMAGE_MARKER_END)}|$)`,
535
+ "g",
536
+ );
537
+ const SECTION_END_STRIP_RE = new RegExp(`${escapeRe(BATCH_IMAGE_MARKER_END)}\\s*$`);
538
+
539
+ /** Build the user-message text block for a (possibly batched) describer call.
540
+ *
541
+ * For a single image (count <= 1) this returns the same simple prompt the
542
+ * original per-image pipeline used (no markers), so single-image behaviour is
543
+ * unchanged. For count > 1 it instructs the vision model to emit one delimited
544
+ * `<<<IMAGE k>>> … <<<END>>>` section per image, in order, so the response can
545
+ * be split back into per-image descriptions by {@link parseBatchedDescriptions}.
546
+ *
547
+ * The user's turn prompt (if any) is folded in via {@link DEFAULT_USER_PROMPT_PREFIX}
548
+ * so every image in the batch is described in the context of the same request —
549
+ * one describer call covers the whole prompt's image set, dataloader-style,
550
+ * rather than spinning up one agent call per image. */
551
+ export function batchUserPrompt(count: number, userPrompt: string, prefix: string): string {
552
+ const userLine = userPrompt && userPrompt.trim() ? `${prefix}${userPrompt}` : "";
553
+ if (count <= 1) {
554
+ return userLine || "Describe this image.";
555
+ }
556
+ return [
557
+ `Describe each of the ${count} images below exhaustively, in order. For image k (1 through ${count}), emit exactly:`,
558
+ "",
559
+ `${BATCH_IMAGE_MARKER_START} k>>>`,
560
+ "<your exhaustive description>",
561
+ BATCH_IMAGE_MARKER_END,
562
+ ...(userLine ? ["", userLine] : []),
563
+ ].join("\n");
564
+ }
565
+
566
+ /** Parse a batched vision-model response into one description per image, in order.
567
+ *
568
+ * Returns an array of length `count`; each slot is the trimmed description for
569
+ * image k (1-indexed → array index k-1), or `null` when that image's section is
570
+ * missing, empty, or unparseable. Callers treat `null` as "description
571
+ * unavailable" (graceful degradation) — one image failing to parse must not
572
+ * void the rest of the batch.
573
+ *
574
+ * For `count <= 1` the whole response is treated as the single image's
575
+ * description (a lone `<<<IMAGE 1>>> … <<<END>>>` wrapper is stripped if the
576
+ * model emitted one anyway), preserving the original single-image behaviour. */
577
+ export function parseBatchedDescriptions(text: string, count: number): (string | null)[] {
578
+ const trimmed = (text ?? "").trim();
579
+ if (count <= 1) {
580
+ const m = SECTION_SINGLE_RE.exec(trimmed);
581
+ const body = (m ? m[1] : trimmed).trim();
582
+ return [body || null];
583
+ }
584
+ const out: (string | null)[] = new Array(count).fill(null);
585
+ let m: RegExpExecArray | null;
586
+ SECTION_MULTI_RE.lastIndex = 0;
587
+ while ((m = SECTION_MULTI_RE.exec(trimmed)) !== null) {
588
+ const idx = parseInt(m[1], 10) - 1;
589
+ const body = m[2].replace(SECTION_END_STRIP_RE, "").trim();
590
+ if (idx >= 0 && idx < count && !out[idx]) out[idx] = body || null;
591
+ }
592
+ return out;
593
+ }
594
+
595
+ /** Truncate (if configured) and wrap a raw description in the `[Image: …]`
596
+ * placeholder envelope used by every insertion path. Centralises the wrap+
597
+ * truncate logic shared by the read-tool and context injection paths so both
598
+ * surfaces stay identical for the same image hash. */
599
+ export function wrapDescription(description: string, cfg: VisionHandoffConfig): string {
600
+ const { text: final } =
601
+ cfg.maxDescriptionLines && cfg.maxDescriptionLines > 0
602
+ ? truncateDescription(description, cfg.maxDescriptionLines)
603
+ : { text: description };
604
+ return `${IMAGE_PLACEHOLDER_PREFIX}${final}${IMAGE_PLACEHOLDER_SUFFIX}`;
605
+ }