@bismawy/pi-vision-watcher 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +3 -0
- package/package.json +1 -1
- package/src/describer.ts +92 -12
- package/src/index.ts +605 -583
package/src/index.ts
CHANGED
|
@@ -1,583 +1,605 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Shared constants, types, and utilities for pi-vision-watcher.
|
|
3
|
-
*
|
|
4
|
-
* Config lives at ~/.pi/agent/extensions/pi-vision-watcher.json — the same
|
|
5
|
-
* convention pi-model-sort uses for picker-backed extensions.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import type { ThinkingLevel } from "@earendil-works/pi-ai";
|
|
9
|
-
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
10
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
11
|
-
import { join } from "node:path";
|
|
12
|
-
|
|
13
|
-
export * from "./usage.js";
|
|
14
|
-
|
|
15
|
-
/** Subdirectory under the pi agent dir where picker extensions store config. */
|
|
16
|
-
const CONFIG_SUBDIR = "extensions";
|
|
17
|
-
|
|
18
|
-
/** Config file name. */
|
|
19
|
-
export const CONFIG_FILENAME = "pi-vision-watcher.json";
|
|
20
|
-
|
|
21
|
-
/** Full config path: ~/.pi/agent/extensions/pi-vision-watcher.json */
|
|
22
|
-
export function getConfigPath(): string {
|
|
23
|
-
return join(getAgentDir(), CONFIG_SUBDIR, CONFIG_FILENAME);
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/** Description shown in the / commands list. */
|
|
27
|
-
export const HANDOFF_COMMAND_DESCRIPTION =
|
|
28
|
-
"Configure vision watcher — pick a vision model to describe images for text-only models";
|
|
29
|
-
|
|
30
|
-
/** Default system prompt for the vision describer. Mirrors the pi-umans-provider pipeline. */
|
|
31
|
-
export const DEFAULT_VISION_PROMPT =
|
|
32
|
-
"You are a vision assistant for a coding agent. Describe this image exhaustively. Cover: all visible text (verbatim if possible), code snippets, UI layout and widgets, diagrams and flow arrows, error messages and stack traces, file trees, terminal output, color and style details, spatial relationships between elements, and anything else a developer would need to act on this image. Do not summarize — be exhaustive.";
|
|
33
|
-
|
|
34
|
-
/** Prefix prepended to the user's original prompt when describing an image. */
|
|
35
|
-
export const DEFAULT_USER_PROMPT_PREFIX = "The user's request about this image: ";
|
|
36
|
-
|
|
37
|
-
/** Placeholder text block injected in place of an image block. */
|
|
38
|
-
export const IMAGE_PLACEHOLDER_PREFIX = "[Image: ";
|
|
39
|
-
export const IMAGE_PLACEHOLDER_SUFFIX = "]";
|
|
40
|
-
|
|
41
|
-
/** Marker appended to a description whose `completeSimple()` call ended with
|
|
42
|
-
* `stopReason: "length"` — i.e. the vision model hit a token limit (either the
|
|
43
|
-
* configured `maxTokens` or the provider's hard output cap) before finishing.
|
|
44
|
-
* A truncated description is still useful, but the agent must not mistake it
|
|
45
|
-
* for the complete picture (it would reason as if the image contained only
|
|
46
|
-
* what made it into the text). The marker makes the cut-off visible to both
|
|
47
|
-
* the agent (it reads the marker in the tool result / context) and the user
|
|
48
|
-
* (it renders inline). */
|
|
49
|
-
export const DESCRIPTION_TRUNCATED_MARKER =
|
|
50
|
-
"\n\n[... description truncated — the vision model reached its output token limit. The above may be incomplete; raise maxTokens or use a higher-output vision model for the full description.]";
|
|
51
|
-
|
|
52
|
-
/** Append the {@link DESCRIPTION_TRUNCATED_MARKER} to a raw description. */
|
|
53
|
-
export function markDescriptionTruncated(text: string): string {
|
|
54
|
-
return text + DESCRIPTION_TRUNCATED_MARKER;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/** Default vision cache size (number of described images kept in memory per session). */
|
|
58
|
-
export const DEFAULT_CACHE_MAX = 50;
|
|
59
|
-
|
|
60
|
-
/** The pi thinking levels, ordered from lightest to deepest reasoning.
|
|
61
|
-
* Mirrors `@earendil-works/pi-ai`'s `ThinkingLevel`; "off" is handled
|
|
62
|
-
* separately via {@link VisionHandoffConfig.thinking} (a boolean on/off
|
|
63
|
-
* switch) so the level persists across toggles. */
|
|
64
|
-
export const THINKING_LEVELS: readonly ThinkingLevel[] = [
|
|
65
|
-
"minimal",
|
|
66
|
-
"low",
|
|
67
|
-
"medium",
|
|
68
|
-
"high",
|
|
69
|
-
"xhigh",
|
|
70
|
-
"max",
|
|
71
|
-
];
|
|
72
|
-
|
|
73
|
-
/** Default thinking effort for the vision describer when thinking is enabled. */
|
|
74
|
-
export const DEFAULT_THINKING_LEVEL: ThinkingLevel = "medium";
|
|
75
|
-
|
|
76
|
-
/** Whether `level` is one of the supported {@link ThinkingLevel} values. */
|
|
77
|
-
export function isThinkingLevel(level: unknown): level is ThinkingLevel {
|
|
78
|
-
return typeof level === "string" && (THINKING_LEVELS as readonly string[]).includes(level);
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/** Default per-image description timeout (ms). Kept modest so a slow vision
|
|
82
|
-
* provider aborts early and fails over to {@link VisionHandoffConfig.fallbackModels}
|
|
83
|
-
* instead of blocking the agent turn for minutes on a free-tier queue. A single
|
|
84
|
-
* image commonly takes ~20s; raise via `describeTimeoutMs` in config only if
|
|
85
|
-
* a specific model genuinely needs it. {@link describeTimeoutMs} scales this
|
|
86
|
-
* per batch size. */
|
|
87
|
-
export const DESCRIBE_TIMEOUT_MS = 45_000;
|
|
88
|
-
|
|
89
|
-
/** Per-batch timeout: `baseTimeout` is the per-image budget, so the wall-clock
|
|
90
|
-
* budget scales with the number of images in the call (a 5-image batch isn't
|
|
91
|
-
* held to a single image's budget — the describer generates exhaustive prose
|
|
92
|
-
* per image and websocket transport adds latency). `baseTimeout` defaults to
|
|
93
|
-
* {@link DESCRIBE_TIMEOUT_MS} and is overridden by the configured
|
|
94
|
-
* `describeTimeoutMs` field, so lowering it fails over sooner for the *whole*
|
|
95
|
-
* batch. */
|
|
96
|
-
export function describeTimeoutMs(imageCount: number, baseTimeout: number = DESCRIBE_TIMEOUT_MS): number {
|
|
97
|
-
return baseTimeout * Math.max(1, imageCount);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*/
|
|
110
|
-
export const
|
|
111
|
-
|
|
112
|
-
/**
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
/** Opt-in:
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
|
|
175
|
-
/**
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
*
|
|
185
|
-
*
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
/**
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
*
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
thinking
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
export
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
);
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
if (
|
|
275
|
-
|
|
276
|
-
return
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
*
|
|
285
|
-
*
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
providers
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
const
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
if (
|
|
353
|
-
base.
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
) {
|
|
376
|
-
base.
|
|
377
|
-
}
|
|
378
|
-
if (
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
if (
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
export
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
const
|
|
510
|
-
`${
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
*
|
|
546
|
-
*
|
|
547
|
-
*
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
*
|
|
575
|
-
*
|
|
576
|
-
*
|
|
577
|
-
export function
|
|
578
|
-
const
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Shared constants, types, and utilities for pi-vision-watcher.
|
|
3
|
+
*
|
|
4
|
+
* Config lives at ~/.pi/agent/extensions/pi-vision-watcher.json — the same
|
|
5
|
+
* convention pi-model-sort uses for picker-backed extensions.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { ThinkingLevel } from "@earendil-works/pi-ai";
|
|
9
|
+
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
10
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
|
|
13
|
+
export * from "./usage.js";
|
|
14
|
+
|
|
15
|
+
/** Subdirectory under the pi agent dir where picker extensions store config. */
|
|
16
|
+
const CONFIG_SUBDIR = "extensions";
|
|
17
|
+
|
|
18
|
+
/** Config file name. */
|
|
19
|
+
export const CONFIG_FILENAME = "pi-vision-watcher.json";
|
|
20
|
+
|
|
21
|
+
/** Full config path: ~/.pi/agent/extensions/pi-vision-watcher.json */
|
|
22
|
+
export function getConfigPath(): string {
|
|
23
|
+
return join(getAgentDir(), CONFIG_SUBDIR, CONFIG_FILENAME);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Description shown in the / commands list. */
|
|
27
|
+
export const HANDOFF_COMMAND_DESCRIPTION =
|
|
28
|
+
"Configure vision watcher — pick a vision model to describe images for text-only models";
|
|
29
|
+
|
|
30
|
+
/** Default system prompt for the vision describer. Mirrors the pi-umans-provider pipeline. */
|
|
31
|
+
export const DEFAULT_VISION_PROMPT =
|
|
32
|
+
"You are a vision assistant for a coding agent. Describe this image exhaustively. Cover: all visible text (verbatim if possible), code snippets, UI layout and widgets, diagrams and flow arrows, error messages and stack traces, file trees, terminal output, color and style details, spatial relationships between elements, and anything else a developer would need to act on this image. Do not summarize — be exhaustive.";
|
|
33
|
+
|
|
34
|
+
/** Prefix prepended to the user's original prompt when describing an image. */
|
|
35
|
+
export const DEFAULT_USER_PROMPT_PREFIX = "The user's request about this image: ";
|
|
36
|
+
|
|
37
|
+
/** Placeholder text block injected in place of an image block. */
|
|
38
|
+
export const IMAGE_PLACEHOLDER_PREFIX = "[Image: ";
|
|
39
|
+
export const IMAGE_PLACEHOLDER_SUFFIX = "]";
|
|
40
|
+
|
|
41
|
+
/** Marker appended to a description whose `completeSimple()` call ended with
|
|
42
|
+
* `stopReason: "length"` — i.e. the vision model hit a token limit (either the
|
|
43
|
+
* configured `maxTokens` or the provider's hard output cap) before finishing.
|
|
44
|
+
* A truncated description is still useful, but the agent must not mistake it
|
|
45
|
+
* for the complete picture (it would reason as if the image contained only
|
|
46
|
+
* what made it into the text). The marker makes the cut-off visible to both
|
|
47
|
+
* the agent (it reads the marker in the tool result / context) and the user
|
|
48
|
+
* (it renders inline). */
|
|
49
|
+
export const DESCRIPTION_TRUNCATED_MARKER =
|
|
50
|
+
"\n\n[... description truncated — the vision model reached its output token limit. The above may be incomplete; raise maxTokens or use a higher-output vision model for the full description.]";
|
|
51
|
+
|
|
52
|
+
/** Append the {@link DESCRIPTION_TRUNCATED_MARKER} to a raw description. */
|
|
53
|
+
export function markDescriptionTruncated(text: string): string {
|
|
54
|
+
return text + DESCRIPTION_TRUNCATED_MARKER;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Default vision cache size (number of described images kept in memory per session). */
|
|
58
|
+
export const DEFAULT_CACHE_MAX = 50;
|
|
59
|
+
|
|
60
|
+
/** The pi thinking levels, ordered from lightest to deepest reasoning.
|
|
61
|
+
* Mirrors `@earendil-works/pi-ai`'s `ThinkingLevel`; "off" is handled
|
|
62
|
+
* separately via {@link VisionHandoffConfig.thinking} (a boolean on/off
|
|
63
|
+
* switch) so the level persists across toggles. */
|
|
64
|
+
export const THINKING_LEVELS: readonly ThinkingLevel[] = [
|
|
65
|
+
"minimal",
|
|
66
|
+
"low",
|
|
67
|
+
"medium",
|
|
68
|
+
"high",
|
|
69
|
+
"xhigh",
|
|
70
|
+
"max",
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
/** Default thinking effort for the vision describer when thinking is enabled. */
|
|
74
|
+
export const DEFAULT_THINKING_LEVEL: ThinkingLevel = "medium";
|
|
75
|
+
|
|
76
|
+
/** Whether `level` is one of the supported {@link ThinkingLevel} values. */
|
|
77
|
+
export function isThinkingLevel(level: unknown): level is ThinkingLevel {
|
|
78
|
+
return typeof level === "string" && (THINKING_LEVELS as readonly string[]).includes(level);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Default per-image description timeout (ms). Kept modest so a slow vision
|
|
82
|
+
* provider aborts early and fails over to {@link VisionHandoffConfig.fallbackModels}
|
|
83
|
+
* instead of blocking the agent turn for minutes on a free-tier queue. A single
|
|
84
|
+
* image commonly takes ~20s; raise via `describeTimeoutMs` in config only if
|
|
85
|
+
* a specific model genuinely needs it. {@link describeTimeoutMs} scales this
|
|
86
|
+
* per batch size. */
|
|
87
|
+
export const DESCRIBE_TIMEOUT_MS = 45_000;
|
|
88
|
+
|
|
89
|
+
/** Per-batch timeout: `baseTimeout` is the per-image budget, so the wall-clock
|
|
90
|
+
* budget scales with the number of images in the call (a 5-image batch isn't
|
|
91
|
+
* held to a single image's budget — the describer generates exhaustive prose
|
|
92
|
+
* per image and websocket transport adds latency). `baseTimeout` defaults to
|
|
93
|
+
* {@link DESCRIBE_TIMEOUT_MS} and is overridden by the configured
|
|
94
|
+
* `describeTimeoutMs` field, so lowering it fails over sooner for the *whole*
|
|
95
|
+
* batch. */
|
|
96
|
+
export function describeTimeoutMs(imageCount: number, baseTimeout: number = DESCRIBE_TIMEOUT_MS): number {
|
|
97
|
+
return baseTimeout * Math.max(1, imageCount);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Default byte budget for ONE describer request body, images included.
|
|
101
|
+
*
|
|
102
|
+
* Aggregator proxies cap the whole JSON body (text plus base64 images), not the
|
|
103
|
+
* model context: Remote Magpie answers a 1.1 MB body with HTTP 413. A base64
|
|
104
|
+
* image is 4/3 its raw size, so one 2000px screenshot can exceed that cap alone
|
|
105
|
+
* — and a batched call multiplies it by the image count. The describer splits a
|
|
106
|
+
* batch into groups that each fit this budget, so a multi-image read degrades
|
|
107
|
+
* into a few calls instead of one 413 that loses every description. Override it
|
|
108
|
+
* with `maxRequestBytes` when the provider takes a different limit; `0` keeps
|
|
109
|
+
* the old always-one-call behaviour. */
|
|
110
|
+
export const DEFAULT_MAX_REQUEST_BYTES = 700_000;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Default cap on the number of lines kept from a description before truncation.
|
|
114
|
+
*
|
|
115
|
+
* Mirrors pi core's read-result truncation: tool output is bounded for both the
|
|
116
|
+
* TUI render and the model context. The stored `result.content` is the single
|
|
117
|
+
* source for both surfaces (no decouple point exists at `tool_result` — it's the
|
|
118
|
+
* only hook that still has the raw image before pi-ai strips it), so the cap
|
|
119
|
+
* applies uniformly. 0 = unbounded (default): the read tool's native collapse
|
|
120
|
+
* handles compactness and `ctrl+o` expands to the full description.
|
|
121
|
+
*/
|
|
122
|
+
export const DEFAULT_MAX_DESCRIPTION_LINES = 0;
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* The note pi core's `read` tool appends to image-result text blocks when the
|
|
126
|
+
* active model lacks image input (see `getNonVisionImageNote` in pi core).
|
|
127
|
+
*
|
|
128
|
+
* Once this extension inserts a description, the note becomes self-
|
|
129
|
+
* contradicting ("image will be omitted" vs. the vivid description that
|
|
130
|
+
* follows) and confuses the model. {@link stripNonVisionImageNote} removes it.
|
|
131
|
+
*/
|
|
132
|
+
export const NON_VISION_IMAGE_NOTE =
|
|
133
|
+
"[Current model does not support images. The image will be omitted from this request.]";
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Remove {@link NON_VISION_IMAGE_NOTE} from a text block.
|
|
137
|
+
*
|
|
138
|
+
* The read tool appends the note as `\n${NOTE}` (always the trailing segment of
|
|
139
|
+
* the metadata text block), so this also collapses the orphaned newline it
|
|
140
|
+
* leaves behind. Safe to call on text that does not contain the note (no-op).
|
|
141
|
+
*/
|
|
142
|
+
export function stripNonVisionImageNote(text: string): string {
|
|
143
|
+
if (!text.includes(NON_VISION_IMAGE_NOTE)) return text;
|
|
144
|
+
return text.split(NON_VISION_IMAGE_NOTE).join("").replace(/\n+$/, "");
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export interface VisionHandoffConfig {
|
|
148
|
+
/** Master switch. When false, no handoff occurs even if a vision model is configured. */
|
|
149
|
+
enabled: boolean;
|
|
150
|
+
/** The vision-capable model that describes images, as "provider/id". null = not configured. */
|
|
151
|
+
visionModel: string | null;
|
|
152
|
+
/** Optional fallback describers, tried in order when the primary vision
|
|
153
|
+
* model's batched call fails entirely (auth, rate limit, timeout, network,
|
|
154
|
+
* empty/error stopReason) and the same-model retry didn't recover.
|
|
155
|
+
* Each entry is a "provider/id" ref; providers usually differ from the
|
|
156
|
+
* primary so their rate limits / auth / availability are independent.
|
|
157
|
+
* Empty (default) = no failover. */
|
|
158
|
+
fallbackModels: string[];
|
|
159
|
+
/** When true (default), handoff is applied to every model whose input does not include "image". */
|
|
160
|
+
autoHandoff: boolean;
|
|
161
|
+
/** Extra "provider/id" refs that should ALSO receive handoff (e.g. weak vision models). */
|
|
162
|
+
handoffModels: string[];
|
|
163
|
+
/** Opt-in: when true, the editor is wrapped at session start to describe
|
|
164
|
+
* pasted clipboard images the instant their temp-file path lands in the
|
|
165
|
+
* prompt (pre-submit), instead of waiting for submit. Trades a bit of
|
|
166
|
+
* description quality (the description is generated without the user's
|
|
167
|
+
* typed question as context, since the question usually isn't entered yet
|
|
168
|
+
* at paste time) and a speculative vision call on paste-then-abandon, for
|
|
169
|
+
* earlier prewarm. Off by default — submit-time prewarm
|
|
170
|
+
* (before_agent_start) still covers this case when off. TUI mode only, and
|
|
171
|
+
* inactive when another extension has replaced the editor (installing over
|
|
172
|
+
* a custom editor would break clipboard paste, since pi wires paste-image
|
|
173
|
+
* to the outermost editor only). */
|
|
174
|
+
prewarmPastedImages: boolean;
|
|
175
|
+
/** Opt-in: asynchronously inject descriptions for pasted clipboard paths when
|
|
176
|
+
* the target model does not read them. The normal read/tool_result path wins
|
|
177
|
+
* the race: a matching read cancels the queued fallback injection while
|
|
178
|
+
* reusing the same in-flight description. */
|
|
179
|
+
asyncClipboardHandoff: boolean;
|
|
180
|
+
/** Max output tokens for a single description. `undefined` (default) = use the
|
|
181
|
+
* vision model's declared max output (`model.maxTokens`) as the cap — the
|
|
182
|
+
* highest the model supports — so the "be exhaustive" prompt isn't defeated
|
|
183
|
+
* by a provider's small omitted-default. Set a number only to cap lower.
|
|
184
|
+
* A truncation is always surfaced via {@link DESCRIPTION_TRUNCATED_MARKER}
|
|
185
|
+
* when the model hits the limit. */
|
|
186
|
+
maxTokens?: number;
|
|
187
|
+
/** Base per-image description timeout in ms (defaults to
|
|
188
|
+
* {@link DESCRIBE_TIMEOUT_MS}). The per-batch timeout scales from this base
|
|
189
|
+
* by image count. Lower it to fail over to {@link fallbackModels} sooner on a
|
|
190
|
+
* slow primary; raise it for a high-latency vision model that times out
|
|
191
|
+
* spuriously. */
|
|
192
|
+
describeTimeoutMs: number;
|
|
193
|
+
/** Byte budget for ONE describer request body (prompt text + base64 images).
|
|
194
|
+
* Images are grouped into as few calls as fit this budget; a single image
|
|
195
|
+
* above it still gets its own call. `0` disables splitting. Defaults to
|
|
196
|
+
* {@link DEFAULT_MAX_REQUEST_BYTES}. */
|
|
197
|
+
maxRequestBytes: number;
|
|
198
|
+
/** Max images kept in the in-memory description cache. */
|
|
199
|
+
cacheMax: number;
|
|
200
|
+
/**
|
|
201
|
+
* Max lines kept from a description before truncation (0 = unbounded).
|
|
202
|
+
* Bounds both the TUI render and the model context, like pi core's tool-output
|
|
203
|
+
* truncation.
|
|
204
|
+
*/
|
|
205
|
+
maxDescriptionLines: number;
|
|
206
|
+
/** Override the describer system prompt (defaults to DEFAULT_VISION_PROMPT). */
|
|
207
|
+
prompt?: string;
|
|
208
|
+
/** Override the user-prompt prefix (defaults to DEFAULT_USER_PROMPT_PREFIX). */
|
|
209
|
+
userPromptPrefix?: string;
|
|
210
|
+
/** Whether the vision describer should reason (think) before describing.
|
|
211
|
+
* Off by default — describing images is a perception task, not a reasoning
|
|
212
|
+
* one, and thinking adds latency + cost. When on, the level below is sent
|
|
213
|
+
* to the vision model via pi-ai's `reasoning` option (only honoured when
|
|
214
|
+
* the configured vision model declares `reasoning: true`). */
|
|
215
|
+
thinking: boolean;
|
|
216
|
+
/** Thinking effort for the describer when {@link thinking} is on. One of
|
|
217
|
+
* {@link THINKING_LEVELS}. Ignored when {@link thinking} is off or the
|
|
218
|
+
* vision model lacks reasoning support. */
|
|
219
|
+
thinkingLevel: ThinkingLevel;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export const DEFAULT_CONFIG: VisionHandoffConfig = {
|
|
223
|
+
enabled: true,
|
|
224
|
+
visionModel: null,
|
|
225
|
+
fallbackModels: [],
|
|
226
|
+
autoHandoff: true,
|
|
227
|
+
handoffModels: [],
|
|
228
|
+
prewarmPastedImages: false,
|
|
229
|
+
asyncClipboardHandoff: false,
|
|
230
|
+
maxTokens: undefined,
|
|
231
|
+
describeTimeoutMs: DESCRIBE_TIMEOUT_MS,
|
|
232
|
+
maxRequestBytes: DEFAULT_MAX_REQUEST_BYTES,
|
|
233
|
+
cacheMax: DEFAULT_CACHE_MAX,
|
|
234
|
+
maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
|
|
235
|
+
thinking: false,
|
|
236
|
+
thinkingLevel: DEFAULT_THINKING_LEVEL,
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
/** Parse a "provider/id" reference. Returns null if malformed. */
|
|
240
|
+
export function parseModelRef(ref: string): { provider: string; id: string } | null {
|
|
241
|
+
const trimmed = ref.trim();
|
|
242
|
+
if (!trimmed) return null;
|
|
243
|
+
const slashIndex = trimmed.indexOf("/");
|
|
244
|
+
if (slashIndex <= 0) return null; // no slash, or empty provider
|
|
245
|
+
const provider = trimmed.slice(0, slashIndex);
|
|
246
|
+
const id = trimmed.slice(slashIndex + 1);
|
|
247
|
+
if (!provider || !id) return null;
|
|
248
|
+
return { provider, id };
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** Format a provider/id reference string. */
|
|
252
|
+
export function formatModelRef(provider: string, id: string): string {
|
|
253
|
+
return `${provider}/${id}`;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Match provider errors meaning "this model can't take image input" — e.g.
|
|
257
|
+
* AgentRouter's 400 `"This model does not support image"`, OpenRouter's
|
|
258
|
+
* "does not support image inputs". Used by the message_end auto-recovery:
|
|
259
|
+
* a model whose registry entry FALSELY declares `input: ["text","image"]`
|
|
260
|
+
* passes the autoHandoff vision check, so the raw image reaches the provider
|
|
261
|
+
* and 400s — we learn from that error and force handoff for the model. */
|
|
262
|
+
export function isImageNotSupportedError(text: string): boolean {
|
|
263
|
+
return /not support (?:the )?image|image[s]? (?:input[s]? )?(?:is |are )?not supported|does not accept image|unsupported (?:image|multimodal)|only (?:supports|accepts) text|unsupported content type[\s\S]{0,40}image|can(?:not|'t) see (?:the )?images?/i.test(
|
|
264
|
+
text,
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** Models whose registry entries commonly declare image input while the
|
|
269
|
+
* backend rejects images (DeepSeek V4, GLM 4/5 non-V via aggregator
|
|
270
|
+
* proxies). Conservative: VL / Vision / Janus / *V variants keep image
|
|
271
|
+
* input. Used so autoHandoff covers them on the FIRST send — waiting for a
|
|
272
|
+
* 400 is too late. */
|
|
273
|
+
export function isKnownTextOnlyFalselyVision(id: string | undefined | null): boolean {
|
|
274
|
+
if (!id) return false;
|
|
275
|
+
const n = id.toLowerCase();
|
|
276
|
+
if (n.includes("vl") || n.includes("vision") || n.includes("janus")) return false;
|
|
277
|
+
if (/\dv(?=[-_./\s]|$)/.test(n)) return false; // glm-4v, glm-4.5v — vision variants
|
|
278
|
+
if (/deepseek[-_./]*v4/.test(n)) return true;
|
|
279
|
+
if (/glm[-_./]*[45]/.test(n)) return true;
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** Extract the provider error string from a finalized assistant message.
|
|
284
|
+
* Pi stores it on `errorMessage`; some providers also put the 400 body in a
|
|
285
|
+
* text content block. */
|
|
286
|
+
export function assistantErrorText(msg: {
|
|
287
|
+
errorMessage?: unknown;
|
|
288
|
+
content?: unknown;
|
|
289
|
+
} | null | undefined): string {
|
|
290
|
+
if (!msg) return "";
|
|
291
|
+
if (typeof msg.errorMessage === "string" && msg.errorMessage) return msg.errorMessage;
|
|
292
|
+
if (typeof msg.content === "string") return msg.content;
|
|
293
|
+
if (!Array.isArray(msg.content)) return "";
|
|
294
|
+
return msg.content
|
|
295
|
+
.filter((b): b is { type: string; text: string } => !!b && typeof b === "object" && typeof (b as { text?: unknown }).text === "string")
|
|
296
|
+
.map((b) => b.text)
|
|
297
|
+
.join("\n");
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Set (or add) a `modelOverrides` entry forcing a model's input to
|
|
301
|
+
* `["text"]` in a PARSED models.json config object. Pure: returns a new
|
|
302
|
+
* object, never mutates the input. Preserves every other field (credentials,
|
|
303
|
+
* compat, sibling models) — only the one model's `input` override is touched.
|
|
304
|
+
* Used by the message_end auto-recovery to fix models whose registry entry
|
|
305
|
+
* falsely declares image input the backend rejects with a 400. */
|
|
306
|
+
export function setTextOnlyInputOverride(
|
|
307
|
+
cfg: unknown,
|
|
308
|
+
providerId: string,
|
|
309
|
+
modelId: string,
|
|
310
|
+
): Record<string, unknown> {
|
|
311
|
+
const root =
|
|
312
|
+
cfg && typeof cfg === "object" && !Array.isArray(cfg)
|
|
313
|
+
? { ...(cfg as Record<string, unknown>) }
|
|
314
|
+
: {};
|
|
315
|
+
const providers =
|
|
316
|
+
root.providers && typeof root.providers === "object" && !Array.isArray(root.providers)
|
|
317
|
+
? { ...(root.providers as Record<string, unknown>) }
|
|
318
|
+
: {};
|
|
319
|
+
const provider =
|
|
320
|
+
providers[providerId] && typeof providers[providerId] === "object" && !Array.isArray(providers[providerId])
|
|
321
|
+
? { ...(providers[providerId] as Record<string, unknown>) }
|
|
322
|
+
: {};
|
|
323
|
+
const overrides =
|
|
324
|
+
provider.modelOverrides && typeof provider.modelOverrides === "object" && !Array.isArray(provider.modelOverrides)
|
|
325
|
+
? { ...(provider.modelOverrides as Record<string, unknown>) }
|
|
326
|
+
: {};
|
|
327
|
+
const existing =
|
|
328
|
+
overrides[modelId] && typeof overrides[modelId] === "object" && !Array.isArray(overrides[modelId])
|
|
329
|
+
? { ...(overrides[modelId] as Record<string, unknown>) }
|
|
330
|
+
: {};
|
|
331
|
+
overrides[modelId] = { ...existing, input: ["text"] };
|
|
332
|
+
provider.modelOverrides = overrides;
|
|
333
|
+
providers[providerId] = provider;
|
|
334
|
+
root.providers = providers;
|
|
335
|
+
return root;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** Whether a model declares image input. */
|
|
339
|
+
export function isVisionModel(model: { input?: ("text" | "image")[] } | undefined | null): boolean {
|
|
340
|
+
return !!model && Array.isArray(model.input) && model.input.includes("image");
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** Merge a parsed config object onto defaults, tolerating missing/invalid fields. */
|
|
344
|
+
export function normalizeConfig(raw: unknown): VisionHandoffConfig {
|
|
345
|
+
const base: VisionHandoffConfig = { ...DEFAULT_CONFIG };
|
|
346
|
+
if (!raw || typeof raw !== "object") return base;
|
|
347
|
+
const obj = raw as Record<string, unknown>;
|
|
348
|
+
|
|
349
|
+
if (typeof obj.enabled === "boolean") base.enabled = obj.enabled;
|
|
350
|
+
if (typeof obj.visionModel === "string" && obj.visionModel.trim()) {
|
|
351
|
+
base.visionModel = parseModelRef(obj.visionModel) ? obj.visionModel.trim() : null;
|
|
352
|
+
} else if (obj.visionModel === null) {
|
|
353
|
+
base.visionModel = null;
|
|
354
|
+
}
|
|
355
|
+
if (Array.isArray(obj.fallbackModels)) {
|
|
356
|
+
base.fallbackModels = obj.fallbackModels
|
|
357
|
+
.filter((m): m is string => typeof m === "string")
|
|
358
|
+
.map((m) => m.trim())
|
|
359
|
+
.filter((m) => m && parseModelRef(m));
|
|
360
|
+
}
|
|
361
|
+
if (typeof obj.autoHandoff === "boolean") base.autoHandoff = obj.autoHandoff;
|
|
362
|
+
|
|
363
|
+
if (Array.isArray(obj.handoffModels)) {
|
|
364
|
+
base.handoffModels = obj.handoffModels
|
|
365
|
+
.filter((m): m is string => typeof m === "string")
|
|
366
|
+
.map((m) => m.trim())
|
|
367
|
+
.filter((m) => m && parseModelRef(m));
|
|
368
|
+
}
|
|
369
|
+
if (typeof obj.prewarmPastedImages === "boolean") base.prewarmPastedImages = obj.prewarmPastedImages;
|
|
370
|
+
if (typeof obj.asyncClipboardHandoff === "boolean") {
|
|
371
|
+
base.asyncClipboardHandoff = obj.asyncClipboardHandoff;
|
|
372
|
+
}
|
|
373
|
+
// maxTokens: optional. undefined (default) = no artificial cap. Only set when
|
|
374
|
+
// a valid positive finite number is given; any other value leaves it unset.
|
|
375
|
+
if (typeof obj.maxTokens === "number" && Number.isFinite(obj.maxTokens) && obj.maxTokens > 0) {
|
|
376
|
+
base.maxTokens = Math.floor(obj.maxTokens);
|
|
377
|
+
}
|
|
378
|
+
if (
|
|
379
|
+
typeof obj.describeTimeoutMs === "number" &&
|
|
380
|
+
Number.isFinite(obj.describeTimeoutMs) &&
|
|
381
|
+
obj.describeTimeoutMs > 0
|
|
382
|
+
) {
|
|
383
|
+
base.describeTimeoutMs = Math.floor(obj.describeTimeoutMs);
|
|
384
|
+
}
|
|
385
|
+
if (typeof obj.cacheMax === "number" && Number.isFinite(obj.cacheMax) && obj.cacheMax > 0) {
|
|
386
|
+
base.cacheMax = Math.floor(obj.cacheMax);
|
|
387
|
+
}
|
|
388
|
+
// maxDescriptionLines: any non-negative finite integer. 0 = unbounded.
|
|
389
|
+
if (
|
|
390
|
+
typeof obj.maxDescriptionLines === "number" &&
|
|
391
|
+
Number.isFinite(obj.maxDescriptionLines) &&
|
|
392
|
+
obj.maxDescriptionLines >= 0
|
|
393
|
+
) {
|
|
394
|
+
base.maxDescriptionLines = Math.floor(obj.maxDescriptionLines);
|
|
395
|
+
}
|
|
396
|
+
// maxRequestBytes: non-negative finite integer. 0 = never split a batch.
|
|
397
|
+
if (typeof obj.maxRequestBytes === "number" && Number.isFinite(obj.maxRequestBytes) && obj.maxRequestBytes >= 0) {
|
|
398
|
+
base.maxRequestBytes = Math.floor(obj.maxRequestBytes);
|
|
399
|
+
}
|
|
400
|
+
if (typeof obj.prompt === "string" && obj.prompt.trim()) base.prompt = obj.prompt;
|
|
401
|
+
if (typeof obj.userPromptPrefix === "string") base.userPromptPrefix = obj.userPromptPrefix;
|
|
402
|
+
|
|
403
|
+
if (typeof obj.thinking === "boolean") base.thinking = obj.thinking;
|
|
404
|
+
if (isThinkingLevel(obj.thinkingLevel)) base.thinkingLevel = obj.thinkingLevel;
|
|
405
|
+
|
|
406
|
+
return base;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/** Read config from disk (falls back to defaults on missing/corrupt file). */
|
|
410
|
+
export function readConfig(): VisionHandoffConfig {
|
|
411
|
+
const path = getConfigPath();
|
|
412
|
+
if (!existsSync(path)) return { ...DEFAULT_CONFIG };
|
|
413
|
+
try {
|
|
414
|
+
const raw = readFileSync(path, "utf8");
|
|
415
|
+
return normalizeConfig(JSON.parse(raw));
|
|
416
|
+
} catch {
|
|
417
|
+
return { ...DEFAULT_CONFIG };
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/** Write config to disk. Creates the directory if needed. Returns the path written. */
|
|
422
|
+
export function writeConfig(config: VisionHandoffConfig): string {
|
|
423
|
+
const path = getConfigPath();
|
|
424
|
+
const dir = join(getAgentDir(), CONFIG_SUBDIR);
|
|
425
|
+
if (!existsSync(dir)) {
|
|
426
|
+
mkdirSync(dir, { recursive: true });
|
|
427
|
+
}
|
|
428
|
+
writeFileSync(path, JSON.stringify(config, null, 2) + "\n", "utf8");
|
|
429
|
+
return path;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
export interface ExtractedImage {
|
|
433
|
+
data: string;
|
|
434
|
+
mimeType: string;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** Parse a `data:<mime>;base64,<data>` URL into raw base64 + mime. Returns null if not a data URL. */
|
|
438
|
+
export function parseDataUrl(url: string): ExtractedImage | null {
|
|
439
|
+
const match = /^data:([^;,]+)?(?:;base64)?,(.*)$/s.exec(url);
|
|
440
|
+
if (!match) return null;
|
|
441
|
+
return { mimeType: match[1] || "image/png", data: match[2] };
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* Detect an image block by shape across the three request formats pi uses:
|
|
446
|
+
* openai-completions: { type: "image_url", image_url: { url: "data:..." } }
|
|
447
|
+
* openai-responses: { type: "input_image", image_url: "data:..." | { url } }
|
|
448
|
+
* anthropic-messages: { type: "image", source: { type: "base64", media_type, data } }
|
|
449
|
+
*/
|
|
450
|
+
export function extractImageFromBlock(block: unknown): ExtractedImage | null {
|
|
451
|
+
if (!block || typeof block !== "object") return null;
|
|
452
|
+
const b = block as Record<string, any>;
|
|
453
|
+
|
|
454
|
+
if (b.type === "image_url" && typeof b.image_url?.url === "string") {
|
|
455
|
+
return parseDataUrl(b.image_url.url);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
if (b.type === "input_image") {
|
|
459
|
+
const url = typeof b.image_url === "string" ? b.image_url : b.image_url?.url;
|
|
460
|
+
if (typeof url === "string") return parseDataUrl(url);
|
|
461
|
+
return null;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// anthropic-messages image block (wrapped in `source`).
|
|
465
|
+
if (b.type === "image" && b.source?.type === "base64" && typeof b.source.data === "string") {
|
|
466
|
+
return { data: b.source.data, mimeType: b.source.media_type || "image/png" };
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// pi-ai internal image block — emitted by the `read` tool and carried by
|
|
470
|
+
// ToolResultEvent.content / user-message content blocks (regardless of whether
|
|
471
|
+
// the active model declares image input):
|
|
472
|
+
// { type: "image", data: "<base64>", mimeType }
|
|
473
|
+
// Distinct from the anthropic shape above: no `source` wrapper, direct fields.
|
|
474
|
+
if (b.type === "image" && typeof b.data === "string") {
|
|
475
|
+
return { data: b.data, mimeType: b.mimeType || "image/png" };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
return null;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/** Outcome of {@link truncateDescription}. */
|
|
482
|
+
export interface TruncatedDescription {
|
|
483
|
+
text: string;
|
|
484
|
+
/** True iff lines were dropped. */
|
|
485
|
+
truncated: boolean;
|
|
486
|
+
/** Number of trailing lines removed. */
|
|
487
|
+
hidden: number;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* Head-truncate a description to its first `maxLines` lines with a
|
|
492
|
+
* "... (N more lines)" footer, mirroring pi core's read-result truncation
|
|
493
|
+
* notice (which itself surfaces "... (N more lines, ctrl+o to expand)").
|
|
494
|
+
*
|
|
495
|
+
* Returns the text unchanged when it is at or below the limit, or when
|
|
496
|
+
* `maxLines` is non-positive (0 = unbounded). Head-truncation keeps the leading
|
|
497
|
+
* layout / text-content / structure sections, which are typically the most
|
|
498
|
+
* actionable for a coding agent.
|
|
499
|
+
*/
|
|
500
|
+
export function truncateDescription(text: string, maxLines: number): TruncatedDescription {
|
|
501
|
+
if (!maxLines || maxLines <= 0) {
|
|
502
|
+
return { text, truncated: false, hidden: 0 };
|
|
503
|
+
}
|
|
504
|
+
const lines = text.split("\n");
|
|
505
|
+
if (lines.length <= maxLines) {
|
|
506
|
+
return { text, truncated: false, hidden: 0 };
|
|
507
|
+
}
|
|
508
|
+
const hidden = lines.length - maxLines;
|
|
509
|
+
const kept = lines.slice(0, maxLines).join("\n");
|
|
510
|
+
return { text: `${kept}\n... (${hidden} more lines)`, truncated: true, hidden };
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** Start marker for the k-th image section in a batched describer response. */
|
|
514
|
+
export const BATCH_IMAGE_MARKER_START = "<<<IMAGE";
|
|
515
|
+
/** End marker closing a single image section in a batched describer response. */
|
|
516
|
+
export const BATCH_IMAGE_MARKER_END = "<<<END>>>";
|
|
517
|
+
|
|
518
|
+
/** Escape a literal string for use in a RegExp. */
|
|
519
|
+
function escapeRe(s: string): string {
|
|
520
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// Section regexes built from the marker constants so the producer
|
|
524
|
+
// (batchUserPrompt) and consumer (parseBatchedDescriptions) share one source of
|
|
525
|
+
// truth for the delimiter format. A lone `<<<IMAGE k>>> … <<<END>>>` wrapper
|
|
526
|
+
// for a single image; a global scan for the multi-image case that stops at the
|
|
527
|
+
// next start marker, the end marker, or end-of-text.
|
|
528
|
+
const SECTION_SINGLE_RE = new RegExp(
|
|
529
|
+
`${escapeRe(BATCH_IMAGE_MARKER_START)}\\s+1>>>\\s*([\\s\\S]*?)${escapeRe(BATCH_IMAGE_MARKER_END)}`,
|
|
530
|
+
);
|
|
531
|
+
const SECTION_MULTI_RE = new RegExp(
|
|
532
|
+
`${escapeRe(BATCH_IMAGE_MARKER_START)}\\s+(\\d+)>>>\\s*([\\s\\S]*?)(?=${escapeRe(
|
|
533
|
+
BATCH_IMAGE_MARKER_START,
|
|
534
|
+
)}\\s+\\d+>>>|${escapeRe(BATCH_IMAGE_MARKER_END)}|$)`,
|
|
535
|
+
"g",
|
|
536
|
+
);
|
|
537
|
+
const SECTION_END_STRIP_RE = new RegExp(`${escapeRe(BATCH_IMAGE_MARKER_END)}\\s*$`);
|
|
538
|
+
|
|
539
|
+
/** Build the user-message text block for a (possibly batched) describer call.
|
|
540
|
+
*
|
|
541
|
+
* For a single image (count <= 1) this returns the same simple prompt the
|
|
542
|
+
* original per-image pipeline used (no markers), so single-image behaviour is
|
|
543
|
+
* unchanged. For count > 1 it instructs the vision model to emit one delimited
|
|
544
|
+
* `<<<IMAGE k>>> … <<<END>>>` section per image, in order, so the response can
|
|
545
|
+
* be split back into per-image descriptions by {@link parseBatchedDescriptions}.
|
|
546
|
+
*
|
|
547
|
+
* The user's turn prompt (if any) is folded in via {@link DEFAULT_USER_PROMPT_PREFIX}
|
|
548
|
+
* so every image in the batch is described in the context of the same request —
|
|
549
|
+
* one describer call covers the whole prompt's image set, dataloader-style,
|
|
550
|
+
* rather than spinning up one agent call per image. */
|
|
551
|
+
export function batchUserPrompt(count: number, userPrompt: string, prefix: string): string {
|
|
552
|
+
const userLine = userPrompt && userPrompt.trim() ? `${prefix}${userPrompt}` : "";
|
|
553
|
+
if (count <= 1) {
|
|
554
|
+
return userLine || "Describe this image.";
|
|
555
|
+
}
|
|
556
|
+
return [
|
|
557
|
+
`Describe each of the ${count} images below exhaustively, in order. For image k (1 through ${count}), emit exactly:`,
|
|
558
|
+
"",
|
|
559
|
+
`${BATCH_IMAGE_MARKER_START} k>>>`,
|
|
560
|
+
"<your exhaustive description>",
|
|
561
|
+
BATCH_IMAGE_MARKER_END,
|
|
562
|
+
...(userLine ? ["", userLine] : []),
|
|
563
|
+
].join("\n");
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/** Parse a batched vision-model response into one description per image, in order.
|
|
567
|
+
*
|
|
568
|
+
* Returns an array of length `count`; each slot is the trimmed description for
|
|
569
|
+
* image k (1-indexed → array index k-1), or `null` when that image's section is
|
|
570
|
+
* missing, empty, or unparseable. Callers treat `null` as "description
|
|
571
|
+
* unavailable" (graceful degradation) — one image failing to parse must not
|
|
572
|
+
* void the rest of the batch.
|
|
573
|
+
*
|
|
574
|
+
* For `count <= 1` the whole response is treated as the single image's
|
|
575
|
+
* description (a lone `<<<IMAGE 1>>> … <<<END>>>` wrapper is stripped if the
|
|
576
|
+
* model emitted one anyway), preserving the original single-image behaviour. */
|
|
577
|
+
export function parseBatchedDescriptions(text: string, count: number): (string | null)[] {
|
|
578
|
+
const trimmed = (text ?? "").trim();
|
|
579
|
+
if (count <= 1) {
|
|
580
|
+
const m = SECTION_SINGLE_RE.exec(trimmed);
|
|
581
|
+
const body = (m ? m[1] : trimmed).trim();
|
|
582
|
+
return [body || null];
|
|
583
|
+
}
|
|
584
|
+
const out: (string | null)[] = new Array(count).fill(null);
|
|
585
|
+
let m: RegExpExecArray | null;
|
|
586
|
+
SECTION_MULTI_RE.lastIndex = 0;
|
|
587
|
+
while ((m = SECTION_MULTI_RE.exec(trimmed)) !== null) {
|
|
588
|
+
const idx = parseInt(m[1], 10) - 1;
|
|
589
|
+
const body = m[2].replace(SECTION_END_STRIP_RE, "").trim();
|
|
590
|
+
if (idx >= 0 && idx < count && !out[idx]) out[idx] = body || null;
|
|
591
|
+
}
|
|
592
|
+
return out;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/** Truncate (if configured) and wrap a raw description in the `[Image: …]`
|
|
596
|
+
* placeholder envelope used by every insertion path. Centralises the wrap+
|
|
597
|
+
* truncate logic shared by the read-tool and context injection paths so both
|
|
598
|
+
* surfaces stay identical for the same image hash. */
|
|
599
|
+
export function wrapDescription(description: string, cfg: VisionHandoffConfig): string {
|
|
600
|
+
const { text: final } =
|
|
601
|
+
cfg.maxDescriptionLines && cfg.maxDescriptionLines > 0
|
|
602
|
+
? truncateDescription(description, cfg.maxDescriptionLines)
|
|
603
|
+
: { text: description };
|
|
604
|
+
return `${IMAGE_PLACEHOLDER_PREFIX}${final}${IMAGE_PLACEHOLDER_SUFFIX}`;
|
|
605
|
+
}
|