@bismawy/pi-vision-watcher 1.0.10 โ 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +0 -0
- package/README.md +129 -140
- package/package.json +4 -4
- package/src/dataloader.ts +7 -1
- package/src/describer.ts +15 -0
- package/src/dispose.ts +0 -0
- package/src/error-log.ts +3 -0
- package/src/image.ts +0 -0
- package/src/index.ts +9 -88
- package/src/prewarm-editor.ts +0 -0
- package/src/usage.ts +0 -0
- package/src/vision-model-selector.ts +223 -89
- package/vision-watcher.ts +58 -22
- package/vitest.config.ts +0 -0
package/LICENSE
CHANGED
|
File without changes
|
package/README.md
CHANGED
|
@@ -1,140 +1,129 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
[
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
|
48
|
-
|
|
49
|
-
| `/vision-watcher`
|
|
50
|
-
| `/vision-watcher
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
"
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
<
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
```
|
|
131
|
-
|
|
132
|
-
</details>
|
|
133
|
-
|
|
134
|
-
---
|
|
135
|
-
|
|
136
|
-
## ๐ License & Acknowledgments
|
|
137
|
-
|
|
138
|
-
- Built for the **[pi coding agent](https://github.com/earendil-works/pi-coding-agent)** ecosystem.
|
|
139
|
-
- Evolved from concepts in `pi-vision-handoff` by Tom X Nguyen and `pi-umans-provider`.
|
|
140
|
-
- Distributed under the **[MIT License](./LICENSE)**.
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# pi-vision-watcher
|
|
4
|
+
|
|
5
|
+
Give text-only [pi](https://github.com/earendil-works/pi-coding-agent) models vision โ images are described by a vision model you pick, then handed off to text-only coding models without interrupting your workflow.
|
|
6
|
+
|
|
7
|
+
[pi package](https://pi.dev/packages/@bismawy/pi-vision-watcher) ยท [npm](https://www.npmjs.com/package/@bismawy/pi-vision-watcher) ยท [Issues](https://github.com/bismawy/pi-vision-watcher/issues)
|
|
8
|
+
|
|
9
|
+

|
|
10
|
+

|
|
11
|
+
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
<img src="assets/screenshot.webp" alt="pi-vision-watcher" width="100%">
|
|
15
|
+
|
|
16
|
+
## What it does
|
|
17
|
+
|
|
18
|
+
Paste an image, attach a file, or have the agent `read` one โ pi-vision-watcher describes it in the background with your chosen vision model and feeds the description to whatever text-only model you're using (DeepSeek, local models, etc.).
|
|
19
|
+
|
|
20
|
+
- **Connected-only picker:** `/vision-watcher` shows only vision-capable models from providers where you actually have credentials.
|
|
21
|
+
- **Batching & cache:** multiple images across parallel tool calls are batched into one describer request; cached images (SHA-256) are never re-described.
|
|
22
|
+
- **False-vision healing:** aggregator providers sometimes flag text-only models as multimodal, causing HTTP 400s. The extension proactively forces handoff for them and auto-heals `models.json` in-process.
|
|
23
|
+
- **Thinking controls:** adjust reasoning effort (`off` โ `max`) for reasoning-capable vision models.
|
|
24
|
+
- **Fallback chains:** automatically falls back to backup vision models when the primary is rate-limited or down.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pi install npm:@bismawy/pi-vision-watcher
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Then run `/vision-watcher` to pick your vision model (or set it directly: `/vision-watcher model openai/gpt-4o`). Handoff is on by default โ just switch to any text-only model and work as usual.
|
|
33
|
+
|
|
34
|
+
## Commands
|
|
35
|
+
|
|
36
|
+
| Command | Action |
|
|
37
|
+
| :--- | :--- |
|
|
38
|
+
| `/vision-watcher` | Interactive picker for connected vision models |
|
|
39
|
+
| `/vision-watcher model <provider/id>` | Set primary vision describer directly |
|
|
40
|
+
| `/vision-watcher status` | View current configuration |
|
|
41
|
+
| `/vision-watcher auto <on\|off>` | Toggle automatic handoff (default: `on`) |
|
|
42
|
+
| `/vision-watcher add <provider/id>` | Force handoff on a specific model |
|
|
43
|
+
| `/vision-watcher remove <provider/id>` | Remove model from forced handoff list |
|
|
44
|
+
| `/vision-watcher thinking <level>` | Configure reasoning effort |
|
|
45
|
+
| `/vision-watcher timeout <ms>` | Set the base per-image description timeout (default `45000`) |
|
|
46
|
+
| `/vision-watcher prewarm <on\|off>` | Describe pasted images at paste-time (opt-in) |
|
|
47
|
+
| `/vision-watcher async <on\|off>` | Inject pasted-image descriptions asynchronously when no matching `read` wins (alias: `fallback`) |
|
|
48
|
+
| `/vision-watcher clear` | Clear the configured vision model |
|
|
49
|
+
| `/vision-watcher enable` / `disable` | Toggle extension active state |
|
|
50
|
+
| `/vision-watcher help` | List all subcommands |
|
|
51
|
+
|
|
52
|
+
`async` is the async *clipboard* fallback and has nothing to do with the
|
|
53
|
+
`fallbackModels` failover chain.
|
|
54
|
+
|
|
55
|
+
In the picker:
|
|
56
|
+
|
|
57
|
+
| Key | Action |
|
|
58
|
+
|---|---|
|
|
59
|
+
| `space` | Select the highlighted model as the primary describer (press again to clear) |
|
|
60
|
+
| `ctrl+q` | Toggle the highlighted model in/out of the failover chain (marked ๐, max 3). Not a mnemonic, and that's on purpose: `ctrl+f` is pi's find-text (bound since pi 0.85), `alt+f` is editor word-right, and `ctrl+alt+f` never survives Windows conhost/Windows Terminal. pi 0.85 also binds `ctrl+q` to `app.message.followUp`, which is inert while a picker is open โ if it ever double-fires, `f2` is the free fallback (unbound in 0.84 and 0.85) |
|
|
61
|
+
| `ctrl+t` | Walk the thinking ladder: off โ minimal โ low โ medium โ high โ xhigh โ max โ off โ โฆ |
|
|
62
|
+
| `ctrl+a` | Toggle async paste handoff |
|
|
63
|
+
| `enter` / `ctrl+s` | Save (primary **and** chain together) |
|
|
64
|
+
| `esc` | Cancel |
|
|
65
|
+
|
|
66
|
+
The detail pane always shows the current configuration โ primary, chain,
|
|
67
|
+
thinking, async handoff โ so every keypress shows exactly what will be saved.
|
|
68
|
+
`space` is left to the search box while a filter query is present, so multi-word
|
|
69
|
+
searches like `gemini 3.8` stay typeable.
|
|
70
|
+
|
|
71
|
+
## How it works
|
|
72
|
+
|
|
73
|
+
<details>
|
|
74
|
+
<summary><b>Configuration</b> (<code>~/.pi/agent/extensions/pi-vision-watcher.json</code>)</summary>
|
|
75
|
+
|
|
76
|
+
```json
|
|
77
|
+
{
|
|
78
|
+
"enabled": true,
|
|
79
|
+
"visionModel": "openai/gpt-4o",
|
|
80
|
+
"fallbackModels": [],
|
|
81
|
+
"autoHandoff": true,
|
|
82
|
+
"handoffModels": [],
|
|
83
|
+
"thinking": false,
|
|
84
|
+
"thinkingLevel": "medium",
|
|
85
|
+
"describeTimeoutMs": 45000,
|
|
86
|
+
"prewarmPastedImages": false,
|
|
87
|
+
"asyncClipboardHandoff": false,
|
|
88
|
+
"maxTokens": null,
|
|
89
|
+
"cacheMax": 50,
|
|
90
|
+
"maxDescriptionLines": 0
|
|
91
|
+
}
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Most fields have sane defaults โ `visionModel` is the only one you normally set.
|
|
95
|
+
|
|
96
|
+
`fallbackModels` is the failover chain: when the primary describer fails (timeout,
|
|
97
|
+
rate limit, auth), each entry is tried **in order** โ fallback 1 fails, fallback 2
|
|
98
|
+
runs, and so on until one returns a description. The picker caps the chain at 3
|
|
99
|
+
entries (`ctrl+q`). Set it from the picker instead of hand-editing the file.
|
|
100
|
+
|
|
101
|
+
</details>
|
|
102
|
+
|
|
103
|
+
<details>
|
|
104
|
+
<summary><b>Diagnostics & recovery</b></summary>
|
|
105
|
+
|
|
106
|
+
- Failed vision calls log with stack traces to `~/.pi/agent/logs/pi-vision-watcher/errors.log` and degrade gracefully to `[Image: description unavailable]`.
|
|
107
|
+
- When a model falsely advertises image capability and 400s, the error is captured on `message_end`, `modelOverrides.<model>.input = ["text"]` is written to `models.json`, and the registry refreshes in-process.
|
|
108
|
+
|
|
109
|
+
</details>
|
|
110
|
+
|
|
111
|
+
<details>
|
|
112
|
+
<summary><b>Development</b></summary>
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
bun install
|
|
116
|
+
bun run test # Vitest suite (240+ unit tests)
|
|
117
|
+
bun run typecheck
|
|
118
|
+
bun run lint:dead
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
</details>
|
|
122
|
+
|
|
123
|
+
## License
|
|
124
|
+
|
|
125
|
+
Distributed under the **MIT** license.
|
|
126
|
+
|
|
127
|
+
## Developer
|
|
128
|
+
|
|
129
|
+
Developed and maintained by [Bisma](https://github.com/bismawy).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bismawy/pi-vision-watcher",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.12",
|
|
4
4
|
"description": "Give text-only pi models vision โ describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "bismawy",
|
|
@@ -44,9 +44,9 @@
|
|
|
44
44
|
"lint:dead": "knip --no-gitignore"
|
|
45
45
|
},
|
|
46
46
|
"devDependencies": {
|
|
47
|
-
"@earendil-works/pi-ai": "0.
|
|
48
|
-
"@earendil-works/pi-coding-agent": "0.
|
|
49
|
-
"@earendil-works/pi-tui": "0.
|
|
47
|
+
"@earendil-works/pi-ai": "0.85.1",
|
|
48
|
+
"@earendil-works/pi-coding-agent": "0.85.1",
|
|
49
|
+
"@earendil-works/pi-tui": "0.85.1",
|
|
50
50
|
"@types/node": "25.9.1",
|
|
51
51
|
"@vitest/coverage-v8": "4.1.7",
|
|
52
52
|
"knip": "6.14.1",
|
package/src/dataloader.ts
CHANGED
|
@@ -212,7 +212,13 @@ export class DescriptionLoader implements Disposable {
|
|
|
212
212
|
loadDescription(img: ExtractedImage): Promise<string> {
|
|
213
213
|
const hash = imageHash(img.mimeType, img.data);
|
|
214
214
|
const cached = this.cache.get(hash);
|
|
215
|
-
if (cached)
|
|
215
|
+
if (cached) {
|
|
216
|
+
// Re-insert to refresh Map order: insertion order is FIFO, so without
|
|
217
|
+
// this a hot image ages like a cold one and gets evicted ahead of it.
|
|
218
|
+
this.cache.delete(hash);
|
|
219
|
+
this.cache.set(hash, cached);
|
|
220
|
+
return cached;
|
|
221
|
+
}
|
|
216
222
|
|
|
217
223
|
if (!this.batch) {
|
|
218
224
|
this.batch = { keys: [], imgs: [], callbacks: [] };
|
package/src/describer.ts
CHANGED
|
@@ -187,6 +187,11 @@ export interface DescriberDeps {
|
|
|
187
187
|
/** Set the most-recent describer failure message (surfaced to the user by the engine).
|
|
188
188
|
* Pass `null` to clear before a fresh attempt. */
|
|
189
189
|
setLastError(msg: string | null): void;
|
|
190
|
+
/** Record the model ref ACTUALLY attempted for the latest describer call.
|
|
191
|
+
* Differs from `config.visionModel` when a failover fallback ran โ the
|
|
192
|
+
* warning must name the model that failed, not the configured primary.
|
|
193
|
+
* Optional so older callers/tests compile unchanged. */
|
|
194
|
+
setAttemptedModel?(ref: string): void;
|
|
190
195
|
}
|
|
191
196
|
|
|
192
197
|
/** The result of a batched describer call: per-image raw descriptions keyed by hash. */
|
|
@@ -212,6 +217,7 @@ export async function runBatch(
|
|
|
212
217
|
turnSignal?: AbortSignal,
|
|
213
218
|
): Promise<BatchResult> {
|
|
214
219
|
const out: BatchResult = new Map();
|
|
220
|
+
deps.setAttemptedModel?.(formatModelRef(visionModel.provider, visionModel.id));
|
|
215
221
|
const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
|
|
216
222
|
if (!auth.ok || !auth.apiKey) {
|
|
217
223
|
const reason = !auth.ok
|
|
@@ -222,6 +228,7 @@ export async function runBatch(
|
|
|
222
228
|
phase: "batch",
|
|
223
229
|
reason,
|
|
224
230
|
visionModel: cfg.visionModel,
|
|
231
|
+
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
225
232
|
imageHashes: misses.map((m) => m.hash),
|
|
226
233
|
imageCount: misses.length,
|
|
227
234
|
config: configSnapshot(cfg),
|
|
@@ -277,6 +284,7 @@ export async function runBatch(
|
|
|
277
284
|
phase: "batch",
|
|
278
285
|
reason,
|
|
279
286
|
visionModel: cfg.visionModel,
|
|
287
|
+
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
280
288
|
imageHashes: hashes,
|
|
281
289
|
imageCount: misses.length,
|
|
282
290
|
stopReason: response.stopReason,
|
|
@@ -299,6 +307,7 @@ export async function runBatch(
|
|
|
299
307
|
phase: "batch",
|
|
300
308
|
reason: "vision model returned an empty description",
|
|
301
309
|
visionModel: cfg.visionModel,
|
|
310
|
+
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
302
311
|
imageHashes: hashes,
|
|
303
312
|
imageCount: misses.length,
|
|
304
313
|
config: configSnapshot(cfg),
|
|
@@ -356,6 +365,7 @@ export async function runBatch(
|
|
|
356
365
|
phase: "batch",
|
|
357
366
|
reason,
|
|
358
367
|
visionModel: cfg.visionModel,
|
|
368
|
+
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
359
369
|
imageHashes: misses.map((m) => m.hash),
|
|
360
370
|
imageCount: misses.length,
|
|
361
371
|
timedOut,
|
|
@@ -386,6 +396,7 @@ export async function describeSingle(
|
|
|
386
396
|
deps: DescriberDeps,
|
|
387
397
|
turnSignal?: AbortSignal,
|
|
388
398
|
): Promise<string | null> {
|
|
399
|
+
deps.setAttemptedModel?.(formatModelRef(visionModel.provider, visionModel.id));
|
|
389
400
|
const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
|
|
390
401
|
if (!auth.ok || !auth.apiKey) {
|
|
391
402
|
const reason = !auth.ok
|
|
@@ -396,6 +407,7 @@ export async function describeSingle(
|
|
|
396
407
|
phase: "single",
|
|
397
408
|
reason,
|
|
398
409
|
visionModel: cfg.visionModel,
|
|
410
|
+
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
399
411
|
imageHashes: [imageHash(img.mimeType, img.data)],
|
|
400
412
|
imageCount: 1,
|
|
401
413
|
config: configSnapshot(cfg),
|
|
@@ -449,6 +461,7 @@ export async function describeSingle(
|
|
|
449
461
|
phase: "single",
|
|
450
462
|
reason,
|
|
451
463
|
visionModel: cfg.visionModel,
|
|
464
|
+
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
452
465
|
imageHashes: [hash],
|
|
453
466
|
imageCount: 1,
|
|
454
467
|
stopReason: response.stopReason,
|
|
@@ -471,6 +484,7 @@ export async function describeSingle(
|
|
|
471
484
|
phase: "single",
|
|
472
485
|
reason: "vision model returned an empty description",
|
|
473
486
|
visionModel: cfg.visionModel,
|
|
487
|
+
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
474
488
|
imageHashes: [hash],
|
|
475
489
|
imageCount: 1,
|
|
476
490
|
config: configSnapshot(cfg),
|
|
@@ -496,6 +510,7 @@ export async function describeSingle(
|
|
|
496
510
|
phase: "single",
|
|
497
511
|
reason,
|
|
498
512
|
visionModel: cfg.visionModel,
|
|
513
|
+
attemptedModel: formatModelRef(visionModel.provider, visionModel.id),
|
|
499
514
|
imageHashes: [imageHash(img.mimeType, img.data)],
|
|
500
515
|
imageCount: 1,
|
|
501
516
|
timedOut,
|
package/src/dispose.ts
CHANGED
|
File without changes
|
package/src/error-log.ts
CHANGED
|
@@ -53,6 +53,9 @@ export interface VisionErrorLogEntry {
|
|
|
53
53
|
reason: string;
|
|
54
54
|
/** Configured vision model ref ("provider/id"), or null if unset. */
|
|
55
55
|
visionModel: string | null;
|
|
56
|
+
/** The model ACTUALLY called for this entry ("provider/id") โ differs from
|
|
57
|
+
* {@link visionModel} when a failover fallback was attempted. */
|
|
58
|
+
attemptedModel?: string;
|
|
56
59
|
/** Image hashes the failure covered (batch/single) or the warning named. */
|
|
57
60
|
imageHashes: string[];
|
|
58
61
|
/** Number of images involved. */
|
package/src/image.ts
CHANGED
|
File without changes
|
package/src/index.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* convention pi-model-sort uses for picker-backed extensions.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import type {
|
|
8
|
+
import type { ThinkingLevel } from "@earendil-works/pi-ai";
|
|
9
9
|
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
10
10
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
11
11
|
import { join } from "node:path";
|
|
@@ -86,15 +86,15 @@ export function isThinkingLevel(level: unknown): level is ThinkingLevel {
|
|
|
86
86
|
* per batch size. */
|
|
87
87
|
export const DESCRIBE_TIMEOUT_MS = 45_000;
|
|
88
88
|
|
|
89
|
-
/** Per-batch timeout:
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
89
|
+
/** Per-batch timeout: `baseTimeout` is the per-image budget, so the wall-clock
|
|
90
|
+
* budget scales with the number of images in the call (a 5-image batch isn't
|
|
91
|
+
* held to a single image's budget โ the describer generates exhaustive prose
|
|
92
|
+
* per image and websocket transport adds latency). `baseTimeout` defaults to
|
|
93
|
+
* {@link DESCRIBE_TIMEOUT_MS} and is overridden by the configured
|
|
94
|
+
* `describeTimeoutMs` field, so lowering it fails over sooner for the *whole*
|
|
95
|
+
* batch. */
|
|
95
96
|
export function describeTimeoutMs(imageCount: number, baseTimeout: number = DESCRIBE_TIMEOUT_MS): number {
|
|
96
|
-
|
|
97
|
-
return baseTimeout + Math.max(0, imageCount - 1) * perImage;
|
|
97
|
+
return baseTimeout * Math.max(1, imageCount);
|
|
98
98
|
}
|
|
99
99
|
|
|
100
100
|
/**
|
|
@@ -456,15 +456,6 @@ export function extractImageFromBlock(block: unknown): ExtractedImage | null {
|
|
|
456
456
|
return null;
|
|
457
457
|
}
|
|
458
458
|
|
|
459
|
-
/** Build a text block that replaces an image block, matching the request format. */
|
|
460
|
-
export function makeReplacementText(block: unknown, description: string): Record<string, unknown> {
|
|
461
|
-
const b = (block ?? null) as Record<string, unknown> | null;
|
|
462
|
-
if (b?.type === "input_image") {
|
|
463
|
-
return { type: "input_text", text: description };
|
|
464
|
-
}
|
|
465
|
-
return { type: "text", text: description };
|
|
466
|
-
}
|
|
467
|
-
|
|
468
459
|
/** Outcome of {@link truncateDescription}. */
|
|
469
460
|
export interface TruncatedDescription {
|
|
470
461
|
text: string;
|
|
@@ -590,73 +581,3 @@ export function wrapDescription(description: string, cfg: VisionHandoffConfig):
|
|
|
590
581
|
: { text: description };
|
|
591
582
|
return `${IMAGE_PLACEHOLDER_PREFIX}${final}${IMAGE_PLACEHOLDER_SUFFIX}`;
|
|
592
583
|
}
|
|
593
|
-
|
|
594
|
-
/** Outcome of {@link insertImageDescriptions}. */
|
|
595
|
-
export interface ReplacedContent {
|
|
596
|
-
content: (TextContent | ImageContent)[];
|
|
597
|
-
/** True iff at least one image block had a description inserted before it. */
|
|
598
|
-
changed: boolean;
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
/**
|
|
602
|
-
* Insert a description text block before each image block in a tool-result /
|
|
603
|
-
* message content array, KEEPING the image block in place.
|
|
604
|
-
*
|
|
605
|
-
* Why insert (not replace): the stored tool-result content is the single
|
|
606
|
-
* source for both the TUI render (kitty inline images read it via
|
|
607
|
-
* `result.content.filter(c => c.type === "image")`) and the provider payload.
|
|
608
|
-
* Replacing the image would strip it from the terminal render. Keeping the
|
|
609
|
-
* image preserves kitty rendering; the inserted description still reaches
|
|
610
|
-
* non-vision models because pi-ai's `downgradeUnsupportedImages` only rewrites
|
|
611
|
-
* `type: "image"` blocks for non-vision models โ text blocks (including this
|
|
612
|
-
* description) pass through untouched to the provider.
|
|
613
|
-
*
|
|
614
|
-
* `describe` is injected (rather than calling the vision model directly) so the
|
|
615
|
-
* extract โ describe โ insert pipeline is unit-testable without standing up a
|
|
616
|
-
* provider, registry, and API call. The extension wires its real `describeImage`
|
|
617
|
-
* into this helper in its `tool_result` handler.
|
|
618
|
-
*
|
|
619
|
-
* When at least one description is inserted, also strips pi core's
|
|
620
|
-
* {@link NON_VISION_IMAGE_NOTE} from text blocks โ the note (appended by the
|
|
621
|
-
* read tool for non-vision models) would otherwise contradict the inserted
|
|
622
|
-
* description. Text blocks are reassigned (not mutated in place); the input
|
|
623
|
-
* array is left untouched.
|
|
624
|
-
*
|
|
625
|
-
* Returns a new array and `changed: false` when there were no images, so
|
|
626
|
-
* callers can short-circuit and avoid mutating pi's stored result unnecessarily.
|
|
627
|
-
*/
|
|
628
|
-
export async function insertImageDescriptions(
|
|
629
|
-
content: readonly (TextContent | ImageContent)[] | undefined,
|
|
630
|
-
describe: (img: ExtractedImage) => Promise<string>,
|
|
631
|
-
): Promise<ReplacedContent> {
|
|
632
|
-
if (!Array.isArray(content)) {
|
|
633
|
-
return { content: [], changed: false };
|
|
634
|
-
}
|
|
635
|
-
const next: (TextContent | ImageContent)[] = [];
|
|
636
|
-
let changed = false;
|
|
637
|
-
for (const block of content) {
|
|
638
|
-
const img = extractImageFromBlock(block);
|
|
639
|
-
if (!img) {
|
|
640
|
-
next.push(block);
|
|
641
|
-
continue;
|
|
642
|
-
}
|
|
643
|
-
const description = await describe(img);
|
|
644
|
-
next.push({ type: "text", text: description } satisfies TextContent);
|
|
645
|
-
next.push(block);
|
|
646
|
-
changed = true;
|
|
647
|
-
}
|
|
648
|
-
if (!changed) {
|
|
649
|
-
return { content: next, changed };
|
|
650
|
-
}
|
|
651
|
-
// We inserted at least one description. Strip the read tool's
|
|
652
|
-
// "[Current model does not support images...]" note from text blocks โ it
|
|
653
|
-
// contradicts the description we just inserted ("image will be omitted" vs.
|
|
654
|
-
// the description that follows) and confuses the model.
|
|
655
|
-
for (let i = 0; i < next.length; i++) {
|
|
656
|
-
const block = next[i];
|
|
657
|
-
if (block.type === "text" && typeof block.text === "string" && block.text.includes(NON_VISION_IMAGE_NOTE)) {
|
|
658
|
-
next[i] = { type: "text", text: stripNonVisionImageNote(block.text) } satisfies TextContent;
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
return { content: next, changed };
|
|
662
|
-
}
|
package/src/prewarm-editor.ts
CHANGED
|
File without changes
|
package/src/usage.ts
CHANGED
|
File without changes
|
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
* Uses the same patterns as pi's built-in selectors and pi-hide-providers:
|
|
6
6
|
* - Lists connected (authenticated) models, vision-capable ones first (๐ badge)
|
|
7
7
|
* - A leading "None" row clears the configured vision model
|
|
8
|
-
* -
|
|
9
|
-
* -
|
|
10
|
-
* -
|
|
11
|
-
* -
|
|
8
|
+
* - Space selects the highlighted model as the primary describer (toggle)
|
|
9
|
+
* - Ctrl+Alt+F toggles the highlighted model in/out of the failover chain (max 3)
|
|
10
|
+
* - Ctrl+T walks the thinking ladder, Ctrl+A toggles async paste handoff
|
|
11
|
+
* - Enter or Ctrl+S saves, Esc / Ctrl+C cancels
|
|
12
|
+
* - The primary is marked โ, chain members ๐
|
|
12
13
|
*/
|
|
13
14
|
|
|
14
15
|
import {
|
|
@@ -29,6 +30,47 @@ import type { ThinkingLevel } from "@earendil-works/pi-ai";
|
|
|
29
30
|
import { DynamicBorder, keyText } from "@earendil-works/pi-coding-agent";
|
|
30
31
|
import { formatModelRef, isVisionModel, THINKING_LEVELS } from "./index.js";
|
|
31
32
|
|
|
33
|
+
/** Failover chain length cap โ three is already far past the point where a
|
|
34
|
+
* fourth describer would ever be reached. */
|
|
35
|
+
export const MAX_FALLBACKS = 3;
|
|
36
|
+
|
|
37
|
+
/** Key that toggles failover-chain membership, and the hint shown for it.
|
|
38
|
+
*
|
|
39
|
+
* Deliberately a single control byte nobody else wants: `ctrl+alt+f` never
|
|
40
|
+
* survives Windows conhost/Windows Terminal (AltGr handling drops or downgrades
|
|
41
|
+
* it), `alt+f` is pi's editor word-right, and `ctrl+f` is pi's find-text. */
|
|
42
|
+
const FALLBACK_KEY = Key.ctrl("q");
|
|
43
|
+
const FALLBACK_KEY_HINT = "ctrl+q";
|
|
44
|
+
|
|
45
|
+
/** Provider ids that don't title-case cleanly. Everything else falls back to
|
|
46
|
+
* word-capitalisation (`custom-openrouter-ai` โ "Custom Openrouter AI"). */
|
|
47
|
+
const PROVIDER_LABELS: Record<string, string> = {
|
|
48
|
+
openai: "OpenAI",
|
|
49
|
+
"openai-codex": "OpenAI Codex",
|
|
50
|
+
anthropic: "Anthropic",
|
|
51
|
+
xai: "xAI",
|
|
52
|
+
github: "GitHub",
|
|
53
|
+
huggingface: "Hugging Face",
|
|
54
|
+
vertex: "Vertex AI",
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const ACRONYMS = new Set(["ai", "api", "gpt", "llm", "mcp", "cli", "glm"]);
|
|
58
|
+
|
|
59
|
+
/** Human-readable provider name for the detail pane. */
|
|
60
|
+
export function providerLabel(provider: string): string {
|
|
61
|
+
const known = PROVIDER_LABELS[provider];
|
|
62
|
+
if (known) return known;
|
|
63
|
+
return provider
|
|
64
|
+
.split(/[-_]/)
|
|
65
|
+
.filter(Boolean)
|
|
66
|
+
.map((word) =>
|
|
67
|
+
ACRONYMS.has(word.toLowerCase())
|
|
68
|
+
? word.toUpperCase()
|
|
69
|
+
: word.charAt(0).toUpperCase() + word.slice(1),
|
|
70
|
+
)
|
|
71
|
+
.join(" ");
|
|
72
|
+
}
|
|
73
|
+
|
|
32
74
|
interface DisplayItem {
|
|
33
75
|
/** "provider/id", or null for the synthetic "None" row. */
|
|
34
76
|
ref: string | null;
|
|
@@ -52,6 +94,9 @@ export interface VisionModelSelectorResult {
|
|
|
52
94
|
thinkingLevel: ThinkingLevel;
|
|
53
95
|
/** Whether pasted paths should be injected if no matching read wins. */
|
|
54
96
|
asyncClipboardHandoff: boolean;
|
|
97
|
+
/** The failover chain, in list order. Enter/ctrl+s saves it together with
|
|
98
|
+
* the primary selection โ the picker edits both in one screen. */
|
|
99
|
+
fallbackModels: string[];
|
|
55
100
|
}
|
|
56
101
|
|
|
57
102
|
export class VisionModelSelectorComponent implements Component {
|
|
@@ -70,6 +115,10 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
70
115
|
private thinking: boolean;
|
|
71
116
|
private thinkingLevel: ThinkingLevel;
|
|
72
117
|
private asyncClipboardHandoff: boolean;
|
|
118
|
+
/** Fallback-chain membership, toggled in-place with {@link FALLBACK_KEY}. */
|
|
119
|
+
private fallbacks: Set<string>;
|
|
120
|
+
/** Transient hint (e.g. chain cap hit), rendered in the detail pane. */
|
|
121
|
+
private notice: string | null = null;
|
|
73
122
|
|
|
74
123
|
private _focused = false;
|
|
75
124
|
get focused(): boolean {
|
|
@@ -94,6 +143,7 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
94
143
|
currentThinkingLevel: ThinkingLevel,
|
|
95
144
|
currentAsyncClipboardHandoff: boolean,
|
|
96
145
|
done: (result: VisionModelSelectorResult) => void,
|
|
146
|
+
currentFallbacks: string[] = [],
|
|
97
147
|
) {
|
|
98
148
|
this.theme = theme;
|
|
99
149
|
this.done = done;
|
|
@@ -101,6 +151,7 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
101
151
|
this.thinking = currentThinking;
|
|
102
152
|
this.thinkingLevel = currentThinkingLevel;
|
|
103
153
|
this.asyncClipboardHandoff = currentAsyncClipboardHandoff;
|
|
154
|
+
this.fallbacks = new Set(currentFallbacks);
|
|
104
155
|
this.allItems = this.buildItems(allModels);
|
|
105
156
|
this.filteredItems = this.allItems;
|
|
106
157
|
|
|
@@ -111,10 +162,7 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
111
162
|
this.listContainer = new Container();
|
|
112
163
|
this.footerText = new Text(this.getFooterText(), 0, 0);
|
|
113
164
|
|
|
114
|
-
this.searchInput.onSubmit = () =>
|
|
115
|
-
const item = this.filteredItems[this.selectedIndex];
|
|
116
|
-
if (item) this.confirm(item);
|
|
117
|
-
};
|
|
165
|
+
this.searchInput.onSubmit = () => this.save();
|
|
118
166
|
|
|
119
167
|
this.updateList();
|
|
120
168
|
}
|
|
@@ -172,15 +220,8 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
172
220
|
return;
|
|
173
221
|
}
|
|
174
222
|
|
|
175
|
-
if (kb.matches(data, "tui.select.confirm")) {
|
|
176
|
-
|
|
177
|
-
if (item) this.confirm(item);
|
|
178
|
-
return;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
if (matchesKey(data, Key.ctrl("s"))) {
|
|
182
|
-
const item = this.filteredItems[this.selectedIndex];
|
|
183
|
-
if (item) this.confirm(item);
|
|
223
|
+
if (kb.matches(data, "tui.select.confirm") || matchesKey(data, Key.ctrl("s"))) {
|
|
224
|
+
this.save();
|
|
184
225
|
return;
|
|
185
226
|
}
|
|
186
227
|
|
|
@@ -199,24 +240,38 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
199
240
|
return;
|
|
200
241
|
}
|
|
201
242
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
243
|
+
// Space selects the highlighted model as the primary describer; pressing it
|
|
244
|
+
// again on the same model clears it back to "None". While a filter query is
|
|
245
|
+
// present, space is left to the search input so multi-word queries like
|
|
246
|
+
// "gemini 3.8" stay typeable.
|
|
247
|
+
if ((data === " " || matchesKey(data, Key.space)) && !this.searchInput.getValue()) {
|
|
248
|
+
const item = this.filteredItems[this.selectedIndex];
|
|
249
|
+
if (item) this.selectPrimary(item.ref);
|
|
205
250
|
return;
|
|
206
251
|
}
|
|
207
252
|
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
//
|
|
212
|
-
|
|
213
|
-
|
|
253
|
+
// Toggles the highlighted model in/out of the failover chain โ a per-row
|
|
254
|
+
// flag rather than a separate screen, so the primary and the chain are
|
|
255
|
+
// chosen together. Intercepted before the search input (like the other ctrl
|
|
256
|
+
// shortcuts) so the key never lands in the filter text. See
|
|
257
|
+
// {@link FALLBACK_KEY} for why it isn't ctrl+f.
|
|
258
|
+
if (matchesKey(data, FALLBACK_KEY)) {
|
|
259
|
+
const item = this.filteredItems[this.selectedIndex];
|
|
260
|
+
if (item?.ref) this.toggleFallback(item.ref);
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
if (matchesKey(data, Key.ctrl("a"))) {
|
|
265
|
+
this.asyncClipboardHandoff = !this.asyncClipboardHandoff;
|
|
214
266
|
this.updateList();
|
|
215
267
|
return;
|
|
216
268
|
}
|
|
217
269
|
|
|
218
|
-
|
|
219
|
-
|
|
270
|
+
// ctrl+t walks the whole thinking ladder (off โ minimal โ โฆ โ max โ off) so
|
|
271
|
+
// one key covers on/off *and* effort โ no separate shift+tab binding.
|
|
272
|
+
// Intercepted before the search input so it never lands in the filter text.
|
|
273
|
+
if (kb.matches(data, "app.thinking.toggle") || matchesKey(data, Key.ctrl("t"))) {
|
|
274
|
+
this.cycleThinking();
|
|
220
275
|
this.updateList();
|
|
221
276
|
return;
|
|
222
277
|
}
|
|
@@ -277,22 +332,58 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
277
332
|
|
|
278
333
|
private getFooterText(): string {
|
|
279
334
|
const totalCount = this.allItems.length - 1; // exclude the None row
|
|
335
|
+
const matches = this.searchInput.getValue()
|
|
336
|
+
? `${this.filteredItems.length - 1} matches`
|
|
337
|
+
: `total ${totalCount} vision-capable models`;
|
|
280
338
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
: "current: none";
|
|
284
|
-
|
|
339
|
+
// The current selection lives in the detail pane above, so the footer only
|
|
340
|
+
// carries keys + the model count.
|
|
285
341
|
const parts: string[] = [
|
|
286
|
-
`${keyText("tui.select.confirm")}
|
|
287
|
-
|
|
288
|
-
`${
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
342
|
+
`${keyText("tui.select.confirm")} = done`,
|
|
343
|
+
"space = select vision models (๐)",
|
|
344
|
+
`${FALLBACK_KEY_HINT} = fallback models (๐)`,
|
|
345
|
+
"ctrl+t = thinking",
|
|
346
|
+
"ctrl+a = async fallback",
|
|
347
|
+
"esc = cancel",
|
|
348
|
+
matches,
|
|
293
349
|
];
|
|
294
350
|
|
|
295
|
-
return this.theme.fg("dim", ` ${parts.join(" ยท ")}
|
|
351
|
+
return this.theme.fg("dim", ` ${parts.join(" ยท ")} `);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Toggle a model's membership in the fallback chain, preserving list order
|
|
355
|
+
* (the chain is tried in order, so the config array must be deterministic
|
|
356
|
+
* rather than Set-iteration order). */
|
|
357
|
+
private toggleFallback(ref: string): void {
|
|
358
|
+
if (this.fallbacks.has(ref)) {
|
|
359
|
+
this.fallbacks.delete(ref);
|
|
360
|
+
this.notice = null;
|
|
361
|
+
} else if (this.fallbacks.size >= MAX_FALLBACKS) {
|
|
362
|
+
this.notice = `max ${MAX_FALLBACKS} fallbacks โ remove one first (${FALLBACK_KEY_HINT})`;
|
|
363
|
+
} else {
|
|
364
|
+
this.fallbacks.add(ref);
|
|
365
|
+
this.notice = null;
|
|
366
|
+
}
|
|
367
|
+
this.updateList();
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/** Space toggles the primary describer; picking the current one again clears
|
|
371
|
+
* it (same as the None row), so one key both sets and unsets. */
|
|
372
|
+
private selectPrimary(ref: string | null): void {
|
|
373
|
+
this.currentRef = this.currentRef === ref ? null : ref;
|
|
374
|
+
this.notice = null;
|
|
375
|
+
this.updateList();
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** Fallback refs in list order (models the picker didn't show โ e.g. one that
|
|
379
|
+
* is no longer resolvable โ are appended so a config value can't be
|
|
380
|
+
* silently dropped just by opening the picker). */
|
|
381
|
+
private orderedFallbacks(): string[] {
|
|
382
|
+
const shown = this.allItems
|
|
383
|
+
.map((i) => i.ref)
|
|
384
|
+
.filter((r): r is string => !!r && this.fallbacks.has(r));
|
|
385
|
+
const unshown = [...this.fallbacks].filter((r) => !shown.includes(r));
|
|
386
|
+
return [...shown, ...unshown];
|
|
296
387
|
}
|
|
297
388
|
|
|
298
389
|
private refresh(): void {
|
|
@@ -318,8 +409,6 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
318
409
|
this.listContainer.addChild(
|
|
319
410
|
new Text(this.theme.fg("muted", " No matching models"), 0, 0),
|
|
320
411
|
);
|
|
321
|
-
this.footerText.setText(this.getFooterText());
|
|
322
|
-
return;
|
|
323
412
|
}
|
|
324
413
|
|
|
325
414
|
const startIndex = Math.max(
|
|
@@ -355,8 +444,15 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
355
444
|
: item.none && this.currentRef === null
|
|
356
445
|
? this.theme.fg("success", " โ")
|
|
357
446
|
: "";
|
|
447
|
+
// Fallback marker โ distinct from the primary's โ so a model can visibly
|
|
448
|
+
// be both the primary and a fallback (Sonnet as primary, Gemini as the
|
|
449
|
+
// chain behind it).
|
|
450
|
+
const fallbackMark =
|
|
451
|
+
item.ref && this.fallbacks.has(item.ref)
|
|
452
|
+
? this.theme.fg("warning", " ๐")
|
|
453
|
+
: "";
|
|
358
454
|
|
|
359
|
-
this.listContainer.addChild(new Text(`${prefix}${label}${current}`, 0, 0));
|
|
455
|
+
this.listContainer.addChild(new Text(`${prefix}${label}${current}${fallbackMark}`, 0, 0));
|
|
360
456
|
}
|
|
361
457
|
|
|
362
458
|
if (startIndex > 0 || endIndex < this.filteredItems.length) {
|
|
@@ -368,63 +464,96 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
368
464
|
);
|
|
369
465
|
}
|
|
370
466
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
467
|
+
this.renderDetail();
|
|
468
|
+
this.footerText.setText(this.getFooterText());
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
private itemByRef(ref: string): DisplayItem | undefined {
|
|
472
|
+
return this.allItems.find((i) => i.ref === ref);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/** "Gemini 3.8 Flash (Antigravity)", or the raw ref when it isn't in the
|
|
476
|
+
* registry right now (stale config) so it stays visible instead of blank. */
|
|
477
|
+
private refLabel(ref: string): string {
|
|
478
|
+
const item = this.itemByRef(ref);
|
|
479
|
+
if (!item) return ref;
|
|
480
|
+
const provider = providerLabel(item.provider);
|
|
481
|
+
// Model display names often already carry the vendor โ "Gemini 3.8 Flash
|
|
482
|
+
// (Antigravity)" would otherwise come out as "โฆ (Antigravity) (Antigravity)".
|
|
483
|
+
return item.modelName.toLowerCase().includes(provider.toLowerCase())
|
|
484
|
+
? item.modelName
|
|
485
|
+
: `${item.modelName} (${provider})`;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/** The detail pane summarises the *configuration* (primary, failover chain
|
|
489
|
+
* and toggles) rather than the highlighted row, so each space / ctrl+q
|
|
490
|
+
* press shows exactly what will be saved. */
|
|
491
|
+
private renderDetail(): void {
|
|
492
|
+
const line = (label: string, value: string) =>
|
|
390
493
|
this.listContainer.addChild(
|
|
391
|
-
new Text(this.theme.fg("dim", `
|
|
494
|
+
new Text(this.theme.fg("dim", ` ${label}`) + value, 0, 0),
|
|
392
495
|
);
|
|
393
|
-
}
|
|
394
496
|
|
|
395
|
-
this.
|
|
396
|
-
|
|
497
|
+
this.listContainer.addChild(new Spacer(1));
|
|
498
|
+
line(
|
|
499
|
+
"Vision-capable (๐): ",
|
|
500
|
+
this.currentRef
|
|
501
|
+
? this.refLabel(this.currentRef)
|
|
502
|
+
: this.theme.fg("muted", "none โ vision handoff disabled"),
|
|
503
|
+
);
|
|
504
|
+
|
|
505
|
+
const chain = this.orderedFallbacks();
|
|
506
|
+
line(
|
|
507
|
+
"Fallback (๐): ",
|
|
508
|
+
chain.length
|
|
509
|
+
? `${this.theme.fg("success", "on")} - ${chain.map((r) => this.refLabel(r)).join(", ")}`
|
|
510
|
+
: this.theme.fg("muted", "off"),
|
|
511
|
+
);
|
|
512
|
+
|
|
513
|
+
line(
|
|
514
|
+
"Thinking: ",
|
|
515
|
+
this.thinking
|
|
516
|
+
? this.theme.fg("success", `on (${this.thinkingLevel})`)
|
|
517
|
+
: this.theme.fg("muted", "off"),
|
|
518
|
+
);
|
|
397
519
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
? this.theme.fg("success", `on (${this.thinkingLevel})`)
|
|
404
|
-
: this.theme.fg("muted", "off");
|
|
405
|
-
this.listContainer.addChild(
|
|
406
|
-
new Text(this.theme.fg("dim", ` Thinking: ${state}`), 0, 0),
|
|
520
|
+
line(
|
|
521
|
+
"Async pasted-path fallback: ",
|
|
522
|
+
this.asyncClipboardHandoff
|
|
523
|
+
? this.theme.fg("success", "on")
|
|
524
|
+
: this.theme.fg("muted", "off"),
|
|
407
525
|
);
|
|
408
|
-
|
|
526
|
+
|
|
527
|
+
// The warning follows the *highlighted* row: it answers "what happens if I
|
|
528
|
+
// pick this model", which is also how you'd notice it while browsing.
|
|
529
|
+
const highlighted = this.filteredItems[this.selectedIndex];
|
|
530
|
+
if (this.thinking && highlighted && !highlighted.none && !highlighted.reasoning) {
|
|
409
531
|
this.listContainer.addChild(
|
|
410
532
|
new Text(
|
|
411
533
|
this.theme.fg(
|
|
412
534
|
"warning",
|
|
413
|
-
` โ ${
|
|
535
|
+
` โ ${highlighted.modelId} declares no reasoning โ thinking will be ignored`,
|
|
414
536
|
),
|
|
415
537
|
0, 0,
|
|
416
538
|
),
|
|
417
539
|
);
|
|
418
540
|
}
|
|
541
|
+
|
|
542
|
+
if (this.notice) {
|
|
543
|
+
this.listContainer.addChild(
|
|
544
|
+
new Text(this.theme.fg("warning", ` ${this.notice}`), 0, 0),
|
|
545
|
+
);
|
|
546
|
+
}
|
|
419
547
|
}
|
|
420
548
|
|
|
421
|
-
private
|
|
549
|
+
private save(): void {
|
|
422
550
|
this.done({
|
|
423
|
-
ref:
|
|
551
|
+
ref: this.currentRef,
|
|
424
552
|
cancelled: false,
|
|
425
553
|
thinking: this.thinking,
|
|
426
554
|
thinkingLevel: this.thinkingLevel,
|
|
427
555
|
asyncClipboardHandoff: this.asyncClipboardHandoff,
|
|
556
|
+
fallbackModels: this.orderedFallbacks(),
|
|
428
557
|
});
|
|
429
558
|
}
|
|
430
559
|
|
|
@@ -435,18 +564,23 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
435
564
|
thinking: this.thinking,
|
|
436
565
|
thinkingLevel: this.thinkingLevel,
|
|
437
566
|
asyncClipboardHandoff: this.asyncClipboardHandoff,
|
|
567
|
+
fallbackModels: this.orderedFallbacks(),
|
|
438
568
|
});
|
|
439
569
|
}
|
|
440
570
|
|
|
441
|
-
/**
|
|
442
|
-
*
|
|
443
|
-
*
|
|
444
|
-
*
|
|
445
|
-
*
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
const
|
|
449
|
-
const
|
|
450
|
-
|
|
571
|
+
/** Walk the thinking ladder with one key: off โ minimal โ low โ medium โ
|
|
572
|
+
* high โ xhigh โ max โ off โ minimal โ โฆ
|
|
573
|
+
*
|
|
574
|
+
* A single index over off + {@link THINKING_LEVELS} so the cycle always
|
|
575
|
+
* advances. Keeping a separate "remembered level" while off turns the tail
|
|
576
|
+
* into a two-position toggle once you reach max (off โ max โ off โ max). */
|
|
577
|
+
private cycleThinking(): void {
|
|
578
|
+
const ladder = THINKING_LEVELS.length + 1; // position 0 = off
|
|
579
|
+
const current = this.thinking
|
|
580
|
+
? THINKING_LEVELS.indexOf(this.thinkingLevel) + 1
|
|
581
|
+
: 0;
|
|
582
|
+
const next = (Math.max(current, 0) + 1) % ladder;
|
|
583
|
+
this.thinking = next > 0;
|
|
584
|
+
if (next > 0) this.thinkingLevel = THINKING_LEVELS[next - 1]!;
|
|
451
585
|
}
|
|
452
586
|
}
|
package/vision-watcher.ts
CHANGED
|
@@ -76,6 +76,11 @@ let config: VisionHandoffConfig = readConfig();
|
|
|
76
76
|
* Cleared at the start of each describer attempt. */
|
|
77
77
|
let lastDescriberError: string | null = null;
|
|
78
78
|
|
|
79
|
+
/** Model ref actually attempted for the most recent describer call. Set by the
|
|
80
|
+
* describer (via {@link LoaderDeps}) so a failover failure is attributed to
|
|
81
|
+
* the fallback that ran, not to `config.visionModel`. */
|
|
82
|
+
let lastDescriberModel: string | null = null;
|
|
83
|
+
|
|
79
84
|
/** Image hashes we've already warned the user about this session. Prevents the
|
|
80
85
|
* `context` hook (which fires before every LLM turn) from re-warning on the
|
|
81
86
|
* same failing images every turn โ describer failures aren't cached, so
|
|
@@ -230,6 +235,9 @@ const loaderDeps: LoaderDeps = {
|
|
|
230
235
|
setLastError: (msg) => {
|
|
231
236
|
lastDescriberError = msg;
|
|
232
237
|
},
|
|
238
|
+
setAttemptedModel: (ref) => {
|
|
239
|
+
lastDescriberModel = ref;
|
|
240
|
+
},
|
|
233
241
|
};
|
|
234
242
|
const loader = new DescriptionLoader(loaderDeps);
|
|
235
243
|
|
|
@@ -390,13 +398,21 @@ function warnFailedImages(
|
|
|
390
398
|
phase: "warn",
|
|
391
399
|
reason,
|
|
392
400
|
visionModel: config.visionModel,
|
|
401
|
+
attemptedModel: lastDescriberModel ?? undefined,
|
|
393
402
|
imageHashes: newlyFailed,
|
|
394
403
|
imageCount: newlyFailed.length,
|
|
395
404
|
activeModel: ctx.model ? formatModelRef(ctx.model.provider, ctx.model.id) : undefined,
|
|
396
405
|
});
|
|
397
406
|
if (!ctx.hasUI) return;
|
|
407
|
+
// Name the model that actually failed. After a failover the error belongs to
|
|
408
|
+
// a fallback, so reporting the configured primary sends users chasing the
|
|
409
|
+
// wrong provider (e.g. a 429 from the fallback attributed to the primary).
|
|
410
|
+
const attempted =
|
|
411
|
+
lastDescriberModel && lastDescriberModel !== config.visionModel
|
|
412
|
+
? `${config.visionModel} โ fallback ${lastDescriberModel}`
|
|
413
|
+
: config.visionModel;
|
|
398
414
|
ctx.ui.notify(
|
|
399
|
-
`pi-vision-watcher: image description failed โ ${reason}. Vision model: ${
|
|
415
|
+
`pi-vision-watcher: image description failed โ ${reason}. Vision model: ${attempted}`,
|
|
400
416
|
"warning",
|
|
401
417
|
);
|
|
402
418
|
}
|
|
@@ -779,9 +795,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
779
795
|
// in-process (modelOverrides is Pi's topmost config layer, so it wins even
|
|
780
796
|
// over a regenerated models[] entry). The model then registers as text-only,
|
|
781
797
|
// autoHandoff covers it naturally, and /model shows it correctly. FALLBACK
|
|
782
|
-
// (write failed): force handoff via handoffModels.
|
|
783
|
-
//
|
|
784
|
-
//
|
|
798
|
+
// (write failed): force handoff via handoffModels. On success we ALSO add the
|
|
799
|
+
// model to handoffModels โ belt and braces, so the retry is described even if
|
|
800
|
+
// the metadata write is later reverted by a models.json regeneration. Either
|
|
801
|
+
// way, the failed turn's images are still in history; the context hook
|
|
802
|
+
// describes them on the retry instead of failing again.
|
|
785
803
|
pi.on("message_end", (event, ctx) => {
|
|
786
804
|
const msg = event.message as {
|
|
787
805
|
role?: string;
|
|
@@ -843,7 +861,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
843
861
|
pi.registerCommand("vision-watcher", {
|
|
844
862
|
description: HANDOFF_COMMAND_DESCRIPTION,
|
|
845
863
|
getArgumentCompletions(prefix: string) {
|
|
846
|
-
const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "fallback", "timeout", "add", "remove", "clear", "help"];
|
|
864
|
+
const subcommands = ["select", "model", "status", "enable", "disable", "auto", "thinking", "prewarm", "async", "fallback", "timeout", "add", "remove", "clear", "help"];
|
|
847
865
|
const matches = subcommands.filter((s) => s.startsWith(prefix));
|
|
848
866
|
return matches.length > 0 ? matches.map((s) => ({ value: s, label: s })) : null;
|
|
849
867
|
},
|
|
@@ -879,8 +897,9 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
879
897
|
" Set the vision describer's thinking effort (off = disabled)",
|
|
880
898
|
" /vision-watcher prewarm <on|off>",
|
|
881
899
|
" Toggle describing pasted images at paste-time (opt-in, off by default)",
|
|
882
|
-
" /vision-watcher
|
|
900
|
+
" /vision-watcher async <on|off>",
|
|
883
901
|
" Inject pasted-image descriptions asynchronously when no matching read wins",
|
|
902
|
+
" (alias: /vision-watcher fallback โ NOT the model failover chain)",
|
|
884
903
|
" /vision-watcher timeout <ms> Set the per-image description timeout (default 45000)",
|
|
885
904
|
" /vision-watcher add <p/id> Force handoff for an extra model",
|
|
886
905
|
" /vision-watcher remove <p/id> Stop forcing handoff for a model",
|
|
@@ -892,7 +911,7 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
892
911
|
" read images through a dataloader (one batched vision call); context swaps",
|
|
893
912
|
" image blocks in the payload for the cached text description.",
|
|
894
913
|
" prewarm on wraps the editor to describe pasted images at paste-time.",
|
|
895
|
-
"
|
|
914
|
+
" async on asynchronously injects a collapsed description unless a matching read wins.",
|
|
896
915
|
].join("\n"),
|
|
897
916
|
"info",
|
|
898
917
|
);
|
|
@@ -939,7 +958,7 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
939
958
|
return;
|
|
940
959
|
}
|
|
941
960
|
|
|
942
|
-
if (subcommand === "fallback") {
|
|
961
|
+
if (subcommand === "async" || subcommand === "fallback") {
|
|
943
962
|
handleFallbackSubcommand(ctx, rest);
|
|
944
963
|
return;
|
|
945
964
|
}
|
|
@@ -1099,19 +1118,19 @@ function handlePrewarmSubcommand(ctx: ExtensionCommandContext, rest: string): vo
|
|
|
1099
1118
|
updateConfig(ctx, (c) => ({ ...c, prewarmPastedImages: on }), note);
|
|
1100
1119
|
}
|
|
1101
1120
|
|
|
1102
|
-
/** Handle
|
|
1121
|
+
/** Handle `/vision-watcher async <on|off>` (alias: `fallback`). */
|
|
1103
1122
|
function handleFallbackSubcommand(ctx: ExtensionCommandContext, rest: string): void {
|
|
1104
1123
|
const value = rest.trim().toLowerCase();
|
|
1105
1124
|
if (!value) {
|
|
1106
1125
|
ctx.ui.notify(
|
|
1107
1126
|
`Async pasted-path fallback: ${config.asyncClipboardHandoff ? "on" : "off"}.\n` +
|
|
1108
|
-
"Usage: /vision-watcher
|
|
1127
|
+
"Usage: /vision-watcher async <on|off>",
|
|
1109
1128
|
"info",
|
|
1110
1129
|
);
|
|
1111
1130
|
return;
|
|
1112
1131
|
}
|
|
1113
1132
|
if (value !== "on" && value !== "off") {
|
|
1114
|
-
ctx.ui.notify("Usage: /vision-watcher
|
|
1133
|
+
ctx.ui.notify("Usage: /vision-watcher async <on|off>", "warning");
|
|
1115
1134
|
return;
|
|
1116
1135
|
}
|
|
1117
1136
|
const on = value === "on";
|
|
@@ -1153,21 +1172,30 @@ function handleTimeoutSubcommand(ctx: ExtensionCommandContext, rest: string): vo
|
|
|
1153
1172
|
);
|
|
1154
1173
|
}
|
|
1155
1174
|
|
|
1175
|
+
/** Connected, vision-capable models โ the same set the pickers show. Text-only
|
|
1176
|
+
* models can't describe images, so they would only produce
|
|
1177
|
+
* "[Image: description unavailable]" errors. getAvailable() excludes the whole
|
|
1178
|
+
* catalogue of unauthenticated models. */
|
|
1179
|
+
function availableVisionModels(ctx: ExtensionCommandContext): Array<{
|
|
1180
|
+
provider: string;
|
|
1181
|
+
id: string;
|
|
1182
|
+
name: string;
|
|
1183
|
+
input?: ("text" | "image")[];
|
|
1184
|
+
reasoning?: boolean;
|
|
1185
|
+
}> {
|
|
1186
|
+
return ctx.modelRegistry
|
|
1187
|
+
.getAvailable()
|
|
1188
|
+
.map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input, reasoning: m.reasoning }))
|
|
1189
|
+
.filter((m) => isVisionModel(m));
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1156
1192
|
async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
|
|
1157
1193
|
if (!ctx.hasUI) {
|
|
1158
1194
|
ctx.ui.notify("/vision-watcher requires interactive mode.", "error");
|
|
1159
1195
|
return;
|
|
1160
1196
|
}
|
|
1161
1197
|
|
|
1162
|
-
|
|
1163
|
-
// /better-custom, a models.json apiKey, or env/command keys) โ the same set
|
|
1164
|
-
// the built-in /model picker shows. getAvailable() excludes the whole
|
|
1165
|
-
// catalogue of unauthenticated models.
|
|
1166
|
-
// Only vision-capable models can describe images โ text-only models are hidden.
|
|
1167
|
-
const availableModels = ctx.modelRegistry
|
|
1168
|
-
.getAvailable()
|
|
1169
|
-
.map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input, reasoning: m.reasoning }))
|
|
1170
|
-
.filter((m) => isVisionModel(m));
|
|
1198
|
+
const availableModels = availableVisionModels(ctx);
|
|
1171
1199
|
|
|
1172
1200
|
if (ctx.mode !== "tui") {
|
|
1173
1201
|
const modelItems = ["None", ...availableModels.map((m) => `${m.provider}/${m.id}`)];
|
|
@@ -1204,6 +1232,7 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
|
|
|
1204
1232
|
config.thinkingLevel,
|
|
1205
1233
|
config.asyncClipboardHandoff,
|
|
1206
1234
|
(r) => done(r),
|
|
1235
|
+
config.fallbackModels,
|
|
1207
1236
|
);
|
|
1208
1237
|
return {
|
|
1209
1238
|
render(width: number) {
|
|
@@ -1233,10 +1262,16 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
|
|
|
1233
1262
|
const thinkingNote = thinking
|
|
1234
1263
|
? `thinking on (${thinkingLevel})${ref ? " โ applies only if the vision model supports reasoning" : ""}`
|
|
1235
1264
|
: "thinking off";
|
|
1265
|
+
const fallbackModels = result.fallbackModels ?? config.fallbackModels;
|
|
1266
|
+
const fallbackNote = fallbackModels.length
|
|
1267
|
+
? ` ยท ๐ fallbacks: ${fallbackModels.join(" โ ")}`
|
|
1268
|
+
: " ยท ๐ no fallbacks";
|
|
1236
1269
|
updateConfig(
|
|
1237
1270
|
ctx,
|
|
1238
|
-
(c) => ({ ...c, visionModel: ref, thinking, thinkingLevel, asyncClipboardHandoff }),
|
|
1239
|
-
ref
|
|
1271
|
+
(c) => ({ ...c, visionModel: ref, thinking, thinkingLevel, asyncClipboardHandoff, fallbackModels }),
|
|
1272
|
+
ref
|
|
1273
|
+
? `Vision model set to ${ref} ยท ${thinkingNote}${fallbackNote}`
|
|
1274
|
+
: `Vision model cleared ยท ${thinkingNote}${fallbackNote}`,
|
|
1240
1275
|
);
|
|
1241
1276
|
if (!ref) {
|
|
1242
1277
|
ctx.ui.notify("Handoff is inactive until you pick a vision model.", "warning");
|
|
@@ -1252,6 +1287,7 @@ function showStatus(ctx: ExtensionCommandContext): void {
|
|
|
1252
1287
|
lines.push(`Thinking: ${config.thinking ? `on (${config.thinkingLevel})` : "off"}`);
|
|
1253
1288
|
lines.push(`Paste-time prewarm: ${config.prewarmPastedImages ? `on${editorInstalled ? "" : " (inactive โ another custom editor is active)"}` : "off"}`);
|
|
1254
1289
|
lines.push(`Async pasted-path fallback: ${config.asyncClipboardHandoff ? "on" : "off"}`);
|
|
1290
|
+
lines.push(`Fallback models: ${config.fallbackModels.length ? config.fallbackModels.join(" โ ") : "(none)"}`);
|
|
1255
1291
|
lines.push(`Timeout (per image): ${config.describeTimeoutMs} ms`);
|
|
1256
1292
|
lines.push(`maxTokens: ${config.maxTokens ?? "unbounded"} ยท cacheMax: ${config.cacheMax} ยท maxDescriptionLines: ${config.maxDescriptionLines === 0 ? "unbounded" : config.maxDescriptionLines}`);
|
|
1257
1293
|
|
package/vitest.config.ts
CHANGED
|
File without changes
|