@bismawy/pi-vision-watcher 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -1
- package/package.json +72 -72
- package/src/describer.ts +7 -0
- package/src/index.ts +77 -11
- package/src/router.ts +291 -0
- package/src/vision-model-selector.ts +17 -11
- package/vision-watcher.ts +222 -44
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
## [
|
|
3
|
+
## [1.4.1] - 2026-11-16
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- Detect text-only models behind routing proxies (such as Magpie, 9router, and omnirouter) using router capabilities and expanded model-family heuristics, so `/vision-watcher` filters them out and `autoHandoff` routes images through the vision model on first send.
|
|
7
|
+
- Skip vision handoff when the active chat model is vision-capable, and only warn about an inactive handoff when the active chat model is text-only without a configured `visionModel`.
|
|
8
|
+
- Treat short HTTP 200 provider quota, billing, auth, and JSON error messages (`isSoftProviderErrorText`) as failures in `callVision` so `fallbackModels` failover triggers automatically.
|
|
4
9
|
|
|
5
10
|
## [1.4.0] - 2026-11-13
|
|
6
11
|
|
package/package.json
CHANGED
|
@@ -1,72 +1,72 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@bismawy/pi-vision-watcher",
|
|
3
|
-
"version": "1.4.
|
|
4
|
-
"description": "Intelligent vision watcher. Connected models. Zero workflow interruption. Text-only Pi models get background image descriptions with failover chains.",
|
|
5
|
-
"type": "module",
|
|
6
|
-
"author": "bismawy",
|
|
7
|
-
"license": "MIT",
|
|
8
|
-
"publishConfig": {
|
|
9
|
-
"access": "public"
|
|
10
|
-
},
|
|
11
|
-
"repository": {
|
|
12
|
-
"type": "git",
|
|
13
|
-
"url": "git+https://github.com/bismawy/pi-vision-watcher.git"
|
|
14
|
-
},
|
|
15
|
-
"homepage": "https://github.com/bismawy/pi-vision-watcher#readme",
|
|
16
|
-
"bugs": {
|
|
17
|
-
"url": "https://github.com/bismawy/pi-vision-watcher/issues"
|
|
18
|
-
},
|
|
19
|
-
"keywords": [
|
|
20
|
-
"pi-package",
|
|
21
|
-
"pi-extension",
|
|
22
|
-
"pi-vision-watcher",
|
|
23
|
-
"vision-watcher",
|
|
24
|
-
"pi",
|
|
25
|
-
"pi-coding-agent",
|
|
26
|
-
"vision",
|
|
27
|
-
"image-describer",
|
|
28
|
-
"image-description",
|
|
29
|
-
"multimodal",
|
|
30
|
-
"model-selector",
|
|
31
|
-
"model-picker",
|
|
32
|
-
"failover",
|
|
33
|
-
"accessibility"
|
|
34
|
-
],
|
|
35
|
-
"files": [
|
|
36
|
-
"*.ts",
|
|
37
|
-
"src/",
|
|
38
|
-
"CHANGELOG.md",
|
|
39
|
-
"LICENSE",
|
|
40
|
-
"README.md"
|
|
41
|
-
],
|
|
42
|
-
"scripts": {
|
|
43
|
-
"test": "vitest run",
|
|
44
|
-
"dev": "pi --extension ./vision-watcher.ts",
|
|
45
|
-
"test:watch": "vitest",
|
|
46
|
-
"test:coverage": "vitest run --coverage",
|
|
47
|
-
"typecheck": "tsc --noEmit",
|
|
48
|
-
"lint:dead": "knip --no-gitignore"
|
|
49
|
-
},
|
|
50
|
-
"devDependencies": {
|
|
51
|
-
"@earendil-works/pi-ai": "0.85.1",
|
|
52
|
-
"@earendil-works/pi-coding-agent": "0.85.1",
|
|
53
|
-
"@earendil-works/pi-tui": "0.85.1",
|
|
54
|
-
"@types/node": "25.9.1",
|
|
55
|
-
"@vitest/coverage-v8": "4.1.7",
|
|
56
|
-
"knip": "6.14.1",
|
|
57
|
-
"typescript": "6.0.3",
|
|
58
|
-
"vitest": "4.1.7"
|
|
59
|
-
},
|
|
60
|
-
"pi": {
|
|
61
|
-
"extensions": [
|
|
62
|
-
"./vision-watcher.ts"
|
|
63
|
-
],
|
|
64
|
-
"image": "https://raw.githubusercontent.com/bismawy/pi-vision-watcher/main/assets/banner.webp"
|
|
65
|
-
},
|
|
66
|
-
"peerDependencies": {
|
|
67
|
-
"@earendil-works/pi-ai": "*",
|
|
68
|
-
"@earendil-works/pi-coding-agent": "*",
|
|
69
|
-
"@earendil-works/pi-tui": "*"
|
|
70
|
-
},
|
|
71
|
-
"packageManager": "pnpm@11.20.0"
|
|
72
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "@bismawy/pi-vision-watcher",
|
|
3
|
+
"version": "1.4.1",
|
|
4
|
+
"description": "Intelligent vision watcher. Connected models. Zero workflow interruption. Text-only Pi models get background image descriptions with failover chains.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"author": "bismawy",
|
|
7
|
+
"license": "MIT",
|
|
8
|
+
"publishConfig": {
|
|
9
|
+
"access": "public"
|
|
10
|
+
},
|
|
11
|
+
"repository": {
|
|
12
|
+
"type": "git",
|
|
13
|
+
"url": "git+https://github.com/bismawy/pi-vision-watcher.git"
|
|
14
|
+
},
|
|
15
|
+
"homepage": "https://github.com/bismawy/pi-vision-watcher#readme",
|
|
16
|
+
"bugs": {
|
|
17
|
+
"url": "https://github.com/bismawy/pi-vision-watcher/issues"
|
|
18
|
+
},
|
|
19
|
+
"keywords": [
|
|
20
|
+
"pi-package",
|
|
21
|
+
"pi-extension",
|
|
22
|
+
"pi-vision-watcher",
|
|
23
|
+
"vision-watcher",
|
|
24
|
+
"pi",
|
|
25
|
+
"pi-coding-agent",
|
|
26
|
+
"vision",
|
|
27
|
+
"image-describer",
|
|
28
|
+
"image-description",
|
|
29
|
+
"multimodal",
|
|
30
|
+
"model-selector",
|
|
31
|
+
"model-picker",
|
|
32
|
+
"failover",
|
|
33
|
+
"accessibility"
|
|
34
|
+
],
|
|
35
|
+
"files": [
|
|
36
|
+
"*.ts",
|
|
37
|
+
"src/",
|
|
38
|
+
"CHANGELOG.md",
|
|
39
|
+
"LICENSE",
|
|
40
|
+
"README.md"
|
|
41
|
+
],
|
|
42
|
+
"scripts": {
|
|
43
|
+
"test": "vitest run",
|
|
44
|
+
"dev": "pi --extension ./vision-watcher.ts",
|
|
45
|
+
"test:watch": "vitest",
|
|
46
|
+
"test:coverage": "vitest run --coverage",
|
|
47
|
+
"typecheck": "tsc --noEmit",
|
|
48
|
+
"lint:dead": "knip --no-gitignore"
|
|
49
|
+
},
|
|
50
|
+
"devDependencies": {
|
|
51
|
+
"@earendil-works/pi-ai": "0.85.1",
|
|
52
|
+
"@earendil-works/pi-coding-agent": "0.85.1",
|
|
53
|
+
"@earendil-works/pi-tui": "0.85.1",
|
|
54
|
+
"@types/node": "25.9.1",
|
|
55
|
+
"@vitest/coverage-v8": "4.1.7",
|
|
56
|
+
"knip": "6.14.1",
|
|
57
|
+
"typescript": "6.0.3",
|
|
58
|
+
"vitest": "4.1.7"
|
|
59
|
+
},
|
|
60
|
+
"pi": {
|
|
61
|
+
"extensions": [
|
|
62
|
+
"./vision-watcher.ts"
|
|
63
|
+
],
|
|
64
|
+
"image": "https://raw.githubusercontent.com/bismawy/pi-vision-watcher/main/assets/banner.webp"
|
|
65
|
+
},
|
|
66
|
+
"peerDependencies": {
|
|
67
|
+
"@earendil-works/pi-ai": "*",
|
|
68
|
+
"@earendil-works/pi-coding-agent": "*",
|
|
69
|
+
"@earendil-works/pi-tui": "*"
|
|
70
|
+
},
|
|
71
|
+
"packageManager": "pnpm@11.20.0"
|
|
72
|
+
}
|
package/src/describer.ts
CHANGED
|
@@ -46,6 +46,7 @@ import {
|
|
|
46
46
|
DEFAULT_VISION_PROMPT,
|
|
47
47
|
describeTimeoutMs,
|
|
48
48
|
formatModelRef,
|
|
49
|
+
isSoftProviderErrorText,
|
|
49
50
|
markDescriptionTruncated,
|
|
50
51
|
parseBatchedDescriptions,
|
|
51
52
|
type ExtractedImage,
|
|
@@ -474,6 +475,12 @@ async function callVision<T>(
|
|
|
474
475
|
appendVisionError({ ...base, reason, config: configSnapshot(cfg) });
|
|
475
476
|
return null;
|
|
476
477
|
}
|
|
478
|
+
if (isSoftProviderErrorText(text)) {
|
|
479
|
+
const reason = `vision model returned an error message as text: ${text.slice(0, 200)}`;
|
|
480
|
+
deps.setLastError(reason);
|
|
481
|
+
appendVisionError({ ...base, reason, config: configSnapshot(cfg) });
|
|
482
|
+
return null;
|
|
483
|
+
}
|
|
477
484
|
return await finish(text, response.stopReason === "length");
|
|
478
485
|
} catch (err) {
|
|
479
486
|
const userAborted = abortWire.userAborted();
|
package/src/index.ts
CHANGED
|
@@ -265,18 +265,65 @@ export function isImageNotSupportedError(text: string): boolean {
|
|
|
265
265
|
);
|
|
266
266
|
}
|
|
267
267
|
|
|
268
|
+
/** Detect upstream provider/proxy errors returned as normal HTTP 200 text
|
|
269
|
+
* (`stopReason: "stop"`) — e.g. aihubmix quota exhaustion ("accounts that have
|
|
270
|
+
* not been recharged can only try 10 times"), raw JSON error bodies, or short
|
|
271
|
+
* billing/auth/rate-limit rejections. Rejecting these in `callVision` lets the
|
|
272
|
+
* dataloader's retry + `fallbackModels` chain take over.
|
|
273
|
+
*
|
|
274
|
+
* Guarded to short responses (`<= 500` chars) without batch delimiters so a
|
|
275
|
+
* real screenshot description that quotes an error message is not rejected. */
|
|
276
|
+
export function isSoftProviderErrorText(text: string): boolean {
|
|
277
|
+
const trimmed = text.trim();
|
|
278
|
+
if (!trimmed || trimmed.length > 500) return false;
|
|
279
|
+
if (/<<<IMAGE\s+\d+>>>/i.test(trimmed)) return false;
|
|
280
|
+
if (isImageNotSupportedError(trimmed)) return true;
|
|
281
|
+
if (/^\s*\{\s*"(?:error|code)"\s*:/i.test(trimmed)) return true;
|
|
282
|
+
if (
|
|
283
|
+
/accounts that have not been recharged can only try|insufficient[_ ]quota|exceeded your current quota|credit balance is too low|balance is insufficient|please recharge|top up your account/i.test(
|
|
284
|
+
trimmed,
|
|
285
|
+
)
|
|
286
|
+
) {
|
|
287
|
+
return true;
|
|
288
|
+
}
|
|
289
|
+
return /^(?:error|401|402|403|429|500|502|503)\s*[:\-]\s*(?:unauthorized|forbidden|rate limit|too many requests|insufficient|invalid api key|bad gateway|service unavailable)/i.test(
|
|
290
|
+
trimmed,
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
|
|
268
294
|
/** Models whose registry entries commonly declare image input while the
|
|
269
|
-
* backend rejects images (DeepSeek V4, GLM 4/5 non-V
|
|
270
|
-
*
|
|
271
|
-
*
|
|
272
|
-
*
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
295
|
+
* backend rejects images (DeepSeek V3/V4/R1, GLM 4/5 non-V, Mercury, SWE,
|
|
296
|
+
* GPT-OSS, Qwen Coder, Llama 3.1/3.3, Codestral/Devstral/Magistral).
|
|
297
|
+
* Conservative: VL / Vision / Janus / OCR / Omni / *V variants keep image
|
|
298
|
+
* input unless explicitly tagged text-only. Used for both `/vision-watcher`
|
|
299
|
+
* filtering and `autoHandoff` so text-only models hand off on FIRST send. */
|
|
300
|
+
export function isKnownTextOnlyFalselyVision(
|
|
301
|
+
id: string | undefined | null,
|
|
302
|
+
label?: string | null,
|
|
303
|
+
): boolean {
|
|
304
|
+
const combined = [id, label]
|
|
305
|
+
.filter((s): s is string => typeof s === "string" && s.trim().length > 0)
|
|
306
|
+
.join(" ");
|
|
307
|
+
if (!combined) return false;
|
|
308
|
+
const n = combined.toLowerCase();
|
|
309
|
+
if (/\btext[-_\s]?only\b/.test(n)) return true;
|
|
310
|
+
if (
|
|
311
|
+
n.includes("vl") ||
|
|
312
|
+
n.includes("vision") ||
|
|
313
|
+
n.includes("janus") ||
|
|
314
|
+
/\b(?:ocr|pixtral|llava|internvl|minicpm-v|cogvlm|molmo|qvq|omni)\b/.test(n)
|
|
315
|
+
) {
|
|
316
|
+
return false;
|
|
317
|
+
}
|
|
318
|
+
if (/\dv(?=[-_./\s:]|$)/.test(n)) return false; // glm-4v, glm-4.5v — vision variants
|
|
319
|
+
if (/deepseek[-_./\s]*(?:v[34]|r1|chat|coder|reasoner)/.test(n)) return true;
|
|
320
|
+
if (/glm[-_./\s]*[45]/.test(n)) return true;
|
|
321
|
+
if (/\bmercury[-_./\s]*(?:2|3|coder|decide|small|mini)\b/.test(n)) return true;
|
|
322
|
+
if (/\bswe[-_./\s]*[12]\b/.test(n)) return true;
|
|
323
|
+
if (/\bgpt[-_./\s]*oss\b/.test(n)) return true;
|
|
324
|
+
if (/\bqwen[-_./\s]*(?:2(?:\.5)?|3(?:\.\d+)?)[-_./\s]*coder\b/.test(n)) return true;
|
|
325
|
+
if (/\bllama[-_./\s]*3(?:\.[13])?(?=[-_./\s:]|$)/.test(n)) return true;
|
|
326
|
+
if (/\b(?:codestral|devstral|magistral)\b/.test(n)) return true;
|
|
280
327
|
return false;
|
|
281
328
|
}
|
|
282
329
|
|
|
@@ -340,6 +387,25 @@ export function isVisionModel(model: { input?: ("text" | "image")[] } | undefine
|
|
|
340
387
|
return !!model && Array.isArray(model.input) && model.input.includes("image");
|
|
341
388
|
}
|
|
342
389
|
|
|
390
|
+
/** Determine effective vision capability by combining Pi's registry `input`,
|
|
391
|
+
* optional live router `/models` metadata, explicit user `modelOverrides`, and
|
|
392
|
+
* false-vision model heuristics. */
|
|
393
|
+
export function isEffectiveVisionModel(
|
|
394
|
+
model: { id?: string; name?: string; input?: ("text" | "image")[] } | undefined | null,
|
|
395
|
+
routerCap?: { vision?: boolean; label?: string },
|
|
396
|
+
options?: { hasExplicitTextOverride?: boolean },
|
|
397
|
+
): boolean {
|
|
398
|
+
if (!model) return false;
|
|
399
|
+
if (options?.hasExplicitTextOverride) return false;
|
|
400
|
+
const label = [model.name, routerCap?.label]
|
|
401
|
+
.filter((s): s is string => typeof s === "string" && s.trim().length > 0)
|
|
402
|
+
.join(" ");
|
|
403
|
+
if (routerCap?.vision === false) return false;
|
|
404
|
+
if (isKnownTextOnlyFalselyVision(model.id, label)) return false;
|
|
405
|
+
if (routerCap?.vision === true) return true;
|
|
406
|
+
return isVisionModel(model);
|
|
407
|
+
}
|
|
408
|
+
|
|
343
409
|
/** Merge a parsed config object onto defaults, tolerating missing/invalid fields. */
|
|
344
410
|
export function normalizeConfig(raw: unknown): VisionHandoffConfig {
|
|
345
411
|
const base: VisionHandoffConfig = { ...DEFAULT_CONFIG };
|
package/src/router.ts
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Router capability discovery for OpenAI-compatible routing proxies (Magpie,
|
|
3
|
+
* 9router, omnirouter, OpenRouter, LiteLLM, etc.).
|
|
4
|
+
*
|
|
5
|
+
* Proxy entries in `models.json` frequently mark every model with
|
|
6
|
+
* `input: ["text", "image"]` (or omit `input` altogether), even when the
|
|
7
|
+
* router's own `/models` endpoint knows which models are text-only. Querying
|
|
8
|
+
* `/models` with a short timeout lets pi-vision-watcher distinguish
|
|
9
|
+
* vision-capable vs. text-only models automatically for both `/vision-watcher`
|
|
10
|
+
* and active-model handoff.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
14
|
+
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
15
|
+
|
|
16
|
+
/** Per-model capability metadata parsed from a router's `/models` endpoint. */
|
|
17
|
+
export interface RouterModelCapability {
|
|
18
|
+
id: string;
|
|
19
|
+
/** `true` = explicitly supports image input; `false` = explicitly text-only;
|
|
20
|
+
* `undefined` = omitted or unparseable. */
|
|
21
|
+
vision?: boolean;
|
|
22
|
+
/** Display name / router label / description used for false-vision heuristics
|
|
23
|
+
* on aliased models (e.g. `"Abliterated Large v2 (GLM-5.3, uncensored)"`). */
|
|
24
|
+
label?: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const DIRECT_VENDOR_HOSTS = new Set([
|
|
28
|
+
"api.openai.com",
|
|
29
|
+
"api.anthropic.com",
|
|
30
|
+
"generativelanguage.googleapis.com",
|
|
31
|
+
"api.deepseek.com",
|
|
32
|
+
"api.groq.com",
|
|
33
|
+
"api.mistral.ai",
|
|
34
|
+
"api.x.ai",
|
|
35
|
+
"api.cerebras.ai",
|
|
36
|
+
"api.together.xyz",
|
|
37
|
+
"api.fireworks.ai",
|
|
38
|
+
"api.minimax.io",
|
|
39
|
+
"api.moonshot.cn",
|
|
40
|
+
"api.moonshot.ai",
|
|
41
|
+
"open.bigmodel.cn",
|
|
42
|
+
"api.z.ai",
|
|
43
|
+
"dashscope.aliyuncs.com",
|
|
44
|
+
"dashscope-intl.aliyuncs.com",
|
|
45
|
+
]);
|
|
46
|
+
|
|
47
|
+
const DEFAULT_TIMEOUT_MS = 1_500;
|
|
48
|
+
const DEFAULT_TTL_MS = 5 * 60_000;
|
|
49
|
+
|
|
50
|
+
function firstStringArray(...candidates: unknown[]): string[] | undefined {
|
|
51
|
+
for (const c of candidates) {
|
|
52
|
+
if (!Array.isArray(c)) continue;
|
|
53
|
+
const strings = c.filter((v): v is string => typeof v === "string" && v.trim().length > 0);
|
|
54
|
+
if (strings.length > 0) return strings;
|
|
55
|
+
}
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function firstBoolean(...candidates: unknown[]): boolean | undefined {
|
|
60
|
+
for (const c of candidates) {
|
|
61
|
+
if (typeof c === "boolean") return c;
|
|
62
|
+
}
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function extractEntryVision(entry: Record<string, unknown>): boolean | undefined {
|
|
67
|
+
const modalities = entry.modalities as Record<string, unknown> | undefined;
|
|
68
|
+
const architecture = entry.architecture as Record<string, unknown> | undefined;
|
|
69
|
+
const capabilities = entry.capabilities as Record<string, unknown> | undefined;
|
|
70
|
+
|
|
71
|
+
const inputMods = firstStringArray(
|
|
72
|
+
modalities?.input,
|
|
73
|
+
architecture?.input_modalities,
|
|
74
|
+
entry.input_modalities,
|
|
75
|
+
capabilities?.input_modalities,
|
|
76
|
+
entry.input,
|
|
77
|
+
);
|
|
78
|
+
if (inputMods) {
|
|
79
|
+
return inputMods.some((m) => /^(?:image|vision|multimodal)$/i.test(m.trim()));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (typeof architecture?.modality === "string" && architecture.modality.trim()) {
|
|
83
|
+
const [inputSide] = architecture.modality.split("->");
|
|
84
|
+
return /\bimage\b/i.test(inputSide ?? architecture.modality);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
return firstBoolean(
|
|
88
|
+
entry.supports_vision,
|
|
89
|
+
capabilities?.vision,
|
|
90
|
+
capabilities?.image,
|
|
91
|
+
capabilities?.multimodal,
|
|
92
|
+
entry.vision,
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function extractEntryLabel(entry: Record<string, unknown>): string | undefined {
|
|
97
|
+
const parts: string[] = [];
|
|
98
|
+
const seen = new Set<string>();
|
|
99
|
+
for (const raw of [entry.magpie_label, entry.display_name, entry.name]) {
|
|
100
|
+
if (typeof raw !== "string") continue;
|
|
101
|
+
const trimmed = raw.trim();
|
|
102
|
+
if (!trimmed || seen.has(trimmed.toLowerCase())) continue;
|
|
103
|
+
seen.add(trimmed.toLowerCase());
|
|
104
|
+
parts.push(trimmed);
|
|
105
|
+
}
|
|
106
|
+
return parts.length > 0 ? parts.join(" · ") : undefined;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Parse a router `/models` JSON payload into a map keyed by model id. */
|
|
110
|
+
export function parseRouterModelsPayload(payload: unknown): Map<string, RouterModelCapability> {
|
|
111
|
+
const out = new Map<string, RouterModelCapability>();
|
|
112
|
+
let list: unknown;
|
|
113
|
+
if (Array.isArray(payload)) {
|
|
114
|
+
list = payload;
|
|
115
|
+
} else if (payload && typeof payload === "object") {
|
|
116
|
+
const obj = payload as Record<string, unknown>;
|
|
117
|
+
list = Array.isArray(obj.data) ? obj.data : Array.isArray(obj.models) ? obj.models : undefined;
|
|
118
|
+
}
|
|
119
|
+
if (!Array.isArray(list)) return out;
|
|
120
|
+
|
|
121
|
+
for (const raw of list) {
|
|
122
|
+
if (!raw || typeof raw !== "object") continue;
|
|
123
|
+
const entry = raw as Record<string, unknown>;
|
|
124
|
+
if (typeof entry.id !== "string" || !entry.id.trim()) continue;
|
|
125
|
+
const id = entry.id.trim();
|
|
126
|
+
const vision = extractEntryVision(entry);
|
|
127
|
+
const label = extractEntryLabel(entry);
|
|
128
|
+
out.set(id, { id, vision, label });
|
|
129
|
+
}
|
|
130
|
+
return out;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** True when `baseUrl` points at a well-known direct cloud vendor API whose
|
|
134
|
+
* built-in Pi catalog already carries accurate capabilities. */
|
|
135
|
+
export function isDirectVendorBaseUrl(baseUrl: string | undefined | null): boolean {
|
|
136
|
+
if (!baseUrl) return false;
|
|
137
|
+
try {
|
|
138
|
+
const host = new URL(baseUrl).hostname.toLowerCase();
|
|
139
|
+
return DIRECT_VENDOR_HOSTS.has(host);
|
|
140
|
+
} catch {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Candidate `/models` URLs for an OpenAI-compatible `baseUrl`. */
|
|
146
|
+
export function routerModelsUrls(baseUrl: string): string[] {
|
|
147
|
+
const clean = baseUrl.trim().replace(/\/+$/, "");
|
|
148
|
+
if (!clean) return [];
|
|
149
|
+
if (/\/v\d+(?:[a-z0-9]*)?$/i.test(clean)) return [`${clean}/models`];
|
|
150
|
+
return [`${clean}/models`, `${clean}/v1/models`];
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Fetch and parse `/models` from a router `baseUrl`. Never throws; returns an
|
|
154
|
+
* empty map on network, timeout, auth, or parse failure. */
|
|
155
|
+
export async function fetchRouterCapabilities(
|
|
156
|
+
baseUrl: string,
|
|
157
|
+
headers?: Record<string, string>,
|
|
158
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
159
|
+
fetchImpl: typeof fetch = fetch,
|
|
160
|
+
): Promise<Map<string, RouterModelCapability>> {
|
|
161
|
+
const urls = routerModelsUrls(baseUrl);
|
|
162
|
+
for (const url of urls) {
|
|
163
|
+
const ac = new AbortController();
|
|
164
|
+
const timer = setTimeout(() => ac.abort(), timeoutMs);
|
|
165
|
+
try {
|
|
166
|
+
const res = await fetchImpl(url, {
|
|
167
|
+
method: "GET",
|
|
168
|
+
headers: { Accept: "application/json", ...headers },
|
|
169
|
+
signal: ac.signal,
|
|
170
|
+
});
|
|
171
|
+
if (!res.ok) continue;
|
|
172
|
+
const json = (await res.json()) as unknown;
|
|
173
|
+
return parseRouterModelsPayload(json);
|
|
174
|
+
} catch {
|
|
175
|
+
// Offline router, timeout, or non-JSON response: try next candidate or degrade.
|
|
176
|
+
} finally {
|
|
177
|
+
clearTimeout(timer);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return new Map();
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
interface CacheEntry {
|
|
184
|
+
baseUrl: string;
|
|
185
|
+
fetchedAt: number;
|
|
186
|
+
models: Map<string, RouterModelCapability>;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Session-scoped cache of router `/models` capabilities keyed by provider id. */
|
|
190
|
+
export class RouterCapabilityCache {
|
|
191
|
+
private readonly entries = new Map<string, CacheEntry>();
|
|
192
|
+
private readonly inflight = new Map<string, Promise<void>>();
|
|
193
|
+
private readonly ttlMs: number;
|
|
194
|
+
private readonly timeoutMs: number;
|
|
195
|
+
private readonly fetchImpl?: typeof fetch;
|
|
196
|
+
|
|
197
|
+
constructor(options?: { ttlMs?: number; timeoutMs?: number; fetchImpl?: typeof fetch }) {
|
|
198
|
+
this.ttlMs = options?.ttlMs ?? DEFAULT_TTL_MS;
|
|
199
|
+
this.timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
200
|
+
this.fetchImpl = options?.fetchImpl;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
clear(): void {
|
|
204
|
+
this.entries.clear();
|
|
205
|
+
this.inflight.clear();
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
get(provider: string | undefined, modelId: string | undefined): RouterModelCapability | undefined {
|
|
209
|
+
if (!provider || !modelId) return undefined;
|
|
210
|
+
return this.entries.get(provider)?.models.get(modelId);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
hasFresh(provider: string | undefined): boolean {
|
|
214
|
+
if (!provider) return false;
|
|
215
|
+
const entry = this.entries.get(provider);
|
|
216
|
+
return !!entry && Date.now() - entry.fetchedAt < this.ttlMs;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Refresh router capabilities for all (or specific) router providers in
|
|
220
|
+
* `modelRegistry`. Concurrent refreshes for the same provider share one
|
|
221
|
+
* request; failures degrade cleanly without throwing. */
|
|
222
|
+
async refreshProviders(
|
|
223
|
+
modelRegistry: Partial<ModelRegistry> | undefined | null,
|
|
224
|
+
options?: { providers?: string[]; force?: boolean },
|
|
225
|
+
): Promise<void> {
|
|
226
|
+
if (!modelRegistry?.getAvailable) return;
|
|
227
|
+
const targetSet = options?.providers ? new Set(options.providers) : null;
|
|
228
|
+
const force = options?.force ?? false;
|
|
229
|
+
|
|
230
|
+
const byProvider = new Map<string, { baseUrl: string; sampleModel: Model<Api> }>();
|
|
231
|
+
for (const model of modelRegistry.getAvailable()) {
|
|
232
|
+
if (!model.provider) continue;
|
|
233
|
+
if (targetSet && !targetSet.has(model.provider)) continue;
|
|
234
|
+
if (byProvider.has(model.provider)) continue;
|
|
235
|
+
const providerCfg = modelRegistry.getProvider?.(model.provider);
|
|
236
|
+
const baseUrl = model.baseUrl || providerCfg?.baseUrl;
|
|
237
|
+
if (!baseUrl || isDirectVendorBaseUrl(baseUrl)) continue;
|
|
238
|
+
byProvider.set(model.provider, { baseUrl, sampleModel: model });
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const tasks: Promise<void>[] = [];
|
|
242
|
+
for (const [provider, { baseUrl, sampleModel }] of byProvider) {
|
|
243
|
+
const existing = this.entries.get(provider);
|
|
244
|
+
const fresh = existing && existing.baseUrl === baseUrl && Date.now() - existing.fetchedAt < this.ttlMs;
|
|
245
|
+
if (!force && fresh) continue;
|
|
246
|
+
|
|
247
|
+
const pending = this.inflight.get(provider);
|
|
248
|
+
if (pending) {
|
|
249
|
+
tasks.push(pending);
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const task = (async () => {
|
|
254
|
+
try {
|
|
255
|
+
const headers = await resolveRequestHeaders(modelRegistry, sampleModel);
|
|
256
|
+
const models = await fetchRouterCapabilities(baseUrl, headers, this.timeoutMs, this.fetchImpl ?? globalThis.fetch);
|
|
257
|
+
this.entries.set(provider, { baseUrl, fetchedAt: Date.now(), models });
|
|
258
|
+
} finally {
|
|
259
|
+
this.inflight.delete(provider);
|
|
260
|
+
}
|
|
261
|
+
})();
|
|
262
|
+
this.inflight.set(provider, task);
|
|
263
|
+
tasks.push(task);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
await Promise.allSettled(tasks);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
async function resolveRequestHeaders(
|
|
271
|
+
modelRegistry: Partial<ModelRegistry>,
|
|
272
|
+
sampleModel: Model<Api>,
|
|
273
|
+
): Promise<Record<string, string> | undefined> {
|
|
274
|
+
if (!modelRegistry.getApiKeyAndHeaders) return undefined;
|
|
275
|
+
try {
|
|
276
|
+
const auth = await modelRegistry.getApiKeyAndHeaders(sampleModel);
|
|
277
|
+
if (!auth?.ok) return undefined;
|
|
278
|
+
const headers: Record<string, string> = {};
|
|
279
|
+
if (auth.headers) {
|
|
280
|
+
for (const [k, v] of Object.entries(auth.headers)) {
|
|
281
|
+
if (typeof v === "string") headers[k] = v;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
if (auth.apiKey && !Object.keys(headers).some((k) => k.toLowerCase() === "authorization")) {
|
|
285
|
+
headers.Authorization = `Bearer ${auth.apiKey}`;
|
|
286
|
+
}
|
|
287
|
+
return Object.keys(headers).length > 0 ? headers : undefined;
|
|
288
|
+
} catch {
|
|
289
|
+
return undefined;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
@@ -29,7 +29,7 @@ import {
|
|
|
29
29
|
import type { Theme, ThemeColor } from "@earendil-works/pi-coding-agent";
|
|
30
30
|
import type { ThinkingLevel } from "@earendil-works/pi-ai";
|
|
31
31
|
import { keyText } from "@earendil-works/pi-coding-agent";
|
|
32
|
-
import { formatModelRef,
|
|
32
|
+
import { formatModelRef, isEffectiveVisionModel, THINKING_LEVELS } from "./index.js";
|
|
33
33
|
|
|
34
34
|
/** Failover chain length cap — three is already far past the point where a
|
|
35
35
|
* fourth describer would ever be reached. */
|
|
@@ -204,6 +204,9 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
204
204
|
private typing = false;
|
|
205
205
|
/** Transient hint (e.g. chain cap hit), rendered above the legend. */
|
|
206
206
|
private notice: string | null = null;
|
|
207
|
+
/** Whether the user explicitly toggled primary with Space or edited the fallback chain. */
|
|
208
|
+
private touchedPrimary = false;
|
|
209
|
+
private editedChain = false;
|
|
207
210
|
|
|
208
211
|
private _focused = false;
|
|
209
212
|
get focused(): boolean {
|
|
@@ -413,15 +416,14 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
413
416
|
return;
|
|
414
417
|
}
|
|
415
418
|
|
|
416
|
-
// Confirming saves the configuration.
|
|
417
|
-
//
|
|
418
|
-
//
|
|
419
|
-
// model
|
|
420
|
-
// be saved silently.
|
|
419
|
+
// Confirming saves the configuration. Adopt the highlighted row when a filter
|
|
420
|
+
// query is active OR when no primary has been picked yet (and the user
|
|
421
|
+
// hasn't toggled it off with Space or edited fallbacks), so highlighting a
|
|
422
|
+
// model and pressing Enter works intuitively on first setup.
|
|
421
423
|
if (kb.matches(data, "tui.select.confirm") || matchesKey(data, Key.ctrl("s"))) {
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
424
|
+
const item = this.filteredItems[this.selectedIndex];
|
|
425
|
+
if (item && (this.searchInput.getValue() || (this.currentRef === null && !this.touchedPrimary && !this.editedChain && item.ref !== null))) {
|
|
426
|
+
this.currentRef = item.ref;
|
|
425
427
|
}
|
|
426
428
|
this.save();
|
|
427
429
|
return;
|
|
@@ -574,13 +576,13 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
574
576
|
provider: m.provider,
|
|
575
577
|
modelId: m.id,
|
|
576
578
|
modelName: m.name || m.id,
|
|
577
|
-
vision:
|
|
579
|
+
vision: isEffectiveVisionModel(m),
|
|
578
580
|
reasoning: !!m.reasoning,
|
|
579
581
|
});
|
|
580
582
|
|
|
581
583
|
// Only vision-capable models are listed — a text-only model can't describe
|
|
582
584
|
// images, so it would only produce "[Image: description unavailable]" errors.
|
|
583
|
-
const visionModels = allModels.filter((m) =>
|
|
585
|
+
const visionModels = allModels.filter((m) => isEffectiveVisionModel(m)).map(make);
|
|
584
586
|
return [...items, ...visionModels];
|
|
585
587
|
}
|
|
586
588
|
|
|
@@ -645,6 +647,7 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
645
647
|
this.allItems.splice(to, 0, item!);
|
|
646
648
|
this.filteredItems = this.allItems;
|
|
647
649
|
this.selectedIndex = to;
|
|
650
|
+
this.editedChain = true;
|
|
648
651
|
this.updateList();
|
|
649
652
|
}
|
|
650
653
|
|
|
@@ -652,6 +655,7 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
652
655
|
if (this.fallbacks.size === 0) return;
|
|
653
656
|
this.fallbacks.clear();
|
|
654
657
|
this.notice = null;
|
|
658
|
+
this.editedChain = true;
|
|
655
659
|
this.updateList();
|
|
656
660
|
}
|
|
657
661
|
|
|
@@ -659,6 +663,7 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
659
663
|
* (the chain is tried in order, so the config array must be deterministic
|
|
660
664
|
* rather than Set-iteration order). */
|
|
661
665
|
private toggleFallback(ref: string): void {
|
|
666
|
+
this.editedChain = true;
|
|
662
667
|
if (this.fallbacks.has(ref)) {
|
|
663
668
|
this.fallbacks.delete(ref);
|
|
664
669
|
this.notice = null;
|
|
@@ -674,6 +679,7 @@ export class VisionModelSelectorComponent implements Component {
|
|
|
674
679
|
/** Space toggles the primary describer; picking the current one again clears
|
|
675
680
|
* it (same as the None row), so one key both sets and unsets. */
|
|
676
681
|
private selectPrimary(ref: string | null): void {
|
|
682
|
+
this.touchedPrimary = true;
|
|
677
683
|
this.currentRef = this.currentRef === ref ? null : ref;
|
|
678
684
|
this.notice = null;
|
|
679
685
|
this.updateList();
|
package/vision-watcher.ts
CHANGED
|
@@ -40,6 +40,7 @@ import {
|
|
|
40
40
|
DESCRIBE_TIMEOUT_MS,
|
|
41
41
|
extractImageFromBlock,
|
|
42
42
|
formatModelRef,
|
|
43
|
+
isEffectiveVisionModel,
|
|
43
44
|
isThinkingLevel,
|
|
44
45
|
assistantErrorText,
|
|
45
46
|
isImageNotSupportedError,
|
|
@@ -55,6 +56,7 @@ import {
|
|
|
55
56
|
type ExtractedImage,
|
|
56
57
|
type VisionHandoffConfig,
|
|
57
58
|
} from "./src/index.js";
|
|
59
|
+
import { RouterCapabilityCache } from "./src/router.js";
|
|
58
60
|
import {
|
|
59
61
|
USAGE_ENTRY_TYPE,
|
|
60
62
|
USAGE_EVENT_CHANNEL,
|
|
@@ -99,6 +101,67 @@ let editorInstalled = false;
|
|
|
99
101
|
|
|
100
102
|
let visionModelCache: { ref: string; model: Model<Api> } | null = null;
|
|
101
103
|
let visionModelUnresolvedRef: string | null = null;
|
|
104
|
+
const routerCache = new RouterCapabilityCache();
|
|
105
|
+
const pastedPathImages = new Map<string, Promise<ExtractedImage | null>>();
|
|
106
|
+
|
|
107
|
+
function preparePastedPathImage(path: string): Promise<ExtractedImage | null> {
|
|
108
|
+
const existing = pastedPathImages.get(path);
|
|
109
|
+
if (existing) {
|
|
110
|
+
pastedPathImages.delete(path);
|
|
111
|
+
pastedPathImages.set(path, existing);
|
|
112
|
+
return existing;
|
|
113
|
+
}
|
|
114
|
+
const task = (async (): Promise<ExtractedImage | null> => {
|
|
115
|
+
try {
|
|
116
|
+
const read = readImageBuffer(path);
|
|
117
|
+
if (!read) return null;
|
|
118
|
+
return await resolvePrewarmImage(read.buf, read.mimeType, resizeImage);
|
|
119
|
+
} catch {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
})();
|
|
123
|
+
pastedPathImages.set(path, task);
|
|
124
|
+
const cap = Math.max(50, config.cacheMax);
|
|
125
|
+
while (pastedPathImages.size > cap) {
|
|
126
|
+
const oldest = pastedPathImages.keys().next().value;
|
|
127
|
+
if (oldest === undefined) break;
|
|
128
|
+
pastedPathImages.delete(oldest);
|
|
129
|
+
}
|
|
130
|
+
return task;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function readExplicitTextOnlyOverrides(): Set<string> {
|
|
134
|
+
const out = new Set<string>();
|
|
135
|
+
const path = join(getAgentDir(), "models.json");
|
|
136
|
+
if (!existsSync(path)) return out;
|
|
137
|
+
try {
|
|
138
|
+
const raw = JSON.parse(readFileSync(path, "utf8")) as {
|
|
139
|
+
providers?: Record<string, { modelOverrides?: Record<string, { input?: unknown }> }>;
|
|
140
|
+
};
|
|
141
|
+
const providers = raw?.providers;
|
|
142
|
+
if (!providers || typeof providers !== "object") return out;
|
|
143
|
+
for (const [providerId, provider] of Object.entries(providers)) {
|
|
144
|
+
const overrides = provider?.modelOverrides;
|
|
145
|
+
if (!overrides || typeof overrides !== "object") continue;
|
|
146
|
+
for (const [modelId, ov] of Object.entries(overrides)) {
|
|
147
|
+
if (Array.isArray(ov?.input) && !ov.input.includes("image")) {
|
|
148
|
+
out.add(formatModelRef(providerId, modelId));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
} catch {
|
|
153
|
+
// Corrupt or unreadable models.json — fall back cleanly.
|
|
154
|
+
}
|
|
155
|
+
return out;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async function ensureRouterCapabilitiesForModel(
|
|
159
|
+
ctx: { modelRegistry?: Partial<ModelRegistry> },
|
|
160
|
+
model: { provider?: string } | undefined | null,
|
|
161
|
+
): Promise<void> {
|
|
162
|
+
if (!model?.provider || routerCache.hasFresh(model.provider)) return;
|
|
163
|
+
await routerCache.refreshProviders(ctx.modelRegistry, { providers: [model.provider] });
|
|
164
|
+
}
|
|
102
165
|
|
|
103
166
|
interface PendingAsyncClipboardHandoff {
|
|
104
167
|
token: symbol;
|
|
@@ -214,8 +277,16 @@ function resolveVisionModel(modelRegistry: ModelRegistry, ref: string): Model<Ap
|
|
|
214
277
|
if (!parsed) return null;
|
|
215
278
|
const model = modelRegistry.find(parsed.provider, parsed.id);
|
|
216
279
|
if (!model) return null;
|
|
217
|
-
|
|
218
|
-
|
|
280
|
+
// If router /models metadata confirmed this model supports image input while
|
|
281
|
+
// models.json omitted `input`, expose `"image"` on the describer clone so
|
|
282
|
+
// pi-ai adapters don't downgrade or strip image blocks.
|
|
283
|
+
const cap = routerCache.get(model.provider, model.id);
|
|
284
|
+
const resolved =
|
|
285
|
+
!isVisionModel(model) && isEffectiveVisionModel(model, cap)
|
|
286
|
+
? ({ ...model, input: ["text", "image"] } as Model<Api>)
|
|
287
|
+
: model;
|
|
288
|
+
visionModelCache = { ref, model: resolved };
|
|
289
|
+
return resolved;
|
|
219
290
|
}
|
|
220
291
|
|
|
221
292
|
/** True when the primary vision model resolves, or — failing that — at least
|
|
@@ -246,18 +317,21 @@ function isConfigured(cfg: VisionHandoffConfig): boolean {
|
|
|
246
317
|
}
|
|
247
318
|
|
|
248
319
|
function isHandoffTarget(
|
|
249
|
-
model: { provider?: string; id?: string; input?: ("text" | "image")[] } | undefined | null,
|
|
320
|
+
model: { provider?: string; id?: string; name?: string; input?: ("text" | "image")[] } | undefined | null,
|
|
250
321
|
cfg: VisionHandoffConfig,
|
|
251
322
|
): boolean {
|
|
252
323
|
if (!model || !model.provider || !model.id) return false;
|
|
253
324
|
const ref = formatModelRef(model.provider, model.id);
|
|
254
325
|
if (cfg.handoffModels.includes(ref)) return true;
|
|
255
|
-
|
|
256
|
-
//
|
|
257
|
-
//
|
|
258
|
-
if (
|
|
259
|
-
|
|
260
|
-
|
|
326
|
+
if (!cfg.autoHandoff) return false;
|
|
327
|
+
// Pi itself strips image blocks when model.input omits "image", so handoff
|
|
328
|
+
// must always run whenever the registry entry is non-vision.
|
|
329
|
+
if (!isVisionModel(model)) return true;
|
|
330
|
+
// Aggregators/routers (Magpie, 9router, omnirouter) often mark text-only
|
|
331
|
+
// models as ["text", "image"] in models.json. Check router /models metadata
|
|
332
|
+
// and false-vision heuristics so FIRST send hands off.
|
|
333
|
+
const cap = routerCache.get(model.provider, model.id);
|
|
334
|
+
return !isEffectiveVisionModel(model, cap);
|
|
261
335
|
}
|
|
262
336
|
|
|
263
337
|
/** Whether paste-time prewarm should fire for a text change right now: the
|
|
@@ -277,9 +351,7 @@ function shouldPrewarmPaste(): boolean {
|
|
|
277
351
|
* tradeoff of the opt-in). If the user submits before this dispatch fires,
|
|
278
352
|
* before_agent_start overwrites the prompt — a benign bonus, not a bug. */
|
|
279
353
|
function prewarmClipboardPath(path: string, modelRegistry: ModelRegistry): void {
|
|
280
|
-
|
|
281
|
-
if (!read) return;
|
|
282
|
-
resolvePrewarmImage(read.buf, read.mimeType, resizeImage)
|
|
354
|
+
preparePastedPathImage(path)
|
|
283
355
|
.then((img) => {
|
|
284
356
|
if (!img) return;
|
|
285
357
|
// Paste happens while the agent is idle (no run → no live signal).
|
|
@@ -350,14 +422,19 @@ function fixModelInputTextOnly(ctx: ExtensionContext, providerId: string, modelI
|
|
|
350
422
|
}
|
|
351
423
|
|
|
352
424
|
/** Persist `input: ["text"]` for a model that falsely declares image input.
|
|
353
|
-
* No-op if the model is already text-only or isn't
|
|
425
|
+
* No-op if the model is already text-only or isn't detected as text-only. */
|
|
354
426
|
function persistFalseVisionOverride(
|
|
355
427
|
ctx: ExtensionContext,
|
|
356
|
-
model: { provider?: string; id?: string; input?: ("text" | "image")[] } | undefined | null,
|
|
428
|
+
model: { provider?: string; id?: string; name?: string; input?: ("text" | "image")[] } | undefined | null,
|
|
357
429
|
): boolean {
|
|
358
430
|
if (!model?.provider || !model.id) return false;
|
|
359
|
-
if (!isKnownTextOnlyFalselyVision(model.id)) return false;
|
|
360
431
|
if (!isVisionModel(model)) return false;
|
|
432
|
+
const cap = routerCache.get(model.provider, model.id);
|
|
433
|
+
const label = [model.name, cap?.label]
|
|
434
|
+
.filter((s): s is string => typeof s === "string" && s.trim().length > 0)
|
|
435
|
+
.join(" ");
|
|
436
|
+
const isFalseVision = cap?.vision === false || isKnownTextOnlyFalselyVision(model.id, label);
|
|
437
|
+
if (!isFalseVision) return false;
|
|
361
438
|
return fixModelInputTextOnly(ctx, model.provider, model.id);
|
|
362
439
|
}
|
|
363
440
|
|
|
@@ -464,16 +541,27 @@ export default function (pi: ExtensionAPI) {
|
|
|
464
541
|
config = readConfig();
|
|
465
542
|
visionModelCache = null;
|
|
466
543
|
visionModelUnresolvedRef = null;
|
|
544
|
+
routerCache.clear();
|
|
545
|
+
pastedPathImages.clear();
|
|
467
546
|
warnedHashes.clear();
|
|
468
547
|
loader.reset();
|
|
469
548
|
currentModel = ctx.model;
|
|
470
549
|
persistFalseVisionOverride(ctx, ctx.model);
|
|
471
550
|
installPrewarmEditor(ctx);
|
|
551
|
+
// Warm router /models metadata in the background so /vision-watcher and
|
|
552
|
+
// active false-vision detection stay accurate without slowing startup.
|
|
553
|
+
void routerCache
|
|
554
|
+
.refreshProviders(ctx.modelRegistry)
|
|
555
|
+
.then(() => {
|
|
556
|
+
persistFalseVisionOverride(ctx, currentModel);
|
|
557
|
+
})
|
|
558
|
+
.catch(() => {});
|
|
472
559
|
});
|
|
473
560
|
|
|
474
561
|
pi.on("before_agent_start", async (event, ctx) => {
|
|
475
562
|
cancelAsyncClipboardHandoff();
|
|
476
563
|
if (!isConfigured(config)) return;
|
|
564
|
+
await ensureRouterCapabilitiesForModel(ctx, ctx.model);
|
|
477
565
|
if (!isHandoffTarget(ctx.model, config)) return;
|
|
478
566
|
|
|
479
567
|
// Fresh turn → fresh turn-abort controller. `before_agent_start` fires
|
|
@@ -541,9 +629,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
541
629
|
const preparedClipboardImages = clipboardPaths.map(
|
|
542
630
|
async (path): Promise<PreparedClipboardImage | null> => {
|
|
543
631
|
try {
|
|
544
|
-
const
|
|
545
|
-
if (!read) return null;
|
|
546
|
-
const image = await resolvePrewarmImage(read.buf, read.mimeType, resizeImage);
|
|
632
|
+
const image = await preparePastedPathImage(path);
|
|
547
633
|
if (!image) return null;
|
|
548
634
|
const description = loader.loadDescription(image);
|
|
549
635
|
description.catch(() => {});
|
|
@@ -605,6 +691,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
605
691
|
const content = event.content;
|
|
606
692
|
if (!Array.isArray(content)) return;
|
|
607
693
|
|
|
694
|
+
await ensureRouterCapabilitiesForModel(ctx, ctx.model);
|
|
608
695
|
// Collect image blocks in this read result. (The read tool emits image
|
|
609
696
|
// blocks even for non-vision models — they reach here untouched.)
|
|
610
697
|
if (!isHandoffTarget(ctx.model, config)) return;
|
|
@@ -731,19 +818,67 @@ export default function (pi: ExtensionAPI) {
|
|
|
731
818
|
if (!Array.isArray(messages)) return;
|
|
732
819
|
|
|
733
820
|
const byHash = new Map<string, ExtractedImage>();
|
|
734
|
-
|
|
821
|
+
const msgImageHashes = new Map<Record<string, unknown>, Set<string>>();
|
|
822
|
+
const userPastedPaths: Array<{ msg: Record<string, unknown>; path: string }> = [];
|
|
823
|
+
|
|
735
824
|
for (const msg of messages) {
|
|
736
825
|
const content = msg.content;
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
const
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
826
|
+
const hashes = new Set<string>();
|
|
827
|
+
if (Array.isArray(content)) {
|
|
828
|
+
for (const block of content) {
|
|
829
|
+
const img = extractImageFromBlock(block);
|
|
830
|
+
if (!img) continue;
|
|
831
|
+
const h = imageHash(img.mimeType, img.data);
|
|
832
|
+
hashes.add(h);
|
|
833
|
+
byHash.set(h, img);
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
if (hashes.size > 0) msgImageHashes.set(msg, hashes);
|
|
837
|
+
|
|
838
|
+
// When asyncClipboardHandoff is off (default), forward pasted temp-dir
|
|
839
|
+
// clipboard image paths directly in the user message on Turn 1 so the
|
|
840
|
+
// active text-only model receives descriptions without needing to call read.
|
|
841
|
+
if (!config.asyncClipboardHandoff && msg.role === "user") {
|
|
842
|
+
const text =
|
|
843
|
+
typeof content === "string"
|
|
844
|
+
? content
|
|
845
|
+
: Array.isArray(content)
|
|
846
|
+
? content
|
|
847
|
+
.filter((b): b is { type: string; text: string } => !!b && typeof b === "object" && (b as { type?: unknown }).type === "text" && typeof (b as { text?: unknown }).text === "string")
|
|
848
|
+
.map((b) => b.text)
|
|
849
|
+
.join("\n")
|
|
850
|
+
: "";
|
|
851
|
+
if (text) {
|
|
852
|
+
for (const path of findPastedImagePaths(text)) {
|
|
853
|
+
userPastedPaths.push({ msg, path });
|
|
854
|
+
}
|
|
855
|
+
}
|
|
743
856
|
}
|
|
744
857
|
}
|
|
745
|
-
|
|
858
|
+
|
|
859
|
+
if (byHash.size === 0 && userPastedPaths.length === 0) return;
|
|
860
|
+
await ensureRouterCapabilitiesForModel(ctx, ctx.model);
|
|
746
861
|
if (!isHandoffTarget(ctx.model, config)) return;
|
|
862
|
+
|
|
863
|
+
const pastedInjections: Array<{ msg: Record<string, unknown>; path: string; hash: string }> = [];
|
|
864
|
+
if (userPastedPaths.length > 0) {
|
|
865
|
+
const resolved = await Promise.all(
|
|
866
|
+
userPastedPaths.map(async ({ msg, path }) => ({
|
|
867
|
+
msg,
|
|
868
|
+
path,
|
|
869
|
+
img: await preparePastedPathImage(path),
|
|
870
|
+
})),
|
|
871
|
+
);
|
|
872
|
+
for (const item of resolved) {
|
|
873
|
+
if (!item.img) continue;
|
|
874
|
+
const h = imageHash(item.img.mimeType, item.img.data);
|
|
875
|
+
if (msgImageHashes.get(item.msg)?.has(h)) continue;
|
|
876
|
+
byHash.set(h, item.img);
|
|
877
|
+
pastedInjections.push({ msg: item.msg, path: item.path, hash: h });
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
if (byHash.size === 0) return;
|
|
747
882
|
loader.bindTurnContext(ctx);
|
|
748
883
|
if (!isAnyVisionModelResolvable(ctx.modelRegistry, config)) {
|
|
749
884
|
notifyUnresolvedVisionModel(ctx, config.visionModel!);
|
|
@@ -783,6 +918,26 @@ export default function (pi: ExtensionAPI) {
|
|
|
783
918
|
changed = true;
|
|
784
919
|
}
|
|
785
920
|
}
|
|
921
|
+
|
|
922
|
+
for (const item of pastedInjections) {
|
|
923
|
+
const desc = descs.get(item.hash) ?? UNAVAILABLE;
|
|
924
|
+
const rawContent = item.msg.content;
|
|
925
|
+
const blocks: Array<{ type: string; text?: string }> =
|
|
926
|
+
typeof rawContent === "string"
|
|
927
|
+
? [{ type: "text", text: rawContent }]
|
|
928
|
+
: Array.isArray(rawContent)
|
|
929
|
+
? (rawContent as Array<{ type: string; text?: string }>).slice()
|
|
930
|
+
: [];
|
|
931
|
+
const blockText = `${item.path}\n${desc}`;
|
|
932
|
+
const alreadyIncluded = blocks.some(
|
|
933
|
+
(b) => b.type === "text" && typeof b.text === "string" && b.text.includes(blockText),
|
|
934
|
+
);
|
|
935
|
+
if (alreadyIncluded) continue;
|
|
936
|
+
blocks.push({ type: "text", text: blockText });
|
|
937
|
+
item.msg.content = blocks;
|
|
938
|
+
changed = true;
|
|
939
|
+
}
|
|
940
|
+
|
|
786
941
|
if (changed) return { messages: event.messages };
|
|
787
942
|
});
|
|
788
943
|
|
|
@@ -843,8 +998,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
843
998
|
);
|
|
844
999
|
});
|
|
845
1000
|
|
|
846
|
-
pi.on("model_select", (event, ctx) => {
|
|
1001
|
+
pi.on("model_select", async (event, ctx) => {
|
|
847
1002
|
currentModel = event.model;
|
|
1003
|
+
await ensureRouterCapabilitiesForModel(ctx, event.model);
|
|
848
1004
|
persistFalseVisionOverride(ctx, event.model);
|
|
849
1005
|
if (!ctx.hasUI) return;
|
|
850
1006
|
if (!isConfigured(config)) return;
|
|
@@ -919,7 +1075,7 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
919
1075
|
}
|
|
920
1076
|
|
|
921
1077
|
if (subcommand === "status") {
|
|
922
|
-
showStatus(ctx);
|
|
1078
|
+
await showStatus(ctx);
|
|
923
1079
|
return;
|
|
924
1080
|
}
|
|
925
1081
|
|
|
@@ -993,10 +1149,12 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
|
|
|
993
1149
|
return;
|
|
994
1150
|
}
|
|
995
1151
|
const ref = formatModelRef(parsed.provider, parsed.id);
|
|
1152
|
+
await ensureRouterCapabilitiesForModel(ctx, model);
|
|
996
1153
|
updateConfig(ctx, (c) => ({ ...c, visionModel: ref }), `Vision model set to ${ref}.`);
|
|
997
|
-
|
|
1154
|
+
const cap = routerCache.get(model.provider, model.id);
|
|
1155
|
+
if (!isEffectiveVisionModel(model, cap)) {
|
|
998
1156
|
ctx.ui.notify(
|
|
999
|
-
`Note: ${ref}
|
|
1157
|
+
`Note: ${ref} appears text-only — it may not describe images well.`,
|
|
1000
1158
|
"warning",
|
|
1001
1159
|
);
|
|
1002
1160
|
}
|
|
@@ -1175,18 +1333,37 @@ function handleTimeoutSubcommand(ctx: ExtensionCommandContext, rest: string): vo
|
|
|
1175
1333
|
/** Connected, vision-capable models — the same set the pickers show. Text-only
|
|
1176
1334
|
* models can't describe images, so they would only produce
|
|
1177
1335
|
* "[Image: description unavailable]" errors. getAvailable() excludes the whole
|
|
1178
|
-
* catalogue of unauthenticated models.
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1336
|
+
* catalogue of unauthenticated models. Router `/models` metadata and
|
|
1337
|
+
* false-vision heuristics filter out text-only models behind proxies like
|
|
1338
|
+
* Magpie, 9router, and omnirouter. */
|
|
1339
|
+
async function availableVisionModels(ctx: ExtensionCommandContext): Promise<
|
|
1340
|
+
Array<{
|
|
1341
|
+
provider: string;
|
|
1342
|
+
id: string;
|
|
1343
|
+
name: string;
|
|
1344
|
+
input?: ("text" | "image")[];
|
|
1345
|
+
reasoning?: boolean;
|
|
1346
|
+
}>
|
|
1347
|
+
> {
|
|
1348
|
+
await routerCache.refreshProviders(ctx.modelRegistry);
|
|
1349
|
+
const explicitTextOverrides = readExplicitTextOnlyOverrides();
|
|
1350
|
+
const out: Array<{
|
|
1351
|
+
provider: string;
|
|
1352
|
+
id: string;
|
|
1353
|
+
name: string;
|
|
1354
|
+
input?: ("text" | "image")[];
|
|
1355
|
+
reasoning?: boolean;
|
|
1356
|
+
}> = [];
|
|
1357
|
+
|
|
1358
|
+
for (const m of ctx.modelRegistry.getAvailable()) {
|
|
1359
|
+
const ref = formatModelRef(m.provider, m.id);
|
|
1360
|
+
const cap = routerCache.get(m.provider, m.id);
|
|
1361
|
+
const hasExplicitTextOverride = explicitTextOverrides.has(ref);
|
|
1362
|
+
if (!isEffectiveVisionModel(m, cap, { hasExplicitTextOverride })) continue;
|
|
1363
|
+
const input = isVisionModel(m) ? m.input : (["text", "image"] as ("text" | "image")[]);
|
|
1364
|
+
out.push({ provider: m.provider, id: m.id, name: m.name, input, reasoning: m.reasoning });
|
|
1365
|
+
}
|
|
1366
|
+
return out;
|
|
1190
1367
|
}
|
|
1191
1368
|
|
|
1192
1369
|
async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
|
|
@@ -1195,7 +1372,7 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
|
|
|
1195
1372
|
return;
|
|
1196
1373
|
}
|
|
1197
1374
|
|
|
1198
|
-
const availableModels = availableVisionModels(ctx);
|
|
1375
|
+
const availableModels = await availableVisionModels(ctx);
|
|
1199
1376
|
|
|
1200
1377
|
if (ctx.mode !== "tui") {
|
|
1201
1378
|
const modelItems = ["None", ...availableModels.map((m) => `${m.provider}/${m.id}`)];
|
|
@@ -1279,7 +1456,8 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
|
|
|
1279
1456
|
}
|
|
1280
1457
|
}
|
|
1281
1458
|
|
|
1282
|
-
function showStatus(ctx: ExtensionCommandContext): void {
|
|
1459
|
+
async function showStatus(ctx: ExtensionCommandContext): Promise<void> {
|
|
1460
|
+
await ensureRouterCapabilitiesForModel(ctx, ctx.model);
|
|
1283
1461
|
const lines: string[] = [];
|
|
1284
1462
|
lines.push(`Vision handoff: ${config.enabled ? "enabled" : "disabled"}`);
|
|
1285
1463
|
lines.push(`Vision model: ${config.visionModel ?? "(none — pick one with /vision-watcher)"}`);
|