@bismawy/pi-vision-watcher 1.3.1 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/vision-watcher.ts CHANGED
@@ -40,6 +40,7 @@ import {
40
40
  DESCRIBE_TIMEOUT_MS,
41
41
  extractImageFromBlock,
42
42
  formatModelRef,
43
+ isEffectiveVisionModel,
43
44
  isThinkingLevel,
44
45
  assistantErrorText,
45
46
  isImageNotSupportedError,
@@ -55,6 +56,7 @@ import {
55
56
  type ExtractedImage,
56
57
  type VisionHandoffConfig,
57
58
  } from "./src/index.js";
59
+ import { RouterCapabilityCache } from "./src/router.js";
58
60
  import {
59
61
  USAGE_ENTRY_TYPE,
60
62
  USAGE_EVENT_CHANNEL,
@@ -99,6 +101,67 @@ let editorInstalled = false;
99
101
 
100
102
  let visionModelCache: { ref: string; model: Model<Api> } | null = null;
101
103
  let visionModelUnresolvedRef: string | null = null;
104
+ const routerCache = new RouterCapabilityCache();
105
+ const pastedPathImages = new Map<string, Promise<ExtractedImage | null>>();
106
+
107
+ function preparePastedPathImage(path: string): Promise<ExtractedImage | null> {
108
+ const existing = pastedPathImages.get(path);
109
+ if (existing) {
110
+ pastedPathImages.delete(path);
111
+ pastedPathImages.set(path, existing);
112
+ return existing;
113
+ }
114
+ const task = (async (): Promise<ExtractedImage | null> => {
115
+ try {
116
+ const read = readImageBuffer(path);
117
+ if (!read) return null;
118
+ return await resolvePrewarmImage(read.buf, read.mimeType, resizeImage);
119
+ } catch {
120
+ return null;
121
+ }
122
+ })();
123
+ pastedPathImages.set(path, task);
124
+ const cap = Math.max(50, config.cacheMax);
125
+ while (pastedPathImages.size > cap) {
126
+ const oldest = pastedPathImages.keys().next().value;
127
+ if (oldest === undefined) break;
128
+ pastedPathImages.delete(oldest);
129
+ }
130
+ return task;
131
+ }
132
+
133
+ function readExplicitTextOnlyOverrides(): Set<string> {
134
+ const out = new Set<string>();
135
+ const path = join(getAgentDir(), "models.json");
136
+ if (!existsSync(path)) return out;
137
+ try {
138
+ const raw = JSON.parse(readFileSync(path, "utf8")) as {
139
+ providers?: Record<string, { modelOverrides?: Record<string, { input?: unknown }> }>;
140
+ };
141
+ const providers = raw?.providers;
142
+ if (!providers || typeof providers !== "object") return out;
143
+ for (const [providerId, provider] of Object.entries(providers)) {
144
+ const overrides = provider?.modelOverrides;
145
+ if (!overrides || typeof overrides !== "object") continue;
146
+ for (const [modelId, ov] of Object.entries(overrides)) {
147
+ if (Array.isArray(ov?.input) && !ov.input.includes("image")) {
148
+ out.add(formatModelRef(providerId, modelId));
149
+ }
150
+ }
151
+ }
152
+ } catch {
153
+ // Corrupt or unreadable models.json — fall back cleanly.
154
+ }
155
+ return out;
156
+ }
157
+
158
+ async function ensureRouterCapabilitiesForModel(
159
+ ctx: { modelRegistry?: Partial<ModelRegistry> },
160
+ model: { provider?: string } | undefined | null,
161
+ ): Promise<void> {
162
+ if (!model?.provider || routerCache.hasFresh(model.provider)) return;
163
+ await routerCache.refreshProviders(ctx.modelRegistry, { providers: [model.provider] });
164
+ }
102
165
 
103
166
  interface PendingAsyncClipboardHandoff {
104
167
  token: symbol;
@@ -214,8 +277,16 @@ function resolveVisionModel(modelRegistry: ModelRegistry, ref: string): Model<Ap
214
277
  if (!parsed) return null;
215
278
  const model = modelRegistry.find(parsed.provider, parsed.id);
216
279
  if (!model) return null;
217
- visionModelCache = { ref, model };
218
- return model;
280
+ // If router /models metadata confirmed this model supports image input while
281
+ // models.json omitted `input`, expose `"image"` on the describer clone so
282
+ // pi-ai adapters don't downgrade or strip image blocks.
283
+ const cap = routerCache.get(model.provider, model.id);
284
+ const resolved =
285
+ !isVisionModel(model) && isEffectiveVisionModel(model, cap)
286
+ ? ({ ...model, input: ["text", "image"] } as Model<Api>)
287
+ : model;
288
+ visionModelCache = { ref, model: resolved };
289
+ return resolved;
219
290
  }
220
291
 
221
292
  /** True when the primary vision model resolves, or — failing that — at least
@@ -246,18 +317,21 @@ function isConfigured(cfg: VisionHandoffConfig): boolean {
246
317
  }
247
318
 
248
319
  function isHandoffTarget(
249
- model: { provider?: string; id?: string; input?: ("text" | "image")[] } | undefined | null,
320
+ model: { provider?: string; id?: string; name?: string; input?: ("text" | "image")[] } | undefined | null,
250
321
  cfg: VisionHandoffConfig,
251
322
  ): boolean {
252
323
  if (!model || !model.provider || !model.id) return false;
253
324
  const ref = formatModelRef(model.provider, model.id);
254
325
  if (cfg.handoffModels.includes(ref)) return true;
255
- // Aggregators often mark DeepSeek V4 as vision; the backend 400s on images.
256
- // Treat as text-only so the FIRST send is handed off — waiting for the 400
257
- // is too late. Persist the models.json override separately so /model agrees.
258
- if (cfg.autoHandoff && isKnownTextOnlyFalselyVision(model.id)) return true;
259
- if (cfg.autoHandoff && !isVisionModel(model)) return true;
260
- return false;
326
+ if (!cfg.autoHandoff) return false;
327
+ // Pi itself strips image blocks when model.input omits "image", so handoff
328
+ // must always run whenever the registry entry is non-vision.
329
+ if (!isVisionModel(model)) return true;
330
+ // Aggregators/routers (Magpie, 9router, omnirouter) often mark text-only
331
+ // models as ["text", "image"] in models.json. Check router /models metadata
332
+ // and false-vision heuristics so FIRST send hands off.
333
+ const cap = routerCache.get(model.provider, model.id);
334
+ return !isEffectiveVisionModel(model, cap);
261
335
  }
262
336
 
263
337
  /** Whether paste-time prewarm should fire for a text change right now: the
@@ -277,9 +351,7 @@ function shouldPrewarmPaste(): boolean {
277
351
  * tradeoff of the opt-in). If the user submits before this dispatch fires,
278
352
  * before_agent_start overwrites the prompt — a benign bonus, not a bug. */
279
353
  function prewarmClipboardPath(path: string, modelRegistry: ModelRegistry): void {
280
- const read = readImageBuffer(path);
281
- if (!read) return;
282
- resolvePrewarmImage(read.buf, read.mimeType, resizeImage)
354
+ preparePastedPathImage(path)
283
355
  .then((img) => {
284
356
  if (!img) return;
285
357
  // Paste happens while the agent is idle (no run → no live signal).
@@ -350,14 +422,19 @@ function fixModelInputTextOnly(ctx: ExtensionContext, providerId: string, modelI
350
422
  }
351
423
 
352
424
  /** Persist `input: ["text"]` for a model that falsely declares image input.
353
- * No-op if the model is already text-only or isn't a known false-vision id. */
425
+ * No-op if the model is already text-only or isn't detected as text-only. */
354
426
  function persistFalseVisionOverride(
355
427
  ctx: ExtensionContext,
356
- model: { provider?: string; id?: string; input?: ("text" | "image")[] } | undefined | null,
428
+ model: { provider?: string; id?: string; name?: string; input?: ("text" | "image")[] } | undefined | null,
357
429
  ): boolean {
358
430
  if (!model?.provider || !model.id) return false;
359
- if (!isKnownTextOnlyFalselyVision(model.id)) return false;
360
431
  if (!isVisionModel(model)) return false;
432
+ const cap = routerCache.get(model.provider, model.id);
433
+ const label = [model.name, cap?.label]
434
+ .filter((s): s is string => typeof s === "string" && s.trim().length > 0)
435
+ .join(" ");
436
+ const isFalseVision = cap?.vision === false || isKnownTextOnlyFalselyVision(model.id, label);
437
+ if (!isFalseVision) return false;
361
438
  return fixModelInputTextOnly(ctx, model.provider, model.id);
362
439
  }
363
440
 
@@ -464,16 +541,27 @@ export default function (pi: ExtensionAPI) {
464
541
  config = readConfig();
465
542
  visionModelCache = null;
466
543
  visionModelUnresolvedRef = null;
544
+ routerCache.clear();
545
+ pastedPathImages.clear();
467
546
  warnedHashes.clear();
468
547
  loader.reset();
469
548
  currentModel = ctx.model;
470
549
  persistFalseVisionOverride(ctx, ctx.model);
471
550
  installPrewarmEditor(ctx);
551
+ // Warm router /models metadata in the background so /vision-watcher and
552
+ // active false-vision detection stay accurate without slowing startup.
553
+ void routerCache
554
+ .refreshProviders(ctx.modelRegistry)
555
+ .then(() => {
556
+ persistFalseVisionOverride(ctx, currentModel);
557
+ })
558
+ .catch(() => {});
472
559
  });
473
560
 
474
561
  pi.on("before_agent_start", async (event, ctx) => {
475
562
  cancelAsyncClipboardHandoff();
476
563
  if (!isConfigured(config)) return;
564
+ await ensureRouterCapabilitiesForModel(ctx, ctx.model);
477
565
  if (!isHandoffTarget(ctx.model, config)) return;
478
566
 
479
567
  // Fresh turn → fresh turn-abort controller. `before_agent_start` fires
@@ -541,9 +629,7 @@ export default function (pi: ExtensionAPI) {
541
629
  const preparedClipboardImages = clipboardPaths.map(
542
630
  async (path): Promise<PreparedClipboardImage | null> => {
543
631
  try {
544
- const read = readImageBuffer(path);
545
- if (!read) return null;
546
- const image = await resolvePrewarmImage(read.buf, read.mimeType, resizeImage);
632
+ const image = await preparePastedPathImage(path);
547
633
  if (!image) return null;
548
634
  const description = loader.loadDescription(image);
549
635
  description.catch(() => {});
@@ -605,6 +691,7 @@ export default function (pi: ExtensionAPI) {
605
691
  const content = event.content;
606
692
  if (!Array.isArray(content)) return;
607
693
 
694
+ await ensureRouterCapabilitiesForModel(ctx, ctx.model);
608
695
  // Collect image blocks in this read result. (The read tool emits image
609
696
  // blocks even for non-vision models — they reach here untouched.)
610
697
  if (!isHandoffTarget(ctx.model, config)) return;
@@ -731,19 +818,67 @@ export default function (pi: ExtensionAPI) {
731
818
  if (!Array.isArray(messages)) return;
732
819
 
733
820
  const byHash = new Map<string, ExtractedImage>();
734
- let anyImage = false;
821
+ const msgImageHashes = new Map<Record<string, unknown>, Set<string>>();
822
+ const userPastedPaths: Array<{ msg: Record<string, unknown>; path: string }> = [];
823
+
735
824
  for (const msg of messages) {
736
825
  const content = msg.content;
737
- if (!Array.isArray(content)) continue;
738
- for (const block of content) {
739
- const img = extractImageFromBlock(block);
740
- if (!img) continue;
741
- anyImage = true;
742
- byHash.set(imageHash(img.mimeType, img.data), img);
826
+ const hashes = new Set<string>();
827
+ if (Array.isArray(content)) {
828
+ for (const block of content) {
829
+ const img = extractImageFromBlock(block);
830
+ if (!img) continue;
831
+ const h = imageHash(img.mimeType, img.data);
832
+ hashes.add(h);
833
+ byHash.set(h, img);
834
+ }
835
+ }
836
+ if (hashes.size > 0) msgImageHashes.set(msg, hashes);
837
+
838
+ // When asyncClipboardHandoff is off (default), forward pasted temp-dir
839
+ // clipboard image paths directly in the user message on Turn 1 so the
840
+ // active text-only model receives descriptions without needing to call read.
841
+ if (!config.asyncClipboardHandoff && msg.role === "user") {
842
+ const text =
843
+ typeof content === "string"
844
+ ? content
845
+ : Array.isArray(content)
846
+ ? content
847
+ .filter((b): b is { type: string; text: string } => !!b && typeof b === "object" && (b as { type?: unknown }).type === "text" && typeof (b as { text?: unknown }).text === "string")
848
+ .map((b) => b.text)
849
+ .join("\n")
850
+ : "";
851
+ if (text) {
852
+ for (const path of findPastedImagePaths(text)) {
853
+ userPastedPaths.push({ msg, path });
854
+ }
855
+ }
743
856
  }
744
857
  }
745
- if (!anyImage) return;
858
+
859
+ if (byHash.size === 0 && userPastedPaths.length === 0) return;
860
+ await ensureRouterCapabilitiesForModel(ctx, ctx.model);
746
861
  if (!isHandoffTarget(ctx.model, config)) return;
862
+
863
+ const pastedInjections: Array<{ msg: Record<string, unknown>; path: string; hash: string }> = [];
864
+ if (userPastedPaths.length > 0) {
865
+ const resolved = await Promise.all(
866
+ userPastedPaths.map(async ({ msg, path }) => ({
867
+ msg,
868
+ path,
869
+ img: await preparePastedPathImage(path),
870
+ })),
871
+ );
872
+ for (const item of resolved) {
873
+ if (!item.img) continue;
874
+ const h = imageHash(item.img.mimeType, item.img.data);
875
+ if (msgImageHashes.get(item.msg)?.has(h)) continue;
876
+ byHash.set(h, item.img);
877
+ pastedInjections.push({ msg: item.msg, path: item.path, hash: h });
878
+ }
879
+ }
880
+
881
+ if (byHash.size === 0) return;
747
882
  loader.bindTurnContext(ctx);
748
883
  if (!isAnyVisionModelResolvable(ctx.modelRegistry, config)) {
749
884
  notifyUnresolvedVisionModel(ctx, config.visionModel!);
@@ -783,6 +918,26 @@ export default function (pi: ExtensionAPI) {
783
918
  changed = true;
784
919
  }
785
920
  }
921
+
922
+ for (const item of pastedInjections) {
923
+ const desc = descs.get(item.hash) ?? UNAVAILABLE;
924
+ const rawContent = item.msg.content;
925
+ const blocks: Array<{ type: string; text?: string }> =
926
+ typeof rawContent === "string"
927
+ ? [{ type: "text", text: rawContent }]
928
+ : Array.isArray(rawContent)
929
+ ? (rawContent as Array<{ type: string; text?: string }>).slice()
930
+ : [];
931
+ const blockText = `${item.path}\n${desc}`;
932
+ const alreadyIncluded = blocks.some(
933
+ (b) => b.type === "text" && typeof b.text === "string" && b.text.includes(blockText),
934
+ );
935
+ if (alreadyIncluded) continue;
936
+ blocks.push({ type: "text", text: blockText });
937
+ item.msg.content = blocks;
938
+ changed = true;
939
+ }
940
+
786
941
  if (changed) return { messages: event.messages };
787
942
  });
788
943
 
@@ -843,8 +998,9 @@ export default function (pi: ExtensionAPI) {
843
998
  );
844
999
  });
845
1000
 
846
- pi.on("model_select", (event, ctx) => {
1001
+ pi.on("model_select", async (event, ctx) => {
847
1002
  currentModel = event.model;
1003
+ await ensureRouterCapabilitiesForModel(ctx, event.model);
848
1004
  persistFalseVisionOverride(ctx, event.model);
849
1005
  if (!ctx.hasUI) return;
850
1006
  if (!isConfigured(config)) return;
@@ -919,7 +1075,7 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
919
1075
  }
920
1076
 
921
1077
  if (subcommand === "status") {
922
- showStatus(ctx);
1078
+ await showStatus(ctx);
923
1079
  return;
924
1080
  }
925
1081
 
@@ -993,10 +1149,12 @@ async function handleHandoffCommand(ctx: ExtensionCommandContext, args: string):
993
1149
  return;
994
1150
  }
995
1151
  const ref = formatModelRef(parsed.provider, parsed.id);
1152
+ await ensureRouterCapabilitiesForModel(ctx, model);
996
1153
  updateConfig(ctx, (c) => ({ ...c, visionModel: ref }), `Vision model set to ${ref}.`);
997
- if (!isVisionModel(model)) {
1154
+ const cap = routerCache.get(model.provider, model.id);
1155
+ if (!isEffectiveVisionModel(model, cap)) {
998
1156
  ctx.ui.notify(
999
- `Note: ${ref} does not declare image input — it may not describe images well.`,
1157
+ `Note: ${ref} appears text-only — it may not describe images well.`,
1000
1158
  "warning",
1001
1159
  );
1002
1160
  }
@@ -1175,18 +1333,37 @@ function handleTimeoutSubcommand(ctx: ExtensionCommandContext, rest: string): vo
1175
1333
  /** Connected, vision-capable models — the same set the pickers show. Text-only
1176
1334
  * models can't describe images, so they would only produce
1177
1335
  * "[Image: description unavailable]" errors. getAvailable() excludes the whole
1178
- * catalogue of unauthenticated models. */
1179
- function availableVisionModels(ctx: ExtensionCommandContext): Array<{
1180
- provider: string;
1181
- id: string;
1182
- name: string;
1183
- input?: ("text" | "image")[];
1184
- reasoning?: boolean;
1185
- }> {
1186
- return ctx.modelRegistry
1187
- .getAvailable()
1188
- .map((m) => ({ provider: m.provider, id: m.id, name: m.name, input: m.input, reasoning: m.reasoning }))
1189
- .filter((m) => isVisionModel(m));
1336
+ * catalogue of unauthenticated models. Router `/models` metadata and
1337
+ * false-vision heuristics filter out text-only models behind proxies like
1338
+ * Magpie, 9router, and omnirouter. */
1339
+ async function availableVisionModels(ctx: ExtensionCommandContext): Promise<
1340
+ Array<{
1341
+ provider: string;
1342
+ id: string;
1343
+ name: string;
1344
+ input?: ("text" | "image")[];
1345
+ reasoning?: boolean;
1346
+ }>
1347
+ > {
1348
+ await routerCache.refreshProviders(ctx.modelRegistry);
1349
+ const explicitTextOverrides = readExplicitTextOnlyOverrides();
1350
+ const out: Array<{
1351
+ provider: string;
1352
+ id: string;
1353
+ name: string;
1354
+ input?: ("text" | "image")[];
1355
+ reasoning?: boolean;
1356
+ }> = [];
1357
+
1358
+ for (const m of ctx.modelRegistry.getAvailable()) {
1359
+ const ref = formatModelRef(m.provider, m.id);
1360
+ const cap = routerCache.get(m.provider, m.id);
1361
+ const hasExplicitTextOverride = explicitTextOverrides.has(ref);
1362
+ if (!isEffectiveVisionModel(m, cap, { hasExplicitTextOverride })) continue;
1363
+ const input = isVisionModel(m) ? m.input : (["text", "image"] as ("text" | "image")[]);
1364
+ out.push({ provider: m.provider, id: m.id, name: m.name, input, reasoning: m.reasoning });
1365
+ }
1366
+ return out;
1190
1367
  }
1191
1368
 
1192
1369
  async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
@@ -1195,7 +1372,7 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
1195
1372
  return;
1196
1373
  }
1197
1374
 
1198
- const availableModels = availableVisionModels(ctx);
1375
+ const availableModels = await availableVisionModels(ctx);
1199
1376
 
1200
1377
  if (ctx.mode !== "tui") {
1201
1378
  const modelItems = ["None", ...availableModels.map((m) => `${m.provider}/${m.id}`)];
@@ -1279,7 +1456,8 @@ async function showSelector(ctx: ExtensionCommandContext): Promise<void> {
1279
1456
  }
1280
1457
  }
1281
1458
 
1282
- function showStatus(ctx: ExtensionCommandContext): void {
1459
+ async function showStatus(ctx: ExtensionCommandContext): Promise<void> {
1460
+ await ensureRouterCapabilitiesForModel(ctx, ctx.model);
1283
1461
  const lines: string[] = [];
1284
1462
  lines.push(`Vision handoff: ${config.enabled ? "enabled" : "disabled"}`);
1285
1463
  lines.push(`Vision model: ${config.visionModel ?? "(none — pick one with /vision-watcher)"}`);