@bismawy/pi-vision-watcher 1.0.7 → 1.0.8

package/README.md

@@ -66,6 +66,7 @@ Then run `/reload` in Pi (or restart Pi).
 | `/vision-watcher add <provider/id>` | Force handoff for a specific model (e.g. weak vision models) |
 | `/vision-watcher remove <provider/id>` | Remove model from forced handoff list |
 | `/vision-watcher thinking <level>` | Set describer thinking level (`off`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`) |
+| `/vision-watcher timeout <ms>` | Set the base per-image description timeout in ms (default 45000) |
 | `/vision-watcher prewarm on` / `off` | Enable paste-time prewarming in TUI editor |
 | `/vision-watcher fallback on` / `off` | Enable async pasted-path description injection |
 | `/vision-watcher clear` | Clear configured vision model |
@@ -86,6 +87,7 @@ Configuration is stored at `~/.pi/agent/extensions/pi-vision-watcher.json`:
   "handoffModels": [],
   "thinking": false,
   "thinkingLevel": "medium",
+  "describeTimeoutMs": 45000,
   "prewarmPastedImages": false,
   "asyncClipboardHandoff": false,
   "maxTokens": null,
@@ -102,6 +104,7 @@ Configuration is stored at `~/.pi/agent/extensions/pi-vision-watcher.json`:
 | `autoHandoff` | `true` | Automatically apply handoff to all models lacking native vision. |
 | `handoffModels` | `[]` | Additional models forced to receive handoff even if vision-capable. |
 | `thinking` / `thinkingLevel` | `false` / `"medium"` | Reasoning effort for vision models that support thinking. |
+| `describeTimeoutMs` | `45000` | Base per-image timeout in ms before failing over to fallback models. |
 | `prewarmPastedImages` | `false` | Describe images immediately upon pasting into the prompt. |
 | `asyncClipboardHandoff` | `false` | Asynchronous injection fallback for pasted image paths. |
 | `maxTokens` | `null` | Output token cap for descriptions (`null` = model default). |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bismawy/pi-vision-watcher",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
   "type": "module",
   "author": "bismawy",

package/src/image.ts

@@ -20,7 +20,7 @@
 
 import { readFileSync, statSync } from "node:fs";
 import { tmpdir } from "node:os";
-import { isAbsolute, join, sep } from "node:path";
+import { isAbsolute, join, normalize, sep } from "node:path";
 import crypto from "node:crypto";
 import type { ExtractedImage } from "./index.js";
 
@@ -299,14 +299,14 @@ const LEADING_WRAP_RE = new RegExp('^[^A-Za-z0-9_/@.~-]+');
  *  dir and mistaken for local files; leading wrapping chars (parentheses,
  *  quotes) are stripped so quoted/parenthesized paths still resolve. */
 export function findPastedImagePaths(prompt: string): string[] {
-  const tmp = tmpdir();
+  const tmp = normalize(tmpdir());
   const paths = new Set<string>();
   for (const m of prompt.matchAll(PASTED_IMAGE_PATH_RE)) {
     const token = m[1];
     if (!token) continue;
     if (URL_RE.test(token)) continue;
     const p = token.replace(LEADING_WRAP_RE, "");
-    const abs = isAbsolute(p) ? p : join(tmp, p);
+    const abs = normalize(isAbsolute(p) ? p : join(tmp, p));
     // Ensure the resolved candidate stays inside the temp directory.
     if (abs.startsWith(tmp + sep) || abs === tmp) paths.add(abs);
   }

package/src/index.ts

@@ -235,6 +235,84 @@ export function formatModelRef(provider: string, id: string): string {
   return `${provider}/${id}`;
 }
 
+/** Match provider errors meaning "this model can't take image input" — e.g.
+ *  AgentRouter's 400 `"This model does not support image"`, OpenRouter's
+ *  "does not support image inputs". Used by the message_end auto-recovery:
+ *  a model whose registry entry FALSELY declares `input: ["text","image"]`
+ *  passes the autoHandoff vision check, so the raw image reaches the provider
+ *  and 400s — we learn from that error and force handoff for the model. */
+export function isImageNotSupportedError(text: string): boolean {
+  return /not support (?:the )?image|image[s]? (?:input[s]? )?(?:is |are )?not supported|does not accept image|unsupported (?:image|multimodal)/i.test(
+    text,
+  );
+}
+
+/** Models whose registry entries commonly declare image input while the
+ *  backend rejects images (DeepSeek V4 via aggregator proxies). Conservative:
+ *  VL / Vision / Janus variants keep image input. Used so autoHandoff covers
+ *  them on the FIRST send — waiting for a 400 is too late. */
+export function isKnownTextOnlyFalselyVision(id: string | undefined | null): boolean {
+  if (!id) return false;
+  const n = id.toLowerCase();
+  if (n.includes("vl") || n.includes("vision") || n.includes("janus")) return false;
+  return /deepseek[-_./]*v4/.test(n);
+}
+
+/** Extract the provider error string from a finalized assistant message.
+ *  Pi stores it on `errorMessage`; some providers also put the 400 body in a
+ *  text content block. */
+export function assistantErrorText(msg: {
+  errorMessage?: unknown;
+  content?: unknown;
+} | null | undefined): string {
+  if (!msg) return "";
+  if (typeof msg.errorMessage === "string" && msg.errorMessage) return msg.errorMessage;
+  if (typeof msg.content === "string") return msg.content;
+  if (!Array.isArray(msg.content)) return "";
+  return msg.content
+    .filter((b): b is { type: string; text: string } => !!b && typeof b === "object" && typeof (b as { text?: unknown }).text === "string")
+    .map((b) => b.text)
+    .join("\n");
+}
+
+/** Set (or add) a `modelOverrides` entry forcing a model's input to
+ *  `["text"]` in a PARSED models.json config object. Pure: returns a new
+ *  object, never mutates the input. Preserves every other field (credentials,
+ * compat, sibling models) — only the one model's `input` override is touched.
+ *  Used by the message_end auto-recovery to fix models whose registry entry
+ *  falsely declares image input the backend rejects with a 400. */
+export function setTextOnlyInputOverride(
+  cfg: unknown,
+  providerId: string,
+  modelId: string,
+): Record<string, unknown> {
+  const root =
+    cfg && typeof cfg === "object" && !Array.isArray(cfg)
+      ? { ...(cfg as Record<string, unknown>) }
+      : {};
+  const providers =
+    root.providers && typeof root.providers === "object" && !Array.isArray(root.providers)
+      ? { ...(root.providers as Record<string, unknown>) }
+      : {};
+  const provider =
+    providers[providerId] && typeof providers[providerId] === "object" && !Array.isArray(providers[providerId])
+      ? { ...(providers[providerId] as Record<string, unknown>) }
+      : {};
+  const overrides =
+    provider.modelOverrides && typeof provider.modelOverrides === "object" && !Array.isArray(provider.modelOverrides)
+      ? { ...(provider.modelOverrides as Record<string, unknown>) }
+      : {};
+  const existing =
+    overrides[modelId] && typeof overrides[modelId] === "object" && !Array.isArray(overrides[modelId])
+      ? { ...(overrides[modelId] as Record<string, unknown>) }
+      : {};
+  overrides[modelId] = { ...existing, input: ["text"] };
+  provider.modelOverrides = overrides;
+  providers[providerId] = provider;
+  root.providers = providers;
+  return root;
+}
+
 /** Whether a model declares image input. */
 export function isVisionModel(model: { input?: ("text" | "image")[] } | undefined | null): boolean {
   return !!model && Array.isArray(model.input) && model.input.includes("image");

package/vision-watcher.ts

@@ -31,17 +31,24 @@
  */
 
 import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, ModelRegistry } from "@earendil-works/pi-coding-agent";
+import { getAgentDir, resizeImage } from "@earendil-works/pi-coding-agent";
 import type { Api, ImageContent, Model, TextContent } from "@earendil-works/pi-ai";
 import { isAbsolute, resolve } from "node:path";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
 import {
   DESCRIBE_TIMEOUT_MS,
   extractImageFromBlock,
   formatModelRef,
   isThinkingLevel,
+  assistantErrorText,
+  isImageNotSupportedError,
+  isKnownTextOnlyFalselyVision,
   isVisionModel,
   NON_VISION_IMAGE_NOTE,
   parseModelRef,
   readConfig,
+  setTextOnlyInputOverride,
   stripNonVisionImageNote,
   writeConfig,
   HANDOFF_COMMAND_DESCRIPTION,
@@ -56,7 +63,6 @@ import {
 import { DescriptionLoader, UNAVAILABLE, type LoaderDeps } from "./src/dataloader.js";
 import { imageHash, findPastedImagePaths, readImageBuffer, readImageBufferBounded, resolvePrewarmImage, isOmittedImageNote } from "./src/image.js";
 import { appendVisionError } from "./src/error-log.js";
-import { resizeImage } from "@earendil-works/pi-coding-agent";
 import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
 import { PrewarmEditor } from "./src/prewarm-editor.js";
 import { Text } from "@earendil-works/pi-tui";
@@ -238,6 +244,10 @@ function isHandoffTarget(
   if (!model || !model.provider || !model.id) return false;
   const ref = formatModelRef(model.provider, model.id);
   if (cfg.handoffModels.includes(ref)) return true;
+  // Aggregators often mark DeepSeek V4 as vision; the backend 400s on images.
+  // Treat as text-only so the FIRST send is handed off — waiting for the 400
+  // is too late. Persist the models.json override separately so /model agrees.
+  if (cfg.autoHandoff && isKnownTextOnlyFalselyVision(model.id)) return true;
   if (cfg.autoHandoff && !isVisionModel(model)) return true;
   return false;
 }
@@ -303,6 +313,46 @@ function installPrewarmEditor(ctx: ExtensionContext): void {
   );
 }
 
+/** Write a `modelOverrides.<modelId>.input = ["text"]` fix to models.json and
+ *  refresh the registry in-process so it applies without /reload (the same
+ *  write+refresh pattern pi-auto-compat uses). Never touches credentials or
+ *  other fields — the merge is done by the pure {@link setTextOnlyInputOverride}.
+ *  Returns true when the override was written; false when models.json is
+ *  unreadable/corrupt or the write fails (callers fall back to handoffModels). */
+function fixModelInputTextOnly(ctx: ExtensionContext, providerId: string, modelId: string): boolean {
+  const path = join(getAgentDir(), "models.json");
+  let cfg: unknown;
+  try {
+    cfg = existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : {};
+  } catch {
+    return false;
+  }
+  try {
+    writeFileSync(path, JSON.stringify(setTextOnlyInputOverride(cfg, providerId, modelId), null, 2) + "\n", "utf8");
+  } catch {
+    return false;
+  }
+  // Refresh the merged registry in-process so the corrected input takes
+  // effect this session. Fire-and-forget: a refresh failure only delays the
+  // fix to the next session start (models.json is already corrected on disk).
+  void ctx.modelRegistry
+    ?.refresh?.({ allowNetwork: false })
+    ?.catch?.(() => {});
+  return true;
+}
+
+/** Persist `input: ["text"]` for a model that falsely declares image input.
+ *  No-op if the model is already text-only or isn't a known false-vision id. */
+function persistFalseVisionOverride(
+  ctx: ExtensionContext,
+  model: { provider?: string; id?: string; input?: ("text" | "image")[] } | undefined | null,
+): boolean {
+  if (!model?.provider || !model.id) return false;
+  if (!isKnownTextOnlyFalselyVision(model.id)) return false;
+  if (!isVisionModel(model)) return false;
+  return fixModelInputTextOnly(ctx, model.provider, model.id);
+}
+
 function notifyUnresolvedVisionModel(ctx: ExtensionContext, ref: string): void {
   if (visionModelUnresolvedRef === ref) return;
   visionModelUnresolvedRef = ref;
@@ -401,6 +451,7 @@ export default function (pi: ExtensionAPI) {
     warnedHashes.clear();
     loader.reset();
     currentModel = ctx.model;
+    persistFalseVisionOverride(ctx, ctx.model);
     installPrewarmEditor(ctx);
   });
 
@@ -719,13 +770,69 @@ export default function (pi: ExtensionAPI) {
     if (changed) return { messages: event.messages };
   });
 
+  // Auto-recovery from "model does not support image" 400s: a model whose
+  // registry entry falsely declares `input: ["text","image"]` (common on
+  // aggregator providers) passes the autoHandoff vision check, so handoff is
+  // skipped and the raw image block reaches the provider — which rejects it.
+  // PRIMARY fix: correct the metadata at its layer — write a `modelOverrides`
+  // entry forcing `input: ["text"]` to models.json and refresh the registry
+  // in-process (modelOverrides is Pi's topmost config layer, so it wins even
+  // over a regenerated models[] entry). The model then registers as text-only,
+  // autoHandoff covers it naturally, and /model shows it correctly. FALLBACK
+  // (write failed): force handoff via handoffModels. Either way, the failed
+  // turn's images are still in history; the context hook describes them on the
+  // retry instead of failing again.
+  pi.on("message_end", (event, ctx) => {
+    const msg = event.message as {
+      role?: string;
+      type?: string;
+      stopReason?: string;
+      errorMessage?: unknown;
+      content?: unknown;
+    };
+    // Pi assistant messages use `role`, not `type`. Checking `type` made this
+    // handler a no-op, so the 400 never taught us anything.
+    if (msg?.role !== "assistant" && msg?.type !== "assistant") return;
+    if (msg.stopReason !== "error") return;
+    if (!isImageNotSupportedError(assistantErrorText(msg))) return;
+    if (!isConfigured(config)) return;
+    const model = ctx.model;
+    if (!model?.provider || !model.id) return;
+    const ref = formatModelRef(model.provider, model.id);
+
+    const fixed = persistFalseVisionOverride(ctx, model) || fixModelInputTextOnly(ctx, model.provider, model.id);
+    if (fixed) {
+      if (!config.handoffModels.includes(ref)) {
+        config = { ...config, handoffModels: [...config.handoffModels, ref] };
+        writeConfig(config);
+      }
+      if (ctx.hasUI) {
+        ctx.ui.notify(
+          `pi-vision-watcher: ${ref} rejected image input — marked as text-only. Resend; images will be described by ${config.visionModel}.`,
+          "warning",
+        );
+      }
+      return;
+    }
+
+    if (config.handoffModels.includes(ref)) return;
+    config = { ...config, handoffModels: [...config.handoffModels, ref] };
+    writeConfig(config);
+    if (!ctx.hasUI) return;
+    ctx.ui.notify(
+      `pi-vision-watcher: ${ref} rejected image input — handoff forced for this model. Resend; images will be described by ${config.visionModel}.`,
+      "warning",
+    );
+  });
+
   pi.on("model_select", (event, ctx) => {
     currentModel = event.model;
+    persistFalseVisionOverride(ctx, event.model);
     if (!ctx.hasUI) return;
     if (!isConfigured(config)) return;
     const model = event.model;
     if (!model) return;
-    if (isHandoffTarget(model, config) && !isVisionModel(model)) {
+    if (isHandoffTarget(model, config)) {
       ctx.ui.notify(
         `pi-vision-watcher: active — images will be described by ${config.visionModel}`,
         "info",