newmark-agent 0.5.3 → 0.5.4

package/dist/conversation-utility-host.bundle.cjs

@@ -336933,6 +336933,10 @@ var ToolExecutor = class {
   async webSearch(query) {
     return this.wsearch(query);
   }
+  /** OCR entry point for the runtime's final visual fallback. */
+  async finalVisualFallbackOcr(dataUrl, signal) {
+    return await this.localOcr.recognizeDataUrl(dataUrl, signal, "sparse-ui");
+  }
   setHostProfile(profile) {
     this.hostProfile = { ...profile };
   }
@@ -340846,7 +340850,12 @@ async function runAgentKernel(agent) {
   try {
     const linkedPlanRevisionBeforeRun = agent.getLinkedPlan().revision;
     const modelBeforeKernelRun = agent.model;
-    let lastTurn = await runWithCompressionResume([], false);
+    const preflightVisualFallback = !agent.activeModelConfig()?.vision ? await agent.finalVisualFallback("vision input not supported by the selected model", processSignal) : null;
+    let lastTurn = preflightVisualFallback ? { text: preflightVisualFallback, stopReason: "stop", errorMessage: "" } : await runWithCompressionResume([], false);
+    if (preflightVisualFallback) {
+      tokens.push({ type: "text", text: preflightVisualFallback });
+      agent.recordWorkStatus("Final visual fallback used: local mini OCR plus conservative text correction.");
+    }
     if (modelBeforeKernelRun && modelBeforeKernelRun !== agent.model && !tokens.some((t3) => t3.text?.includes("[Model fallback]"))) {
       tokens.unshift({ type: "text", text: `[Model fallback] ${modelBeforeKernelRun} unavailable; switched to ${agent.model}.` });
     }
@@ -340876,6 +340885,14 @@ async function runAgentKernel(agent) {
       await agent.waitForPlannedRouteRetry();
       lastTurn = await runWithCompressionResume([], false);
     }
+    if (kernelTurnFailed(agent, lastTurn)) {
+      const visualFallback = await agent.finalVisualFallback(lastTurn.errorMessage || lastTurn.text, processSignal);
+      if (visualFallback) {
+        tokens.push({ type: "text", text: visualFallback });
+        agent.recordWorkStatus("Final visual fallback used: local mini OCR plus conservative text correction.");
+        lastTurn = { ...lastTurn, text: visualFallback, errorMessage: "", stopReason: "stop" };
+      }
+    }
     if (kernelTurnFailed(agent, lastTurn)) {
       throw new ProviderRunError(normalizePublicProviderError(lastTurn.errorMessage || lastTurn.text, [agent.activeModelConfig()?.api_key]));
     }
@@ -350820,6 +350837,60 @@ ${msg.content}
     };
     return results;
   }
+  /**
+   * Final visual safety net: OCR each submitted image and ask a text-only
+   * request to conservatively repair the OCR. This is intentionally callable
+   * only after a visual-input refusal and after same-provider vision routing
+   * has been exhausted; the original image is never sent again.
+   */
+  async finalVisualFallback(errorText, signal) {
+    if (!/(?:vision|image|multimodal|image_url|input_image).*(?:not supported|unsupported|拒绝|不支持|failed|failure|invalid)|(?:not supported|unsupported|拒绝|不支持).*(?:vision|image|multimodal|image_url|input_image)/i.test(String(errorText || ""))) return null;
+    const current = this.activeModelConfig();
+    if (!current) return null;
+    const alternateVision = this.config.allModels().some(
+      (model) => model.enabled !== false && model.provider_id === current.provider_id && model.name !== current.name && !!model.vision && !!model.api_key && !!model.provider_url && !["unavailable", "auth_error", "invalid_config"].includes(String(model.evaluation?.status || model.validation?.status || "").toLowerCase())
+    );
+    if (alternateVision) return null;
+    const latest = [...this.history].reverse().find((item) => item?.role === "user");
+    const parts = latest?.content && Array.isArray(latest.content) ? latest.content : [];
+    const images = parts.map((part) => {
+      const image = part.image_url;
+      return image && typeof image === "object" ? String(image.url || "") : "";
+    }).filter((value) => /^data:image\/(?:png|jpeg);base64,/i.test(value)).slice(0, 4);
+    if (!images.length) return null;
+    const ocr = [];
+    for (const [index, image] of images.entries()) {
+      try {
+        const result = await this.tools.finalVisualFallbackOcr(image, signal);
+        if (result.ok && result.text.trim()) ocr.push({ index: index + 1, text: result.text.slice(0, 5e4), confidence: result.confidence });
+      } catch {
+      }
+    }
+    if (!ocr.length) return JSON.stringify({ ok: false, fallback: "mini_ocr_llm", error: "Local OCR returned no readable text; no visual content was fabricated." });
+    const task = typeof latest?.content === "string" ? latest.content : "";
+    const evidence = ocr.map((item) => `Image ${item.index} (OCR confidence ${item.confidence.toFixed(1)}):
+${item.text}`).join("\n\n");
+    const prompt = `The provider rejected image input. Answer the user's task using only this approximate OCR evidence. Correct obvious character, spacing, and line-break errors only when supported by context. Preserve [uncertain] markers for ambiguity and never invent missing visual content.
+User task:
+${task.slice(0, 12e3)}
+OCR evidence:
+${evidence}`;
+    let corrected = "";
+    try {
+      const provider = this.engineModel();
+      if (provider) corrected = String(await provider.chat(this.activeModelName(), [{ role: "user", content: prompt }], "You are a text-only OCR correction assistant. Be conservative and explicit about uncertainty.", 0.05, 3e3, signal) || "").trim();
+    } catch {
+    }
+    return JSON.stringify({
+      ok: !!(corrected || ocr.length),
+      fallback: "mini_ocr_llm",
+      approximate: true,
+      warning: "\u89C6\u89C9\u8F93\u5165\u88AB\u62D2\u7EDD\uFF1B\u4EE5\u4E0B\u5185\u5BB9\u6765\u81EA\u672C\u5730 OCR\uFF0C\u5E76\u7ECF\u6587\u672C\u6A21\u578B\u4FDD\u5B88\u6821\u6B63\uFF0C\u53EF\u80FD\u4E0D\u5B8C\u6574\u3002",
+      raw_ocr: ocr,
+      corrected: corrected || ocr.map((item) => item.text).join("\n\n"),
+      uncertainty: corrected ? "preserved" : "raw_ocr_only"
+    }, null, 2);
+  }
   engineModel() {
     if (this.forcedProvider) {
       const active = this.activeDeployment();
@@ -351202,8 +351273,10 @@ ${persisted.map((guide, index) => `Guide ${index + 1}: ${guide.text}`).join("\n"
       }
       const selectedModel = this.activeModelConfig();
       if (images.length && !selectedModel?.vision) {
-        this.status = "idle";
-        return [{ type: "text", text: `[Vision unavailable] ${this.activeModelName() || this.model} has not passed image-input validation. Select a validated vision model before asking about attachments.` }];
+        const hasSameProviderVision = selectedModel && this.config.allModels().some(
+          (model) => model.enabled !== false && model.provider_id === selectedModel.provider_id && model.name !== selectedModel.name && !!model.vision
+        );
+        if (hasSameProviderVision) this.switchToFallbackModel("vision input not supported by the selected model");
       }
       const now2 = this.nowLabel();
       const visibleUserInput = inputEnvelope?.visibleUserInput === void 0 ? text : String(inputEnvelope.visibleUserInput || "");

package/dist/core/agent.d.ts

@@ -990,6 +990,13 @@ export declare class Agent {
     isModelValidationRunning(): boolean;
     modelValidationStatus(): ModelValidationProgress;
     private runModelValidation;
+    /**
+     * Final visual safety net: OCR each submitted image and ask a text-only
+     * request to conservatively repair the OCR. This is intentionally callable
+     * only after a visual-input refusal and after same-provider vision routing
+     * has been exhausted; the original image is never sent again.
+     */
+    finalVisualFallback(errorText: string, signal?: AbortSignal): Promise<string | null>;
     engineModel(): LLMProvider | null;
     /**
      * dev-0.4.3 模型原生思考强度档位映射表（模型名 → thinking_tier_map）。

package/dist/core/agent.js

@@ -6913,6 +6913,62 @@ class Agent {
         };
         return results;
     }
+    /**
+     * Final visual safety net: OCR each submitted image and ask a text-only
+     * request to conservatively repair the OCR. This is intentionally callable
+     * only after a visual-input refusal and after same-provider vision routing
+     * has been exhausted; the original image is never sent again.
+     */
+    async finalVisualFallback(errorText, signal) {
+        if (!/(?:vision|image|multimodal|image_url|input_image).*(?:not supported|unsupported|拒绝|不支持|failed|failure|invalid)|(?:not supported|unsupported|拒绝|不支持).*(?:vision|image|multimodal|image_url|input_image)/i.test(String(errorText || '')))
+            return null;
+        const current = this.activeModelConfig();
+        if (!current)
+            return null;
+        const alternateVision = this.config.allModels().some(model => model.enabled !== false && model.provider_id === current.provider_id &&
+            model.name !== current.name && !!model.vision && !!model.api_key && !!model.provider_url &&
+            !['unavailable', 'auth_error', 'invalid_config'].includes(String(model.evaluation?.status || model.validation?.status || '').toLowerCase()));
+        if (alternateVision)
+            return null;
+        const latest = [...this.history].reverse().find(item => item?.role === 'user');
+        const parts = latest?.content && Array.isArray(latest.content) ? latest.content : [];
+        const images = parts.map(part => {
+            const image = part.image_url;
+            return image && typeof image === 'object' ? String(image.url || '') : '';
+        }).filter(value => /^data:image\/(?:png|jpeg);base64,/i.test(value)).slice(0, 4);
+        if (!images.length)
+            return null;
+        const ocr = [];
+        for (const [index, image] of images.entries()) {
+            try {
+                const result = await this.tools.finalVisualFallbackOcr(image, signal);
+                if (result.ok && result.text.trim())
+                    ocr.push({ index: index + 1, text: result.text.slice(0, 50_000), confidence: result.confidence });
+            }
+            catch { }
+        }
+        if (!ocr.length)
+            return JSON.stringify({ ok: false, fallback: 'mini_ocr_llm', error: 'Local OCR returned no readable text; no visual content was fabricated.' });
+        const task = typeof latest?.content === 'string' ? latest.content : '';
+        const evidence = ocr.map(item => `Image ${item.index} (OCR confidence ${item.confidence.toFixed(1)}):\n${item.text}`).join('\n\n');
+        const prompt = `The provider rejected image input. Answer the user's task using only this approximate OCR evidence. Correct obvious character, spacing, and line-break errors only when supported by context. Preserve [uncertain] markers for ambiguity and never invent missing visual content.\nUser task:\n${task.slice(0, 12_000)}\nOCR evidence:\n${evidence}`;
+        let corrected = '';
+        try {
+            const provider = this.engineModel();
+            if (provider)
+                corrected = String(await provider.chat(this.activeModelName(), [{ role: 'user', content: prompt }], 'You are a text-only OCR correction assistant. Be conservative and explicit about uncertainty.', 0.05, 3000, signal) || '').trim();
+        }
+        catch { }
+        return JSON.stringify({
+            ok: !!(corrected || ocr.length),
+            fallback: 'mini_ocr_llm',
+            approximate: true,
+            warning: '视觉输入被拒绝；以下内容来自本地 OCR，并经文本模型保守校正，可能不完整。',
+            raw_ocr: ocr,
+            corrected: corrected || ocr.map(item => item.text).join('\n\n'),
+            uncertainty: corrected ? 'preserved' : 'raw_ocr_only',
+        }, null, 2);
+    }
     engineModel() {
         if (this.forcedProvider) {
             const active = this.activeDeployment();
@@ -7353,8 +7409,13 @@ class Agent {
             }
             const selectedModel = this.activeModelConfig();
             if (images.length && !selectedModel?.vision) {
-                this.status = 'idle';
-                return [{ type: 'text', text: `[Vision unavailable] ${this.activeModelName() || this.model} has not passed image-input validation. Select a validated vision model before asking about attachments.` }];
+                // Give the normal route planner first chance to select another
+                // same-provider vision deployment. If none is available, the kernel
+                // preflight invokes the final mini-OCR + text-only correction path.
+                const hasSameProviderVision = selectedModel && this.config.allModels().some(model => model.enabled !== false && model.provider_id === selectedModel.provider_id &&
+                    model.name !== selectedModel.name && !!model.vision);
+                if (hasSameProviderVision)
+                    this.switchToFallbackModel('vision input not supported by the selected model');
             }
             const now = this.nowLabel();
             const visibleUserInput = inputEnvelope?.visibleUserInput === undefined

package/dist/core/agentKernelRunner.js

@@ -429,7 +429,16 @@ async function runAgentKernel(agent) {
     try {
         const linkedPlanRevisionBeforeRun = agent.getLinkedPlan().revision;
         const modelBeforeKernelRun = agent.model;
-        let lastTurn = await runWithCompressionResume([], false);
+        const preflightVisualFallback = !agent.activeModelConfig()?.vision
+            ? await agent.finalVisualFallback('vision input not supported by the selected model', processSignal)
+            : null;
+        let lastTurn = preflightVisualFallback
+            ? { text: preflightVisualFallback, stopReason: 'stop', errorMessage: '' }
+            : await runWithCompressionResume([], false);
+        if (preflightVisualFallback) {
+            tokens.push({ type: 'text', text: preflightVisualFallback });
+            agent.recordWorkStatus('Final visual fallback used: local mini OCR plus conservative text correction.');
+        }
         if (modelBeforeKernelRun && modelBeforeKernelRun !== agent.model && !tokens.some(t => t.text?.includes('[Model fallback]'))) {
             tokens.unshift({ type: 'text', text: `[Model fallback] ${modelBeforeKernelRun} unavailable; switched to ${agent.model}.` });
         }
@@ -460,6 +469,14 @@ async function runAgentKernel(agent) {
             await agent.waitForPlannedRouteRetry();
             lastTurn = await runWithCompressionResume([], false);
         }
+        if (kernelTurnFailed(agent, lastTurn)) {
+            const visualFallback = await agent.finalVisualFallback(lastTurn.errorMessage || lastTurn.text, processSignal);
+            if (visualFallback) {
+                tokens.push({ type: 'text', text: visualFallback });
+                agent.recordWorkStatus('Final visual fallback used: local mini OCR plus conservative text correction.');
+                lastTurn = { ...lastTurn, text: visualFallback, errorMessage: '', stopReason: 'stop' };
+            }
+        }
         if (kernelTurnFailed(agent, lastTurn)) {
             throw new ProviderRunError(normalizePublicProviderError(lastTurn.errorMessage || lastTurn.text, [agent.activeModelConfig()?.api_key]));
         }

package/dist/tools/index.d.ts

@@ -2,6 +2,7 @@ import { ConfigManager } from '../core/config';
 import { NewmarkToolDefinition, NewmarkToolResult } from '../core/compat';
 import { SshManager } from '../core/ssh';
 import { WorkspaceManager } from '../core/workspace';
+import { LocalOcrResult } from '../core/localOcr';
 export interface ToolExecutionContext {
     mode?: string;
     workspacePath?: string;
@@ -30,6 +31,8 @@ export declare class ToolExecutor {
     private hostProfile;
     constructor(root: string, config: ConfigManager, ssh?: SshManager | undefined, workspace?: WorkspaceManager | undefined);
     webSearch(query: string): Promise<string>;
+    /** OCR entry point for the runtime's final visual fallback. */
+    finalVisualFallbackOcr(dataUrl: string, signal?: AbortSignal): Promise<LocalOcrResult>;
     setHostProfile(profile: ToolHostProfile): void;
     definitions(mode?: string): unknown[];
     canonicalDefinitions(mode?: string): NewmarkToolDefinition[];

package/dist/tools/index.js

@@ -234,6 +234,10 @@ class ToolExecutor {
     async webSearch(query) {
         return this.wsearch(query);
     }
+    /** OCR entry point for the runtime's final visual fallback. */
+    async finalVisualFallbackOcr(dataUrl, signal) {
+        return await this.localOcr.recognizeDataUrl(dataUrl, signal, 'sparse-ui');
+    }
     setHostProfile(profile) {
         this.hostProfile = { ...profile };
     }

package/dist/wsl-agent-host.bundle.cjs

@@ -336937,6 +336937,10 @@ var ToolExecutor = class {
   async webSearch(query) {
     return this.wsearch(query);
   }
+  /** OCR entry point for the runtime's final visual fallback. */
+  async finalVisualFallbackOcr(dataUrl, signal) {
+    return await this.localOcr.recognizeDataUrl(dataUrl, signal, "sparse-ui");
+  }
   setHostProfile(profile) {
     this.hostProfile = { ...profile };
   }
@@ -340850,7 +340854,12 @@ async function runAgentKernel(agent) {
   try {
     const linkedPlanRevisionBeforeRun = agent.getLinkedPlan().revision;
     const modelBeforeKernelRun = agent.model;
-    let lastTurn = await runWithCompressionResume([], false);
+    const preflightVisualFallback = !agent.activeModelConfig()?.vision ? await agent.finalVisualFallback("vision input not supported by the selected model", processSignal) : null;
+    let lastTurn = preflightVisualFallback ? { text: preflightVisualFallback, stopReason: "stop", errorMessage: "" } : await runWithCompressionResume([], false);
+    if (preflightVisualFallback) {
+      tokens.push({ type: "text", text: preflightVisualFallback });
+      agent.recordWorkStatus("Final visual fallback used: local mini OCR plus conservative text correction.");
+    }
     if (modelBeforeKernelRun && modelBeforeKernelRun !== agent.model && !tokens.some((t3) => t3.text?.includes("[Model fallback]"))) {
       tokens.unshift({ type: "text", text: `[Model fallback] ${modelBeforeKernelRun} unavailable; switched to ${agent.model}.` });
     }
@@ -340880,6 +340889,14 @@ async function runAgentKernel(agent) {
       await agent.waitForPlannedRouteRetry();
       lastTurn = await runWithCompressionResume([], false);
     }
+    if (kernelTurnFailed(agent, lastTurn)) {
+      const visualFallback = await agent.finalVisualFallback(lastTurn.errorMessage || lastTurn.text, processSignal);
+      if (visualFallback) {
+        tokens.push({ type: "text", text: visualFallback });
+        agent.recordWorkStatus("Final visual fallback used: local mini OCR plus conservative text correction.");
+        lastTurn = { ...lastTurn, text: visualFallback, errorMessage: "", stopReason: "stop" };
+      }
+    }
     if (kernelTurnFailed(agent, lastTurn)) {
       throw new ProviderRunError(normalizePublicProviderError(lastTurn.errorMessage || lastTurn.text, [agent.activeModelConfig()?.api_key]));
     }
@@ -350824,6 +350841,60 @@ ${msg.content}
     };
     return results;
   }
+  /**
+   * Final visual safety net: OCR each submitted image and ask a text-only
+   * request to conservatively repair the OCR. This is intentionally callable
+   * only after a visual-input refusal and after same-provider vision routing
+   * has been exhausted; the original image is never sent again.
+   */
+  async finalVisualFallback(errorText, signal) {
+    if (!/(?:vision|image|multimodal|image_url|input_image).*(?:not supported|unsupported|拒绝|不支持|failed|failure|invalid)|(?:not supported|unsupported|拒绝|不支持).*(?:vision|image|multimodal|image_url|input_image)/i.test(String(errorText || ""))) return null;
+    const current = this.activeModelConfig();
+    if (!current) return null;
+    const alternateVision = this.config.allModels().some(
+      (model) => model.enabled !== false && model.provider_id === current.provider_id && model.name !== current.name && !!model.vision && !!model.api_key && !!model.provider_url && !["unavailable", "auth_error", "invalid_config"].includes(String(model.evaluation?.status || model.validation?.status || "").toLowerCase())
+    );
+    if (alternateVision) return null;
+    const latest = [...this.history].reverse().find((item) => item?.role === "user");
+    const parts = latest?.content && Array.isArray(latest.content) ? latest.content : [];
+    const images = parts.map((part) => {
+      const image = part.image_url;
+      return image && typeof image === "object" ? String(image.url || "") : "";
+    }).filter((value) => /^data:image\/(?:png|jpeg);base64,/i.test(value)).slice(0, 4);
+    if (!images.length) return null;
+    const ocr = [];
+    for (const [index, image] of images.entries()) {
+      try {
+        const result = await this.tools.finalVisualFallbackOcr(image, signal);
+        if (result.ok && result.text.trim()) ocr.push({ index: index + 1, text: result.text.slice(0, 5e4), confidence: result.confidence });
+      } catch {
+      }
+    }
+    if (!ocr.length) return JSON.stringify({ ok: false, fallback: "mini_ocr_llm", error: "Local OCR returned no readable text; no visual content was fabricated." });
+    const task = typeof latest?.content === "string" ? latest.content : "";
+    const evidence = ocr.map((item) => `Image ${item.index} (OCR confidence ${item.confidence.toFixed(1)}):
+${item.text}`).join("\n\n");
+    const prompt = `The provider rejected image input. Answer the user's task using only this approximate OCR evidence. Correct obvious character, spacing, and line-break errors only when supported by context. Preserve [uncertain] markers for ambiguity and never invent missing visual content.
+User task:
+${task.slice(0, 12e3)}
+OCR evidence:
+${evidence}`;
+    let corrected = "";
+    try {
+      const provider = this.engineModel();
+      if (provider) corrected = String(await provider.chat(this.activeModelName(), [{ role: "user", content: prompt }], "You are a text-only OCR correction assistant. Be conservative and explicit about uncertainty.", 0.05, 3e3, signal) || "").trim();
+    } catch {
+    }
+    return JSON.stringify({
+      ok: !!(corrected || ocr.length),
+      fallback: "mini_ocr_llm",
+      approximate: true,
+      warning: "\u89C6\u89C9\u8F93\u5165\u88AB\u62D2\u7EDD\uFF1B\u4EE5\u4E0B\u5185\u5BB9\u6765\u81EA\u672C\u5730 OCR\uFF0C\u5E76\u7ECF\u6587\u672C\u6A21\u578B\u4FDD\u5B88\u6821\u6B63\uFF0C\u53EF\u80FD\u4E0D\u5B8C\u6574\u3002",
+      raw_ocr: ocr,
+      corrected: corrected || ocr.map((item) => item.text).join("\n\n"),
+      uncertainty: corrected ? "preserved" : "raw_ocr_only"
+    }, null, 2);
+  }
   engineModel() {
     if (this.forcedProvider) {
       const active = this.activeDeployment();
@@ -351206,8 +351277,10 @@ ${persisted.map((guide, index) => `Guide ${index + 1}: ${guide.text}`).join("\n"
       }
       const selectedModel = this.activeModelConfig();
       if (images.length && !selectedModel?.vision) {
-        this.status = "idle";
-        return [{ type: "text", text: `[Vision unavailable] ${this.activeModelName() || this.model} has not passed image-input validation. Select a validated vision model before asking about attachments.` }];
+        const hasSameProviderVision = selectedModel && this.config.allModels().some(
+          (model) => model.enabled !== false && model.provider_id === selectedModel.provider_id && model.name !== selectedModel.name && !!model.vision
+        );
+        if (hasSameProviderVision) this.switchToFallbackModel("vision input not supported by the selected model");
       }
       const now2 = this.nowLabel();
       const visibleUserInput = inputEnvelope?.visibleUserInput === void 0 ? text : String(inputEnvelope.visibleUserInput || "");

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "newmark-agent",
   "productName": "Newmark Agent",
-  "version": "0.5.3",
+  "version": "0.5.4",
   "description": "Newmark Agent — Portable AI coding agent with rich GUI and CLI (TypeScript)",
   "homepage": "https://github.com/positer/Newmark-Agent",
   "repository": {