dsh-vision-fallback 0.5.0 → 0.7.0

package/lib/index.js

@@ -30,6 +30,13 @@ const DEFAULT_PROMPT = [
 
 const Config = z.object({
   enabled: z.boolean().default(true),
+  /**
+   * auto：仅当主模型不支持图片输入时才接管（默认）。主模型自带视觉
+   * （如 kimi-k3、mimo-v2.5）时图片原样交给它，不再转文字。
+   * always：无论主模型是否支持图片都转成文字（旧行为）。
+   * never：完全不接管（等于停用视觉桥；纯文本模型收图会照常报"模型不支持"）。
+   */
+  mode: z.union(["auto", "always", "never"]).default("auto"),
   model: z.string().default("mimo-v2.5"),
   baseURL: z.string().default("https://opencode.ai/zen/go/v1"),
   apiKeyRef: z.string().role("credential-ref").default("OPENCODE_GO_API_KEY"),
@@ -229,7 +236,7 @@ function assertVisionConfig(cfg) {
 }
 
 function withImageCapability(info, cfg) {
-  if (!cfg.enabled) return info;
+  if (!cfg.enabled || cfg.mode === "never") return info;
   const inputModalities = Array.isArray(info?.inputModalities) ? info.inputModalities : ["text"];
   if (inputModalities.includes("image")) return info;
   return { ...info, inputModalities: [...new Set([...inputModalities, "text", "image"])] };
@@ -449,13 +456,16 @@ class ReplacementCoordinator {
       replacements = new Map();
       this.pending.set(session, replacements);
     }
+    const activeMessageIds = new Set(originalMessages.map((message) => message.id));
+    for (const messageId of replacements.keys()) {
+      if (!activeMessageIds.has(messageId)) replacements.delete(messageId);
+    }
     for (let index = 0; index < originalMessages.length; index += 1) {
       const original = originalMessages[index];
       const rewritten = rewrittenMessages[index];
       if (original === rewritten || !contentHasImage(original.content ?? [])) continue;
       replacements.set(original.id, rewritten);
     }
-    if (replacements.size > 128) replacements.clear();
   }
 
   onSessionEvent(session, event) {
@@ -485,7 +495,11 @@ class ReplacementCoordinator {
   }
 }
 
+/** 原始的 resolveModelInfo 实现（绑定原持有者，未被能力覆盖污染）。 */
+let rawResolveModelInfo = null;
+
 function installCapabilityOverride(ctx, current) {
+  rawResolveModelInfo = ctx.llm.resolveModelInfo.bind(ctx.llm);
   const previous = ctx.llm.resolveModelInfo;
   const overridden = async function (provider, model, signal) {
     const info = await previous.call(this, provider, model, signal);
@@ -494,9 +508,25 @@ function installCapabilityOverride(ctx, current) {
   ctx.llm.resolveModelInfo = overridden;
   return () => {
     if (ctx.llm.resolveModelInfo === overridden) ctx.llm.resolveModelInfo = previous;
+    rawResolveModelInfo = null;
   };
 }
 
+/**
+ * 用未被覆盖的 resolveModelInfo 查询主模型的真实图片能力。
+ * 解析失败保守返回 false（沿用接管路径，至少不会让请求挂掉）。
+ */
+async function modelActuallySupportsImage(provider, model, signal) {
+  if (typeof provider !== "string" || typeof model !== "string" || provider === "" || model === "") return false;
+  if (typeof rawResolveModelInfo !== "function") return false;
+  try {
+    const info = await rawResolveModelInfo(provider, model, signal);
+    return Array.isArray(info?.inputModalities) && info.inputModalities.includes("image");
+  } catch {
+    return false;
+  }
+}
+
 function apply(ctx, config) {
   let current = () => config;
   let settingsService;
@@ -512,9 +542,18 @@ function apply(ctx, config) {
   ctx.on("agent/pre-step", async ({ agent, signal }, next) => {
     const decision = await next();
     if (decision.kind === "reject" || signal.aborted) return decision;
-    const contextMessages = [...agent.session.deriveMessages(), ...decision.messages];
-    const rewritten = await controller.preprocess(decision.messages, contextMessages, signal);
-    replacements.stage(agent.session, decision.messages, rewritten);
+    const cfg = current();
+    if (!cfg.enabled || cfg.mode === "never") return decision;
+    // auto 模式：主模型自带视觉能力时让它直接看原图，不接管。
+    if (cfg.mode === "auto") {
+      const header = typeof agent?.session?.requestHeader === "function" ? agent.session.requestHeader() : undefined;
+      const provider = header?.config?.provider ?? agent?.options?.provider;
+      const model = header?.config?.model ?? agent?.options?.model;
+      if (await modelActuallySupportsImage(provider, model, signal)) return decision;
+    }
+    const modelMessages = [...agent.session.deriveMessages(), ...decision.messages];
+    const rewritten = await controller.preprocess(modelMessages, modelMessages, signal);
+    replacements.stage(agent.session, modelMessages, rewritten);
     return decision;
   });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-vision-fallback",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "DSH 静默视觉增强：主模型照常选择，图片自动交给固定视觉模型后以隐藏上下文返回主模型。",
   "type": "module",
   "main": "lib/index.js",

package/test/vision-fallback.test.mjs

@@ -198,6 +198,52 @@ test("surface replacement 事件不会再次触发替换", async () => {
   assert.deepEqual(appends[0].options.surfaceOp, { op: "replace", start: 12, end: 12 });
 });
 
+test("历史工具结果中的图片只以文字形式进入模型视图", async () => {
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async () => ({
+    ok: true,
+    json: async () => ({ choices: [{ message: { content: "这是主题插件的界面截图" } }] })
+  });
+  const historicalToolResult = userMessage("tool-result-1", [{
+    type: "tool-result",
+    toolCallId: "read-image-1",
+    content: [
+      { type: "text", text: "图片读取成功" },
+      { type: "image", attachment }
+    ],
+    isError: false
+  }]);
+  const current = userMessage("u2", [{ type: "text", text: "这是什么插件？" }]);
+  const surface = [historicalToolResult];
+  const session = {
+    deriveMessages: () => surface,
+    append() {
+      throw new Error("历史消息投影不应改写界面事件");
+    }
+  };
+  const controller = new VisionFallbackController(createServiceContext(), () => config);
+  const coordinator = new ReplacementCoordinator({ error() {} });
+
+  try {
+    const modelMessages = [...session.deriveMessages(), current];
+    const rewritten = await controller.preprocess(modelMessages, modelMessages);
+    coordinator.stage(session, modelMessages, rewritten);
+
+    const projected = session.deriveMessages();
+    assert.equal(projected.length, 1);
+    assert.equal(projected[0].id, historicalToolResult.id);
+    assert.equal(projected[0].content[0].type, "tool-result");
+    assert.equal(projected[0].content[0].content[1].type, "text");
+    assert.match(projected[0].content[0].content[1].text, /这是主题插件的界面截图/);
+    assert.equal(JSON.stringify(projected).includes('"type":"image"'), false);
+    assert.equal(surface[0], historicalToolResult);
+    assert.equal(surface[0].content[0].content[1].type, "image");
+  } finally {
+    coordinator.dispose();
+    globalThis.fetch = originalFetch;
+  }
+});
+
 test("图片能力覆盖可随开关变化并能恢复", async () => {
   let current = config;
   const original = async (provider, model) => ({
@@ -441,3 +487,93 @@ test("浏览器端只显示静默视觉配置", async () => {
     globalThis.fetch = previousFetch;
   }
 });
+
+test("auto 模式按主模型真实视觉能力决定是否接管", async () => {
+  const imageBlock = { type: "image", attachment: { attachmentId: "sha256:test", mediaType: "image/png" } };
+  let visionRequests = 0;
+  const previousFetch = globalThis.fetch;
+  globalThis.fetch = async () => {
+    visionRequests += 1;
+    return {
+      ok: true,
+      status: 200,
+      json: async () => ({ choices: [{ message: { content: "红色" } }] })
+    };
+  };
+
+  const makeSignal = () => ({ aborted: false, addEventListener() {}, removeEventListener() {} });
+  const makeCtx = (realModalities) => {
+    let preStep;
+    const ctx = {
+      llm: {
+        resolveModelInfo: async () => ({ provider: "opencode-go", id: "x", name: "x", inputModalities: realModalities })
+      },
+      logger: { error() {} },
+      effect() {},
+      on(name, handler) {
+        if (name === "agent/pre-step") preStep = handler;
+      },
+      inject(deps, cb) {
+        if (deps[0] === "settings") {
+          cb({
+            settings: {
+              register: (_ns, _schema, options) => ({ get: () => options.base, watch() {} })
+            },
+            effect() {}
+          });
+        } else {
+          cb({ webServer: { register: () => () => {} }, effect() {} });
+        }
+      },
+      get(name) {
+        if (name === "credentials") return { resolve: async () => ({ value: "sk-test" }) };
+        if (name === "attachments") return { readImage: async () => ({ data: Buffer.from("test"), ref: { mediaType: "image/png" } }) };
+        return undefined;
+      }
+    };
+    return { ctx, preStep: () => preStep };
+  };
+
+  // 1) 主模型自带视觉 → 不接管，视觉 API 不被调用
+  const vision = makeCtx(["text", "image"]);
+  apply(vision.ctx, { ...config, mode: "auto" });
+  const messages1 = [{ id: "m1", role: "user", source: { kind: "user" }, content: [{ type: "text", text: "看图" }, imageBlock] }];
+  const decision1 = { kind: "enter", messages: messages1 };
+  const out1 = await vision.preStep()(
+    {
+      agent: {
+        options: { provider: "opencode-go", model: "kimi-k3" },
+        session: {
+          requestHeader: () => ({ config: { provider: "opencode-go", model: "kimi-k3" } }),
+          deriveMessages: () => []
+        }
+      },
+      signal: makeSignal()
+    },
+    () => decision1
+  );
+  assert.equal(out1, decision1);
+  assert.equal(visionRequests, 0);
+
+  // 2) 纯文本主模型 → 接管，视觉 API 被调用一次
+  const text = makeCtx(["text"]);
+  apply(text.ctx, { ...config, mode: "auto" });
+  const messages2 = [{ id: "m2", role: "user", source: { kind: "user" }, content: [{ type: "text", text: "看图" }, imageBlock] }];
+  const decision2 = { kind: "enter", messages: messages2 };
+  await text.preStep()(
+    {
+      agent: {
+        options: { provider: "opencode-go", model: "deepseek-v4-flash" },
+        session: {
+          requestHeader: () => ({ config: { provider: "opencode-go", model: "deepseek-v4-flash" } }),
+          deriveMessages: () => []
+        }
+      },
+      signal: makeSignal()
+    },
+    () => decision2
+  );
+  assert.equal(visionRequests, 1);
+
+  globalThis.fetch = previousFetch;
+});