dsh-vision-router 1.2.3 → 1.4.0

package/index.js

@@ -233,6 +233,12 @@ export const Config = z.object({
   tool: z.boolean().default(true),
   progressiveTools: z.boolean().default(true),
   autoActivateOnImage: z.boolean().default(true),
+  // User feedback (Zhipu official channel): some channels expose vision
+  // models whose catalog metadata does not declare image input. Models the
+  // built-in name inference does not recognize can be forced here — one model
+  // id (or "provider/model") per entry. Only consulted for vision BACKEND
+  // capability (the session-side admission stays host-owned).
+  extraVisionModels: z.array(z.string()).default([]),
   // Client-persisted UI state (issue #78): DSH Desktop serves the Web UI from
   // a random port on every launch, so origin-scoped localStorage forgets the
   // first-run onboarding dialog and it re-appeared on every boot. These keys
@@ -2005,6 +2011,158 @@ export function modelInfoAcceptsImages(info) {
   return Array.isArray(info && info.inputModalities) && info.inputModalities.includes('image')
 }
 
+// User feedback: channels like the Zhipu official one (open.bigmodel.cn,
+// configured with a custom model list) expose vision models whose catalog
+// metadata does NOT declare image input, even though the models accept images
+// (e.g. glm-4.6v). DSH's Web settings do not write the `input: [text, image]`
+// declaration for custom channels either, so a strict metadata check hides
+// perfectly usable vision backends. The conservative, curated name patterns
+// below recognize well-known multimodal model families as a fallback; models
+// that still do not match can be forced via the `extraVisionModels` setting.
+// A vision-looking name does not necessarily identify a generative chat model.
+// Embedding and reranker endpoints often share the same VL family prefix but
+// cannot answer vision_describe. Keep them out of the automatic candidate
+// set; an explicit extraVisionModels override remains the expert escape hatch.
+const NON_GENERATIVE_VISION_MODEL_HINTS = [
+  /(^|[\/_.-])(embedding|embeddings|embed)(?=$|[\/_.-])/i,
+  /(^|[\/_.-])(rerank|reranker|reranking)(?=$|[\/_.-])/i,
+]
+
+export function looksLikeNonGenerativeVisionModel(modelId) {
+  const id = String(modelId ?? '').trim()
+  if (id === '') return false
+  return NON_GENERATIVE_VISION_MODEL_HINTS.some((pattern) => pattern.test(id))
+}
+
+const VISION_MODEL_NAME_HINTS = [
+  // Zhipu VLM family: glm-4.6v, glm-4.6v-flash, glm-4v-plus, glm-4.5v(-plus)…
+  /(^|\/)glm-4[\w.-]*v(?=$|[-/])/i,
+  /(^|\/)glm-4v(?=$|[-/])/i,
+  // Qwen VL / QVQ vision-reasoning family (excludes plain qwen3-14b etc.).
+  /(^|\/)qwen[\w.-]*(vl|vision)/i,
+  /(^|\/)qvq(?=$|[-.])/i,
+  // OpenAI multimodal line (gpt-4o*, gpt-4.1*, gpt-5*, gpt-oss*).
+  /(^|\/)gpt-(4o|4\.1|5|oss)(?=$|[-.])/i,
+  /(^|\/)gemini/i,
+  // Claude 3+ / Sonnet/Opus/Haiku are multimodal (claude-2 is not).
+  /(^|\/)(claude-(3|4)(?=$|[-.])|claude[\w.-]*(sonnet|opus|haiku))/i,
+  /(^|\/)(internvl|cogvlm|llava|pixtral)/i,
+  /(^|\/)(doubao|hunyuan|minimax|ernie)[\w.-]*(vl|vision)/i,
+  /(^|\/)ernie-4\.5/i,
+  /(^|\/)(yi-vision|kimi[\w.-]*vision|moonshot[\w.-]*vision)/i,
+  /(^|\/)step[\w.-]*(v|vision)(?=$|[-/])/i,
+  /(^|\/)grok[\w.-]*vision/i,
+  /(^|\/)grok-4(?=$|[-.])/i,
+  /(^|\/)llama[\w.-]*vision/i,
+  /(^|\/)mistral[\w.-]*pixtral/i,
+  /(^|\/)(phi[\w.-]*vision|florence[\w.-]*)/i,
+]
+
+/**
+ * Conservative name-based inference for vision capability: true only when the
+ * model id matches a well-known multimodal naming pattern. Used as a fallback
+ * when catalog metadata does not declare image input; never overrides an
+ * explicit text-only declaration on the session/twin paths.
+ */
+export function looksLikeVisionModel(modelId) {
+  const id = String(modelId ?? '').trim()
+  if (id === '' || looksLikeNonGenerativeVisionModel(id)) return false
+  return VISION_MODEL_NAME_HINTS.some((pattern) => pattern.test(id))
+}
+
+/**
+ * Pure capability decision for a vision backend: an explicit user override
+ * wins first, known non-generative endpoint roles are excluded next, then
+ * declared image metadata and conservative name inference are considered.
+ *
+ * @param info - resolved model metadata (may be undefined when the lookup failed).
+ * @param provider - provider id, used to match "provider/model" override entries.
+ * @param model - model id.
+ * @param extraVisionModels - user-configured model ids (or "provider/model") forced vision-capable.
+ * @returns { image, inputModalities, inferred, reason } where `inferred` is
+ * false for declared image input, 'override' for the user list, 'name' for the
+ * naming heuristic, and `reason` explains a text-only verdict.
+ */
+export function decideVisionBackendCapability(info, provider, model, extraVisionModels) {
+  const inputModalities = Array.isArray(info && info.inputModalities)
+    ? info.inputModalities.filter((item) => typeof item === 'string')
+    : []
+  const modelId = String(model ?? '').trim()
+  const providerId = String(provider ?? '').trim()
+  const extras = Array.isArray(extraVisionModels)
+    ? extraVisionModels.map((entry) => String(entry ?? '').trim()).filter((entry) => entry !== '')
+    : []
+  const forced =
+    modelId !== '' &&
+    extras.some((entry) => entry === modelId || (providerId !== '' && entry === `${providerId}/${modelId}`))
+  // Manual override is deliberately strongest: advanced users can still
+  // force an unusual endpoint that our role/name heuristics reject.
+  if (forced) {
+    return { image: true, inputModalities: [...new Set([...inputModalities, 'image'])], inferred: 'override', reason: undefined }
+  }
+  // A model can consume images and still be the wrong KIND of endpoint
+  // for this plugin: embedding/reranking produces no assistant answer.
+  if (modelId !== '' && looksLikeNonGenerativeVisionModel(modelId)) {
+    return {
+      image: false,
+      inputModalities,
+      inferred: false,
+      reason: 'model name indicates an embedding/reranker endpoint, not a generative vision backend',
+    }
+  }
+  if (inputModalities.includes('image')) {
+    return { image: true, inputModalities, inferred: false, reason: undefined }
+  }
+  if (modelId !== '' && looksLikeVisionModel(modelId)) {
+    return { image: true, inputModalities: [...new Set([...inputModalities, 'image'])], inferred: 'name', reason: undefined }
+  }
+  return {
+    image: false,
+    inputModalities,
+    inferred: false,
+    reason: 'model metadata does not declare image input',
+  }
+}
+
+/**
+ * Resolve transport facts for the direct channel compatibility bridge.
+ * Raw llm-pi-ai settings commonly omit baseURL/api for built-in catalog
+ * providers; the materialized pi-ai model carries the effective values.
+ */
+export function resolveChannelBridgeTransport(rawProfile, resolvedProfile, modelId) {
+  let resolvedModel
+  try {
+    const getModels = resolvedProfile && resolvedProfile.piProvider && resolvedProfile.piProvider.getModels
+    const models = typeof getModels === 'function'
+      ? getModels.call(resolvedProfile.piProvider)
+      : []
+    resolvedModel = Array.isArray(models)
+      ? models.find((entry) => entry && String(entry.id) === String(modelId))
+      : undefined
+  } catch {
+    resolvedModel = undefined
+  }
+  const firstString = (...values) =>
+    values.find((value) => typeof value === 'string' && value.trim() !== '')
+  return {
+    baseURL: firstString(
+      resolvedModel && resolvedModel.baseUrl,
+      rawProfile && rawProfile.baseURL,
+      resolvedProfile && resolvedProfile.baseURL,
+      resolvedProfile && resolvedProfile.piProvider && resolvedProfile.piProvider.baseUrl,
+    ),
+    api: firstString(
+      resolvedModel && resolvedModel.api,
+      rawProfile && rawProfile.api,
+      resolvedProfile && resolvedProfile.api,
+    ),
+    apiKeyEnv: firstString(
+      rawProfile && rawProfile.apiKeyEnv,
+      resolvedProfile && resolvedProfile.apiKeyEnv,
+    ),
+  }
+}
+
 export function apply(ctx, config = {}) {
   // Route sharp version diagnostics (issue #75) through the harness logger
   // instead of console.warn, so the warning lands in the server log.
@@ -2640,15 +2798,13 @@ export function apply(ctx, config = {}) {
     }
     try {
       const info = await ctx.llm.resolveModelInfo(provider, model)
-      const inputModalities = Array.isArray(info && info.inputModalities)
-        ? info.inputModalities.filter((item) => typeof item === 'string')
-        : []
-      return {
-        image: modelInfoAcceptsImages(info),
-        inputModalities,
-        reason: modelInfoAcceptsImages(info) ? undefined : 'model metadata does not declare image input',
-      }
+      return decideVisionBackendCapability(info, provider, model, current().extraVisionModels)
     } catch (error) {
+      // Metadata lookup failed (custom model lists often do): fall back to
+      // the user override list and the name heuristic before declaring the
+      // model text-only.
+      const fallback = decideVisionBackendCapability(undefined, provider, model, current().extraVisionModels)
+      if (fallback.image) return fallback
       return {
         image: false,
         inputModalities: [],
@@ -2657,6 +2813,123 @@ export function apply(ctx, config = {}) {
     }
   }
 
+  // ── direct OpenAI-compatible bridge for undeclared vision channels ─────────
+  //
+  // User feedback (Zhipu official channel, open.bigmodel.cn): some channels
+  // expose vision models whose catalog metadata does NOT declare image input,
+  // so the channel adapter refuses image requests at the wire
+  // (UNSUPPORTED_CONTENT: model "x" does not support image input) even though
+  // the models accept images. For backends recognized only through the name
+  // inference or the extraVisionModels override, fall back to calling the
+  // channel's OpenAI-compatible endpoint directly with the channel's own
+  // baseURL and credential — no hand-edited settings.yaml needed. Defensive
+  // reads only: if the channel settings section, the baseURL, or the
+  // credential cannot be resolved, the bridge is simply unavailable and the
+  // adapter's own error is reported.
+  const rawChannelProfileOf = (provider) => {
+    try {
+      const settings = ctx.get('settings')
+      const section =
+        settings && typeof settings.get === 'function' ? settings.get('llm-pi-ai') : undefined
+      return section && section.providers ? section.providers[provider] : undefined
+    } catch {
+      return undefined
+    }
+  }
+  // DSH's public model metadata intentionally omits endpoint/protocol
+  // details. PiAiAdapter has already materialized those facts in its
+  // resolved profile, so feature-detect that shape as a compatibility
+  // shim. If upstream changes it, this fails closed to the normal chain.
+  const resolvedPiAiProfileOf = (provider) => {
+    try {
+      const registration = ctx.llm.registration(provider)
+      const adapter = registration && registration.adapter
+      const config = adapter && adapter.config
+      const profiles = config && typeof config.profiles === 'function' ? config.profiles() : undefined
+      return profiles && typeof profiles.get === 'function' ? profiles.get(provider) : undefined
+    } catch {
+      return undefined
+    }
+  }
+  const channelBridgePlan = (provider, model) => {
+    const rawProfile = rawChannelProfileOf(provider)
+    const resolvedProfile = resolvedPiAiProfileOf(provider)
+    const transport = resolveChannelBridgeTransport(rawProfile, resolvedProfile, model)
+    if (!transport.baseURL) {
+      return { ok: false, reason: 'no resolved channel baseURL', rawProfile, resolvedProfile, transport }
+    }
+    // callOpenAICompatible speaks Chat Completions. Never send another
+    // provider protocol through this bridge just because its name looks visual.
+    if (transport.api !== 'openai-completions') {
+      return {
+        ok: false,
+        reason: `channel protocol ${transport.api || 'unknown'} is not OpenAI Chat Completions`,
+        rawProfile,
+        resolvedProfile,
+        transport,
+      }
+    }
+    return { ok: true, rawProfile, resolvedProfile, transport }
+  }
+  const resolveChannelApiKey = async (plan) => {
+    const ref = plan && plan.transport && plan.transport.apiKeyEnv
+    if (typeof ref === 'string' && ref !== '') {
+      try {
+        const credentials = ctx.get('credentials')
+        if (credentials !== undefined) {
+          const hit = await credentials.resolve(ref)
+          if (hit && typeof hit.value === 'string' && hit.value.length > 0) return hit.value
+        }
+      } catch {
+        /* fall through to the ambient environment */
+      }
+      if (typeof process !== 'undefined' && process.env && typeof process.env[ref] === 'string') {
+        return process.env[ref]
+      }
+    }
+    // Catalog routes may use provider-native environment discovery and
+    // therefore carry no explicit Harness credential reference.
+    try {
+      const auth = plan && plan.resolvedProfile && plan.resolvedProfile.piProvider
+        && plan.resolvedProfile.piProvider.auth && plan.resolvedProfile.piProvider.auth.apiKey
+      if (auth && typeof auth.resolve === 'function') {
+        const hit = await auth.resolve({ credential: undefined })
+        const value = hit && hit.auth && hit.auth.apiKey
+        if (typeof value === 'string' && value.length > 0) return value
+      }
+    } catch {
+      /* unavailable native auth */
+    }
+    return undefined
+  }
+  const directChannelVisionAnswer = async (provider, model, blocks, instruction, signal) => {
+    const plan = channelBridgePlan(provider, model)
+    if (!plan.ok) throw new Error(`vision bridge unavailable: ${plan.reason}`)
+    const apiKey = await resolveChannelApiKey(plan)
+    if (apiKey === undefined || apiKey === '') {
+      throw new Error('vision bridge unavailable: channel credential could not be resolved')
+    }
+    const attachments = ctx.get('attachments')
+    if (attachments === undefined) {
+      throw new Error('vision bridge unavailable: attachment service is not registered')
+    }
+    const content = []
+    for (const block of blocks) {
+      const stored = await attachments.readImage(block.attachment)
+      content.push(...toOpenAIContent([block], () => stored.data))
+    }
+    return callOpenAICompatible(
+      {
+        name: provider,
+        baseURL: plan.transport.baseURL,
+        model,
+        apiKeyEnv: '__vision-router-channel__',
+      },
+      [{ role: 'user', content: [...content, { type: 'text', text: instruction }] }],
+      { maxTokens: 4096, signal, resolveCredential: () => apiKey },
+    )
+  }
+
   const collectVisionBackendCapabilities = async () => {
     const capabilities = {}
     if (typeof ctx.llm.listProviders !== 'function') return capabilities
@@ -3768,6 +4041,7 @@ export function apply(ctx, config = {}) {
       const block = await visionBlocksFromBytes(imageBytes, mediaType)
       const signal = AbortSignal.timeout(timeoutMs())
       const usablePairs = await resolveToolVisionPairs()
+      const pairCapabilities = new Map()
       for (const pair of pairs()) {
         if (!pair || pair.provider === HTTP_ROUTE) continue
         if (!adapterAvailable(ctx.llm, pair.provider)) {
@@ -3775,6 +4049,7 @@ export function apply(ctx, config = {}) {
           continue
         }
         const capability = await resolveVisionBackendCapability(pair.provider, pair.model)
+        pairCapabilities.set(`${pair.provider}/${pair.model}`, capability)
         if (!capability.image) {
           errors.push(
             `${pair.provider}/${pair.model}: not an image-capable backend (${capability.reason ?? 'unknown capability'})`,
@@ -3782,6 +4057,16 @@ export function apply(ctx, config = {}) {
         }
       }
       for (const pair of usablePairs) {
+        // usablePairs also contains auto-discovered models. Before this fix the
+        // map was populated only from explicit config rows, so inferred
+        // SiliconFlow models failed pi-ai image admission and never reached
+        // the direct channel bridge that was meant to rescue them.
+        const pairKey = `${pair.provider}/${pair.model}`
+        let pairCapability = pairCapabilities.get(pairKey)
+        if (pairCapability === undefined) {
+          pairCapability = await resolveVisionBackendCapability(pair.provider, pair.model)
+          pairCapabilities.set(pairKey, pairCapability)
+        }
         try {
           const text = await visionAnswer(ctx.llm, {
             provider: pair.provider,
@@ -3794,6 +4079,30 @@ export function apply(ctx, config = {}) {
           })
           if (text && text.trim() !== '') return { text: text.trim() }
         } catch (error) {
+          // Channels whose catalog does not declare image input reject images
+          // at the adapter wire. When the backend was recognized by the name
+          // inference or the extraVisionModels override, call the channel's
+          // OpenAI-compatible endpoint directly with its own baseURL and
+          // credential before giving up on this pair.
+          const capability = pairCapabilities.get(`${pair.provider}/${pair.model}`)
+          if (capability && capability.inferred) {
+            try {
+              const direct = await directChannelVisionAnswer(
+                pair.provider,
+                pair.model,
+                [block],
+                instruction,
+                signal,
+              )
+              if (direct && direct.trim() !== '') return { text: direct.trim() }
+            } catch (bridgeError) {
+              errors.push(
+                `${pair.provider}/${pair.model}: direct channel fallback failed (${
+                  bridgeError && bridgeError.message ? bridgeError.message : String(bridgeError)
+                })`,
+              )
+            }
+          }
           errors.push(`${pair.provider}/${pair.model}: ${error && error.message ? error.message : String(error)}`)
         }
       }