@heventure/model-provider-x 0.2.3 → 0.2.4

package/dist/cli/args.js

@@ -29,6 +29,12 @@ export function parseCliArgs(argv) {
                     .map((model) => model.trim())
                     .filter(Boolean);
                 break;
+            case "--modalities":
+                if (!options.modalities) {
+                    options.modalities = [];
+                }
+                options.modalities.push(parseModalitiesArg(next()));
+                break;
             case "--name":
                 options.providerName = next();
                 break;
@@ -107,6 +113,8 @@ Options:
   --proxy              Write agent config through the local compatibility proxy.
   --direct             Write agent config directly to the upstream provider.
   --models <list>      Comma-separated model ids. Skips interactive model selection.
+  --modalities <spec>  Model modalities in format <model>:<input>:<output>.
+                       Example: --modalities qwen-vl:image,text:text
   --config <path>      OpenCode config path to write when targeting OpenCode.
   --print              Print generated JSON and do not write config.
   --yes, -y            Accept defaults in non-interactive prompts.
@@ -134,3 +142,29 @@ function addModelByOneBasedIndex(selected, models, index) {
     }
     selected.add(model);
 }
+function parseModalitiesArg(value) {
+    const parts = value.split(":");
+    if (parts.length !== 3) {
+        throw new Error(`Invalid --modalities format: ${value}. Expected <model>:<input>:<output>`);
+    }
+    const [modelId, inputStr, outputStr] = parts;
+    if (!modelId) {
+        throw new Error("Model id is required in --modalities");
+    }
+    const parseModalityList = (s) => {
+        return s.split(",").map((m) => {
+            const trimmed = m.trim().toLowerCase();
+            if (["text", "image", "audio", "video", "pdf"].includes(trimmed)) {
+                return trimmed;
+            }
+            throw new Error(`Unknown modality: ${trimmed}`);
+        });
+    };
+    return {
+        modelId,
+        modalities: {
+            input: parseModalityList(inputStr),
+            output: parseModalityList(outputStr),
+        },
+    };
+}

package/dist/cli/index.js

@@ -6,6 +6,7 @@ import { readProxyStatus, startProxyProcess, stopProxyProcess } from "../core/pr
 import { createProxyAuthToken, getDefaultToolConfigPath, readToolConfig, upsertProviderProfile, writeToolConfig } from "../core/tool-config.js";
 import { discoverOpenCodeConfigs, getDefaultConfigPath, writeProviderToConfig } from "../core/config.js";
 import { buildProviderConfig, detectProviderCapabilities, recommendProxyMode, validateAndFetchModels } from "../core/provider.js";
+import { isKnownModel } from "../core/model-capabilities.js";
 import { startProxyServer } from "../proxy/server.js";
 import { defaultClaudeModelMapping, getDefaultClaudeSettingsPath, writeClaudeCodeSettings } from "../targets/claude-code.js";
 import { getDefaultCodexConfigPath, writeCodexConfig } from "../targets/codex.js";
@@ -76,12 +77,14 @@ export async function runCli(options) {
             (canUseTui()
                 ? await multiSelectChoices("Select models", createModelChoices(fetched.models))
                 : parseModelSelection(await rl.question(formatModelPrompt(fetched.models)), fetched.models));
+        const modelDetails = fetched.modelDetails.filter((m) => selectedModels.includes(m.id));
         const fragment = buildProviderConfig({
             providerId,
             providerName,
             baseURL: fetched.baseURL,
             apiKey,
             models: selectedModels,
+            modelDetails,
             opencodeApiType: options.opencodeApiType ?? "chat"
         });
         const provider = fragment.provider[providerId];
@@ -151,6 +154,7 @@ async function collectProviderSelection(rl, command, providerInput) {
     if (!defaultModel) {
         throw new Error("Select at least one model");
     }
+    const modelDetails = await resolveModelModalities(rl, command, selectedModels, fetched.modelDetails);
     const toolConfigPath = getDefaultToolConfigPath();
     const config = await upsertProviderProfile(toolConfigPath, {
         id: providerInput.providerId,
@@ -168,6 +172,7 @@ async function collectProviderSelection(rl, command, providerInput) {
         upstreamBaseURL: fetched.baseURL,
         apiKey: providerInput.apiKey,
         selectedModels,
+        modelDetails,
         defaultModel,
         config,
         toolConfigPath
@@ -186,6 +191,7 @@ async function writeOpenCodeSetup(rl, command, selection, useProxy, capabilities
         baseURL,
         apiKey,
         models: selection.selectedModels,
+        modelDetails: selection.modelDetails,
         opencodeApiType
     });
     const provider = fragment.provider[selection.providerId];
@@ -634,4 +640,73 @@ function slugify(value) {
         .replace(/[^a-z0-9]+/g, "-")
         .replace(/^-+|-+$/g, "");
 }
+async function resolveModelModalities(rl, command, selectedModels, fetchedModelDetails) {
+    const userModalityOverrides = new Map();
+    if (command.options.modalities) {
+        for (const override of command.options.modalities) {
+            userModalityOverrides.set(override.modelId, override.modalities);
+        }
+    }
+    const result = [];
+    for (const modelId of selectedModels) {
+        const fetched = fetchedModelDetails.find((m) => m.id === modelId);
+        const userOverride = userModalityOverrides.get(modelId);
+        if (userOverride) {
+            result.push({ id: modelId, modalities: userOverride });
+            continue;
+        }
+        if (fetched?.modalities) {
+            result.push(fetched);
+            continue;
+        }
+        if (command.options.yes || isKnownModel(modelId)) {
+            result.push(fetched ?? { id: modelId });
+            continue;
+        }
+        const modalities = await promptModelModalities(rl, modelId);
+        result.push({ id: modelId, modalities });
+    }
+    return result;
+}
+async function promptModelModalities(rl, modelId) {
+    output.write(`\nModel: ${modelId} - Capabilities unknown\n`);
+    if (canUseTui()) {
+        const inputModalities = await multiSelectChoices("Input modalities", [
+            { label: "text", value: "text", hint: "default" },
+            { label: "image", value: "image" },
+            { label: "audio", value: "audio" },
+            { label: "video", value: "video" },
+            { label: "pdf", value: "pdf" },
+        ]);
+        const outputModalities = await multiSelectChoices("Output modalities", [
+            { label: "text", value: "text", hint: "default" },
+            { label: "image", value: "image" },
+            { label: "audio", value: "audio" },
+            { label: "video", value: "video" },
+            { label: "pdf", value: "pdf" },
+        ]);
+        return {
+            input: inputModalities,
+            output: outputModalities,
+        };
+    }
+    const inputAnswer = await rl.question("Input modalities [text]: ");
+    const outputAnswer = await rl.question("Output modalities [text]: ");
+    const parseModalityList = (answer) => {
+        if (!answer.trim()) {
+            return ["text"];
+        }
+        return answer.split(",").map((m) => {
+            const trimmed = m.trim().toLowerCase();
+            if (["text", "image", "audio", "video", "pdf"].includes(trimmed)) {
+                return trimmed;
+            }
+            throw new Error(`Unknown modality: ${trimmed}`);
+        });
+    };
+    return {
+        input: parseModalityList(inputAnswer),
+        output: parseModalityList(outputAnswer),
+    };
+}
 void main();

package/dist/core/model-capabilities.js

@@ -0,0 +1,174 @@
+const MODEL_CAPABILITIES = {
+    "gpt-4o": { input: ["text", "image"], output: ["text"] },
+    "gpt-4o-mini": { input: ["text", "image"], output: ["text"] },
+    "gpt-4-turbo": { input: ["text", "image"], output: ["text"] },
+    "gpt-4-vision-preview": { input: ["text", "image"], output: ["text"] },
+    "gpt-4.1": { input: ["text", "image"], output: ["text"] },
+    "gpt-4.1-mini": { input: ["text", "image"], output: ["text"] },
+    "gpt-4.1-nano": { input: ["text", "image"], output: ["text"] },
+    "gpt-5": { input: ["text", "image"], output: ["text"] },
+    "gpt-5-mini": { input: ["text", "image"], output: ["text"] },
+    "gpt-5-nano": { input: ["text", "image"], output: ["text"] },
+    "o1": { input: ["text", "image"], output: ["text"] },
+    "o1-mini": { input: ["text", "image"], output: ["text"] },
+    "o1-pro": { input: ["text", "image"], output: ["text"] },
+    "o3": { input: ["text", "image"], output: ["text"] },
+    "o3-mini": { input: ["text", "image"], output: ["text"] },
+    "o3-pro": { input: ["text", "image"], output: ["text"] },
+    "o4-mini": { input: ["text", "image"], output: ["text"] },
+    "claude-3-opus": { input: ["text", "image"], output: ["text"] },
+    "claude-3-sonnet": { input: ["text", "image"], output: ["text"] },
+    "claude-3-haiku": { input: ["text", "image"], output: ["text"] },
+    "claude-3.5-sonnet": { input: ["text", "image"], output: ["text"] },
+    "claude-3.5-haiku": { input: ["text", "image"], output: ["text"] },
+    "claude-4-opus": { input: ["text", "image"], output: ["text"] },
+    "claude-4-sonnet": { input: ["text", "image"], output: ["text"] },
+    "claude-4.5-opus": { input: ["text", "image"], output: ["text"] },
+    "claude-4.5-sonnet": { input: ["text", "image"], output: ["text"] },
+    "claude-4.5-haiku": { input: ["text", "image"], output: ["text"] },
+    "gemini-pro-vision": { input: ["text", "image"], output: ["text"] },
+    "gemini-1.5-pro": { input: ["text", "image", "audio", "video"], output: ["text"] },
+    "gemini-1.5-flash": { input: ["text", "image", "audio", "video"], output: ["text"] },
+    "gemini-2.0-flash": { input: ["text", "image", "audio", "video"], output: ["text"] },
+    "gemini-2.5-pro": { input: ["text", "image", "audio", "video"], output: ["text"] },
+    "gemini-2.5-flash": { input: ["text", "image", "audio", "video"], output: ["text"] },
+    "qwen-vl-plus": { input: ["text", "image"], output: ["text"] },
+    "qwen-vl-max": { input: ["text", "image"], output: ["text"] },
+    "qwen-vl-chat": { input: ["text", "image"], output: ["text"] },
+    "qwen2.5-vl-72b-instruct": { input: ["text", "image"], output: ["text"] },
+    "qwen2.5-vl-7b-instruct": { input: ["text", "image"], output: ["text"] },
+    "qwen2.5-vl-32b-instruct": { input: ["text", "image"], output: ["text"] },
+    "qwen3-vl-235b-a22b": { input: ["text", "image"], output: ["text"] },
+    "qwen3-vl-30b-a3b": { input: ["text", "image"], output: ["text"] },
+    "llama-3.2-11b-vision-instruct": { input: ["text", "image"], output: ["text"] },
+    "llama-3.2-90b-vision-instruct": { input: ["text", "image"], output: ["text"] },
+    "llama-4-scout-17b-16e-instruct": { input: ["text", "image"], output: ["text"] },
+    "llama-4-maverick-17b-128e-instruct": { input: ["text", "image"], output: ["text"] },
+    "deepseek-vl": { input: ["text", "image"], output: ["text"] },
+    "deepseek-vl2": { input: ["text", "image"], output: ["text"] },
+    "glm-4v": { input: ["text", "image"], output: ["text"] },
+    "glm-4.5v": { input: ["text", "image"], output: ["text"] },
+    "glm-4.6v": { input: ["text", "image"], output: ["text"] },
+    "glm-5v-turbo": { input: ["text", "image"], output: ["text"] },
+    "pixtral-large-2411": { input: ["text", "image"], output: ["text"] },
+    "pixtral-large-2502": { input: ["text", "image"], output: ["text"] },
+    "mistral-small-3.1-24b-instruct": { input: ["text", "image"], output: ["text"] },
+    "phi-4-multimodal": { input: ["text", "image", "audio"], output: ["text"] },
+    "gemma-3-4b-it": { input: ["text", "image"], output: ["text"] },
+    "gemma-3-12b-it": { input: ["text", "image"], output: ["text"] },
+    "gemma-3-27b-it": { input: ["text", "image"], output: ["text"] },
+    "gemma-4-26b-a4b-it": { input: ["text", "image"], output: ["text"] },
+    "gemma-4-31b-it": { input: ["text", "image"], output: ["text"] },
+};
+const VISION_KEYWORDS = [
+    "vision",
+    "vl",
+    "visual",
+    "multimodal",
+    "pixtral",
+];
+const KNOWN_MODALITY_MODELS = new Set(Object.keys(MODEL_CAPABILITIES));
+export function lookupModelCapabilities(modelId) {
+    const normalized = modelId.toLowerCase().trim();
+    if (MODEL_CAPABILITIES[normalized]) {
+        return MODEL_CAPABILITIES[normalized];
+    }
+    for (const [pattern, caps] of Object.entries(MODEL_CAPABILITIES)) {
+        if (normalized.includes(pattern) || pattern.includes(normalized)) {
+            return caps;
+        }
+    }
+    const withoutPrefix = normalized.includes("/")
+        ? normalized.split("/").pop()
+        : normalized;
+    if (MODEL_CAPABILITIES[withoutPrefix]) {
+        return MODEL_CAPABILITIES[withoutPrefix];
+    }
+    for (const [pattern, caps] of Object.entries(MODEL_CAPABILITIES)) {
+        if (withoutPrefix.includes(pattern) || pattern.includes(withoutPrefix)) {
+            return caps;
+        }
+    }
+    for (const keyword of VISION_KEYWORDS) {
+        if (normalized.includes(keyword)) {
+            return { input: ["text", "image"], output: ["text"] };
+        }
+    }
+    return undefined;
+}
+export function parseCapabilitiesFromApi(model) {
+    if (model.modalities && typeof model.modalities === "object") {
+        const m = model.modalities;
+        const result = {};
+        if (Array.isArray(m.input)) {
+            result.input = m.input.filter((v) => typeof v === "string");
+        }
+        if (Array.isArray(m.output)) {
+            result.output = m.output.filter((v) => typeof v === "string");
+        }
+        if (result.input || result.output) {
+            return result;
+        }
+    }
+    if (model.capabilities && typeof model.capabilities === "object") {
+        const caps = model.capabilities;
+        const input = ["text"];
+        if (caps.vision || caps.image || caps.multimodal) {
+            input.push("image");
+        }
+        if (caps.audio || caps.speech) {
+            input.push("audio");
+        }
+        if (input.length > 1) {
+            return { input: input, output: ["text"] };
+        }
+    }
+    return undefined;
+}
+export function mergeModelCapabilities(modelId, apiCapabilities, userOverrides) {
+    if (userOverrides && (userOverrides.input || userOverrides.output)) {
+        return userOverrides;
+    }
+    if (apiCapabilities && (apiCapabilities.input || apiCapabilities.output)) {
+        return apiCapabilities;
+    }
+    return lookupModelCapabilities(modelId);
+}
+export function isKnownModel(modelId) {
+    const normalized = modelId.toLowerCase().trim();
+    if (KNOWN_MODALITY_MODELS.has(normalized)) {
+        return true;
+    }
+    const withoutPrefix = normalized.includes("/")
+        ? normalized.split("/").pop()
+        : normalized;
+    if (KNOWN_MODALITY_MODELS.has(withoutPrefix)) {
+        return true;
+    }
+    for (const keyword of VISION_KEYWORDS) {
+        if (normalized.includes(keyword)) {
+            return true;
+        }
+    }
+    return false;
+}
+export function parseModalitiesFromString(value) {
+    const parts = value.split(":");
+    if (parts.length !== 2) {
+        throw new Error(`Invalid modalities format: ${value}. Expected <input>:<output>`);
+    }
+    const [inputStr, outputStr] = parts;
+    const parseModalityList = (s) => {
+        return s.split(",").map((m) => {
+            const trimmed = m.trim().toLowerCase();
+            if (["text", "image", "audio", "video", "pdf"].includes(trimmed)) {
+                return trimmed;
+            }
+            throw new Error(`Unknown modality: ${trimmed}`);
+        });
+    };
+    return {
+        input: parseModalityList(inputStr),
+        output: parseModalityList(outputStr),
+    };
+}

package/dist/core/provider.js

@@ -1,3 +1,4 @@
+import { parseCapabilitiesFromApi, mergeModelCapabilities } from "./model-capabilities.js";
 export function normalizeBaseUrl(baseURL) {
     const normalized = baseURL.trim().replace(/\/+$/, "");
     if (!normalized) {
@@ -26,16 +27,25 @@ export async function validateAndFetchModels(input, fetchImpl = globalThis.fetch
     if (!isModelListResponse(body)) {
         throw new Error("Expected /models to return an object with a data array");
     }
+    const compatibleModels = body.data.filter(isOpenCodeCompatibleModel);
     const models = [
-        ...new Set(body.data
-            .filter(isOpenCodeCompatibleModel)
+        ...new Set(compatibleModels
             .map((model) => model.id.trim())
             .filter(Boolean))
     ];
     if (models.length === 0) {
         throw new Error("Provider returned no OpenCode-compatible model ids");
     }
-    return { baseURL, models };
+    const modelDetails = models.map((modelId) => {
+        const rawModel = compatibleModels.find((m) => m.id.trim() === modelId);
+        const apiCapabilities = rawModel ? parseCapabilitiesFromApi(rawModel) : undefined;
+        const mergedCapabilities = mergeModelCapabilities(modelId, apiCapabilities);
+        return {
+            id: modelId,
+            modalities: mergedCapabilities
+        };
+    });
+    return { baseURL, models, modelDetails };
 }
 export async function detectProviderCapabilities(input, fetchImpl = globalThis.fetch) {
     const baseURL = normalizeBaseUrl(input.baseURL);
@@ -78,7 +88,10 @@ export function buildProviderConfig(input) {
         options: {
             baseURL
         },
-        models: Object.fromEntries(input.models.map((model) => [model, { name: model }]))
+        models: Object.fromEntries(input.models.map((model) => {
+            const modelInfo = input.modelDetails?.find((m) => m.id === model);
+            return [model, buildModelConfig(model, modelInfo)];
+        }))
     };
     if (opencodeApiType !== "messages") {
         provider.options.setCacheKey = true;
@@ -93,6 +106,13 @@ export function buildProviderConfig(input) {
         }
     };
 }
+function buildModelConfig(modelId, modelInfo) {
+    const config = { name: modelId };
+    if (modelInfo?.modalities) {
+        config.modalities = modelInfo.modalities;
+    }
+    return config;
+}
 export function npmPackageForOpenCodeApiType(apiType) {
     if (apiType === "responses") {
         return "@ai-sdk/openai";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@heventure/model-provider-x",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "TUI configurator and local API proxy for wiring custom model providers into OpenCode and Claude Code.",
   "private": false,
   "license": "MIT",