@spacex110/core 0.1.15 → 0.1.17

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@spacex110/core",
-    "version": "0.1.15",
+    "version": "0.1.17",
     "description": "Framework-agnostic AI Provider Selector core utilities",
     "type": "module",
     "main": "./dist/index.js",

package/src/channel-manager.ts

@@ -15,10 +15,28 @@
  * 10. 事件系统：on / off / emit
  */
 
-import type { AIConfig, Provider, Model, ModelCapability, StorageAdapter, ApiFormat } from './types';
+import type {
+    AIConfig,
+    Provider,
+    Model,
+    ModelCapability,
+    StorageAdapter,
+    ApiFormat,
+    ChatMessage,
+    MultimodalChatOptions,
+    MultimodalChatStreamOptions,
+    MultimodalChatResult,
+    MultimodalStreamDelta,
+} from './types';
 import { getProvider, getAllProviders, getProvidersByFormat } from './providers';
 import { defaultStorageAdapter } from './storage';
-import { sendDirectChat, sendDirectChatStream, testDirectConnection } from './strategies';
+import {
+    sendDirectChat,
+    sendDirectChatStream,
+    sendMultimodalChat,
+    sendMultimodalChatStream,
+    testDirectConnection,
+} from './strategies';
 import { fetchModels } from './api';
 import { getStaticModels } from './models';
 
@@ -64,6 +82,17 @@ export interface ChannelManagerOptions {
     proxyUrl?: string;
 }
 
+/**
+ * 多模态聊天结果（通道管理器层）。
+ * 在底层 MultimodalChatResult 上附加 channelId 与 model，便于上层定位调用方。
+ */
+export interface MultimodalChatManagerResult extends MultimodalChatResult {
+    /** 调用方通道 ID */
+    channelId: string;
+    /** 实际请求的模型 ID */
+    model: string;
+}
+
 /** 通道查询过滤条件 */
 export interface ChannelFilter {
     /** 按 Provider ID 过滤 */
@@ -775,6 +804,165 @@ export class AIChannelManager {
         return chatResult;
     }
 
+    // ============ Multimodal Chat ============
+
+    /**
+     * 私有：根据聊天结果更新通道状态。复用 chatWith / chatWithStream 的逻辑。
+     */
+    private updateChannelStatusAfterChat(
+        channelId: string,
+        success: boolean,
+        message: string | undefined
+    ): void {
+        const index = this.channels.findIndex(c => c.id === channelId);
+        if (index < 0) return;
+        if (success) {
+            this.channels[index].lastUsedAt = Date.now();
+            this.channels[index].status = 'connected';
+            this.channels[index].lastError = undefined;
+        } else {
+            this.channels[index].status = 'error';
+            this.channels[index].lastError = message;
+        }
+        this.saveChannel(this.channels[index]);
+    }
+
+    /**
+     * 使用当前激活通道发送多模态聊天。
+     * messages 支持 string（纯文本）或 ContentPart[]（多模态）。
+     */
+    async chatMultimodal(
+        message: ChatMessage | ChatMessage[],
+        options?: {
+            messages?: ChatMessage[];
+            maxTokens?: number;
+            outputModalities?: Array<'text' | 'image'>;
+            temperature?: number;
+            timeoutMs?: number;
+        }
+    ): Promise<MultimodalChatManagerResult> {
+        const channel = this.getActiveChannel();
+        if (!channel) {
+            throw new Error('No active channel. Please set an active channel first.');
+        }
+        return this.chatMultimodalWith(channel.id, message, options);
+    }
+
+    /**
+     * 使用指定通道发送多模态聊天。
+     * 与 chatMultimodal 区别：可显式指定 channelId。
+     */
+    async chatMultimodalWith(
+        channelId: string,
+        message: ChatMessage | ChatMessage[],
+        options?: {
+            messages?: ChatMessage[];
+            maxTokens?: number;
+            outputModalities?: Array<'text' | 'image'>;
+            temperature?: number;
+            timeoutMs?: number;
+        }
+    ): Promise<MultimodalChatManagerResult> {
+        const channel = this.getChannel(channelId);
+        if (!channel) {
+            throw new Error(`Channel not found: ${channelId}`);
+        }
+        const provider = getProvider(channel.providerId);
+        if (!provider) {
+            throw new Error(`Provider not found: ${channel.providerId}`);
+        }
+        const messages: ChatMessage[] = options?.messages
+            || (Array.isArray(message) ? message : [message]);
+        const result = await sendMultimodalChat({
+            apiFormat: channel.apiFormat,
+            baseUrl: channel.baseUrl || provider.baseUrl,
+            apiKey: channel.apiKey,
+            model: channel.model,
+            messages,
+            maxTokens: options?.maxTokens,
+            outputModalities: options?.outputModalities,
+            temperature: options?.temperature,
+            timeoutMs: options?.timeoutMs,
+        });
+        const out: MultimodalChatManagerResult = {
+            ...result,
+            channelId: channel.id,
+            model: channel.model,
+        };
+        this.updateChannelStatusAfterChat(channel.id, result.success, result.message);
+        this.emit('chat', out);
+        return out;
+    }
+
+    /**
+     * 使用当前激活通道发送多模态流式聊天。
+     * onDelta 每次收到「累计完整状态」（{ text, images }）。
+     */
+    async chatMultimodalStream(
+        message: ChatMessage | ChatMessage[],
+        options: {
+            messages?: ChatMessage[];
+            maxTokens?: number;
+            outputModalities?: Array<'text' | 'image'>;
+            temperature?: number;
+            timeoutMs?: number;
+            onDelta: (state: MultimodalStreamDelta) => void;
+        }
+    ): Promise<MultimodalChatManagerResult> {
+        const channel = this.getActiveChannel();
+        if (!channel) {
+            throw new Error('No active channel. Please set an active channel first.');
+        }
+        return this.chatMultimodalStreamWith(channel.id, message, options);
+    }
+
+    /**
+     * 使用指定通道发送多模态流式聊天。
+     */
+    async chatMultimodalStreamWith(
+        channelId: string,
+        message: ChatMessage | ChatMessage[],
+        options: {
+            messages?: ChatMessage[];
+            maxTokens?: number;
+            outputModalities?: Array<'text' | 'image'>;
+            temperature?: number;
+            timeoutMs?: number;
+            onDelta: (state: MultimodalStreamDelta) => void;
+        }
+    ): Promise<MultimodalChatManagerResult> {
+        const channel = this.getChannel(channelId);
+        if (!channel) {
+            throw new Error(`Channel not found: ${channelId}`);
+        }
+        const provider = getProvider(channel.providerId);
+        if (!provider) {
+            throw new Error(`Provider not found: ${channel.providerId}`);
+        }
+        const messages: ChatMessage[] = options.messages
+            || (Array.isArray(message) ? message : [message]);
+        const result = await sendMultimodalChatStream({
+            apiFormat: channel.apiFormat,
+            baseUrl: channel.baseUrl || provider.baseUrl,
+            apiKey: channel.apiKey,
+            model: channel.model,
+            messages,
+            maxTokens: options.maxTokens,
+            outputModalities: options.outputModalities,
+            temperature: options.temperature,
+            timeoutMs: options.timeoutMs,
+            onDelta: options.onDelta,
+        });
+        const out: MultimodalChatManagerResult = {
+            ...result,
+            channelId: channel.id,
+            model: channel.model,
+        };
+        this.updateChannelStatusAfterChat(channel.id, result.success, result.message);
+        this.emit('chat', out);
+        return out;
+    }
+
     /** 测试指定通道的连接 */
     async testChannel(id: string): Promise<{ success: boolean; latencyMs?: number; message?: string }> {
         const channel = this.getChannel(id);

package/src/index.ts

@@ -27,6 +27,17 @@ export type {
     FetcherActionType,
     // Model capability
     ModelCapability,
+    // Multimodal types
+    ContentPart,
+    ChatMessage,
+    NormalizedPart,
+    NormalizedMessage,
+    ContentResponsePart,
+    ResponseMessage,
+    MultimodalChatOptions,
+    MultimodalChatStreamOptions,
+    MultimodalChatResult,
+    MultimodalStreamDelta,
 } from './types';
 
 export {
@@ -78,14 +89,20 @@ export {
     type ProviderStrategy,
     strategyRegistry,
     getStrategy,
+    guessMimeFromDataUri,
     sendDirectChat,
     sendDirectChatStream,
+    sendMultimodalChat,
+    sendMultimodalChatStream,
     testDirectConnection,
     type DirectChatOptions,
     type DirectChatStreamOptions,
     type DirectChatResult,
 } from './strategies';
 
+// Multimodal Normalize
+export { normalizeMessages, normalizePart, fetchRemoteToDataUri } from './multimodal-normalize';
+
 // Channel Manager (Headless)
 export {
     AIChannelManager,
@@ -93,5 +110,6 @@ export {
     type ChannelManagerOptions,
     type ChannelFilter,
     type ChatResult,
+    type MultimodalChatManagerResult,
     type ChannelEventType,
 } from './channel-manager';

package/src/multimodal-normalize.ts

@@ -0,0 +1,170 @@
+/**
+ * 多模态消息标准化层
+ * 把调用方传入的 ChatMessage[]（content: string | ContentPart[]）转换为各协议策略都能消费的 NormalizedMessage[]。
+ *
+ * 关键设计：
+ * - 文本直接透传
+ * - 图片/音频统一转换为 data URI（含 MIME 推断）
+ * - HTTP/HTTPS URL 远程拉取并转为 base64 data URI（失败时保留 URL 透传并在 meta 中标记）
+ * - 裸 base64 视为缺少 MIME，按输入块的 type 推测
+ */
+
+import type {
+    ChatMessage,
+    ContentPart,
+    NormalizedMessage,
+    NormalizedPart,
+} from './types';
+
+/**
+ * 把一个 ContentPart 转成 NormalizedPart。
+ * - text：透传
+ * - image_url / image / inline_data：归一化为 image + data URI
+ * - audio：归一化为 audio + data URI
+ */
+export async function normalizePart(p: ContentPart): Promise<NormalizedPart> {
+    // 纯文本块
+    if (p.type === 'text') {
+        return { type: 'text', data: p.text };
+    }
+
+    let dataUri: string;
+    let mime: string | undefined;
+    let meta: Record<string, unknown> | undefined;
+
+    // OpenAI 风格 image_url
+    if (p.type === 'image_url') {
+        dataUri = p.image_url.url;
+        if (p.image_url.detail) meta = { detail: p.image_url.detail };
+    }
+    // Anthropic 风格 image（base64 或 url）
+    else if (p.type === 'image') {
+        if (p.source.type === 'base64') {
+            dataUri = `data:${p.source.media_type};base64,${p.source.data}`;
+            mime = p.source.media_type;
+        } else {
+            // source.type === 'url'
+            dataUri = (p.source as { type: 'url'; url: string }).url;
+        }
+    }
+    // Gemini 风格 inline_data
+    else if (p.type === 'inline_data') {
+        dataUri = `data:${p.inline_data.mime_type};base64,${p.inline_data.data}`;
+        mime = p.inline_data.mime_type;
+    }
+    // OpenAI 风格 input_audio
+    else if (p.type === 'audio') {
+        dataUri = `data:audio/${p.input_audio.format};base64,${p.input_audio.data}`;
+        mime = `audio/${p.input_audio.format}`;
+    }
+    else {
+        throw new Error(`未知的 ContentPart 类型: ${(p as { type: string }).type}`);
+    }
+
+    // HTTP/HTTPS 远程 URL：尝试拉取并转换为 data URI
+    if (/^https?:\/\//i.test(dataUri)) {
+        const remote = await fetchRemoteToDataUri(dataUri);
+        return {
+            type: 'image',
+            data: remote.dataUri,
+            mimeType: remote.mimeType,
+            meta: { ...meta, remoteUrlOnly: remote.fallback ? dataUri : undefined },
+        };
+    }
+
+    // data URI：直接解析 MIME
+    if (/^data:/i.test(dataUri)) {
+        const m = /^data:([^;,]+)/.exec(dataUri);
+        return {
+            type: p.type === 'audio' ? 'audio' : 'image',
+            data: dataUri,
+            mimeType: mime || m?.[1],
+            meta,
+        };
+    }
+
+    // 兜底：当作裸 base64 + 推断 MIME
+    const fallbackMime =
+        mime || (p.type === 'audio' ? 'audio/mpeg' : 'image/png');
+    return {
+        type: p.type === 'audio' ? 'audio' : 'image',
+        data: `data:${fallbackMime};base64,${dataUri}`,
+        mimeType: fallbackMime,
+        meta,
+    };
+}
+
+/**
+ * 远程图片 → data URI。
+ * 拉取失败时返回原始 URL（标记 fallback=true），让协议策略自行决定是否接受 URL。
+ */
+export async function fetchRemoteToDataUri(
+    url: string
+): Promise<{ dataUri: string; mimeType?: string; fallback: boolean }> {
+    try {
+        const resp = await fetch(url);
+        if (!resp.ok) {
+            return { dataUri: url, fallback: true };
+        }
+        const mimeType = resp.headers.get('content-type') || undefined;
+        const buf = await resp.arrayBuffer();
+        // 在 Node 与浏览器中都能用的 base64 转换
+        const b64 = arrayBufferToBase64(buf);
+        const dataUri = `data:${mimeType || 'image/png'};base64,${b64}`;
+        return { dataUri, mimeType, fallback: false };
+    } catch {
+        return { dataUri: url, fallback: true };
+    }
+}
+
+/**
+ * ArrayBuffer → base64。Node 与浏览器通用。
+ */
+function arrayBufferToBase64(buf: ArrayBuffer): string {
+    // Node 环境：使用 globalThis 上的 Buffer（避免引入 @types/node）
+    const g = globalThis as unknown as {
+        Buffer?: { from(data: ArrayBuffer): { toString(encoding: string): string } };
+        btoa?: (data: string) => string;
+    };
+    if (g.Buffer && typeof g.Buffer.from === 'function') {
+        return g.Buffer.from(buf).toString('base64');
+    }
+    // 浏览器：用 btoa
+    let binary = '';
+    const bytes = new Uint8Array(buf);
+    const chunk = 0x8000;
+    for (let i = 0; i < bytes.length; i += chunk) {
+        binary += String.fromCharCode.apply(
+            null,
+            Array.from(bytes.subarray(i, i + chunk)) as number[]
+        );
+    }
+    return g.btoa ? g.btoa(binary) : binary;
+}
+
+/**
+ * 把 ChatMessage[] 转为 NormalizedMessage[]。
+ *
+ * 规则：
+ * - content: string → 一个 text part
+ * - content: ContentPart[] → 逐个 normalizePart
+ * - role: 'tool' → 在统一层视为 'user'（避免各策略重复处理 tool role）
+ */
+export async function normalizeMessages(
+    messages: ChatMessage[]
+): Promise<NormalizedMessage[]> {
+    const out: NormalizedMessage[] = [];
+    for (const m of messages) {
+        const role: NormalizedMessage['role'] = m.role === 'tool' ? 'user' : m.role;
+        if (typeof m.content === 'string') {
+            out.push({ role, parts: [{ type: 'text', data: m.content }] });
+            continue;
+        }
+        const parts: NormalizedPart[] = [];
+        for (const p of m.content) {
+            parts.push(await normalizePart(p));
+        }
+        out.push({ role, parts });
+    }
+    return out;
+}

package/src/providers.ts

@@ -133,7 +133,7 @@ export const PROVIDERS: Record<string, Provider> = {
         baseUrl: 'https://api.minimax.io/v1',
         needsApiKey: true,
         apiFormat: 'openai',
-        supportsModelsApi: false,
+        supportsModelsApi: true,
         icon: `${ICON_CDN_BASE}/minimax.svg`,
     },
     // MiniMax 中文站（同服务商，独立域名；模型列表与海外版一致）
@@ -143,7 +143,7 @@ export const PROVIDERS: Record<string, Provider> = {
         baseUrl: 'https://api.minimaxi.com/v1',
         needsApiKey: true,
         apiFormat: 'openai',
-        supportsModelsApi: false,
+        supportsModelsApi: true,
         icon: `${ICON_CDN_BASE}/minimax.svg`,
     },
     [PROVIDER_ID.XAI]: {