@idk500/video-vision-mcp 1.2.0

package/dist/hunyuan-client.js

@@ -0,0 +1,319 @@
+/**
+ * Vision + text client backed by an OpenAI-compatible endpoint.
+ *
+ * Originally this file targeted Tencent Cloud Hunyuan. It has been rewritten
+ * to call a generic OpenAI-compatible Chat Completions endpoint so any
+ * provider that accepts `image_url` content parts can serve as the "eyes" —
+ * by default Zhipu Bigmodel's free `glm-4.6v-flash` model.
+ *
+ * The class name (`HunyuanClient`) and its public method signatures are kept
+ * identical on purpose: the rest of the codebase (video-processor.ts,
+ * index.ts) depends only on this surface, so changing the backend does not
+ * require touching callers.
+ */
+import fs from 'fs/promises';
+import fetch from 'node-fetch';
+const DEFAULT_ENDPOINT = 'https://open.bigmodel.cn/api/paas/v4';
+const DEFAULT_VISION_MODEL = 'glm-4.6v-flash';
+const DEFAULT_TEXT_MODEL = 'glm-4.6';
+// Max image size accepted by most OpenAI-compatible vision endpoints (MB).
+const MAX_IMAGE_MB = 5;
+// 5xx / overload responses are retried with backoff.
+const MAX_RETRIES = 4;
+const RETRY_BASE_MS = 3000;
+export class HunyuanClient {
+    apiKey;
+    endpoint;
+    visionModel;
+    textModel;
+    constructor(config) {
+        // `secretId` carries the API key for the OpenAI-compatible backend.
+        this.apiKey = config?.secretId;
+        this.endpoint = (config?.endpoint || DEFAULT_ENDPOINT).replace(/\/+$/, '');
+        this.visionModel = config?.visionModel || DEFAULT_VISION_MODEL;
+        this.textModel = config?.textModel || DEFAULT_TEXT_MODEL;
+    }
+    /**
+     * Set credentials at runtime. For the OpenAI-compatible backend only the
+     * `secretId` (API key) is meaningful; `secretKey` is accepted for
+     * signature compatibility and ignored.
+     */
+    setCredentials(secretId, _secretKey) {
+        this.apiKey = secretId;
+    }
+    resolveApiKey(override) {
+        const key = override || this.apiKey || process.env.VISION_API_KEY;
+        if (!key) {
+            throw new Error([
+                '视觉模型 API Key 未配置。',
+                '',
+                '请通过以下任一方式提供：',
+                '1. 环境变量 VISION_API_KEY',
+                '2. 调用参数 apiKey / 历史字段 secretId',
+                '3. 启动参数（见 index.ts）',
+                '',
+                '默认后端：智谱 Bigmodel ' + DEFAULT_VISION_MODEL + '（OpenAI 兼容）。',
+                '获取 key：https://open.bigmodel.cn/usercenter/apikeys',
+            ].join('\n'));
+        }
+        return key;
+    }
+    resolveEndpoint() {
+        return (process.env.VISION_ENDPOINT || this.endpoint).replace(/\/+$/, '');
+    }
+    resolveVisionModel(override) {
+        return process.env.VISION_MODEL || override || this.visionModel;
+    }
+    resolveTextModel(override) {
+        return process.env.TEXT_MODEL || override || this.textModel;
+    }
+    /** Map a file extension to its MIME type, defaulting to jpeg. */
+    mimeFromExt(imagePath) {
+        const ext = imagePath.toLowerCase().split('.').pop();
+        switch (ext) {
+            case 'png':
+                return 'image/png';
+            case 'webp':
+                return 'image/webp';
+            case 'bmp':
+                return 'image/bmp';
+            case 'gif':
+                return 'image/gif';
+            case 'jpg':
+            case 'jpeg':
+            default:
+                return 'image/jpeg';
+        }
+    }
+    /**
+     * Read an image and return a base64 string suitable for `image_url.url`.
+     * Returns a bare base64 string (no `data:` prefix) — this is what the
+     * Zhipu Bigmodel endpoint expects; an explicit `data:` URL is *not* added
+     * here because the backend rejects it for the glm-4.6v family.
+     */
+    async imageToBase64(imagePath) {
+        try {
+            await fs.access(imagePath);
+        }
+        catch {
+            throw new Error(`图片文件不存在或无法访问: ${imagePath}`);
+        }
+        const buf = await fs.readFile(imagePath);
+        const sizeMB = buf.length / (1024 * 1024);
+        if (sizeMB > MAX_IMAGE_MB) {
+            throw new Error(`图片文件过大 (${sizeMB.toFixed(2)}MB)，请使用小于 ${MAX_IMAGE_MB}MB 的图片。文件: ${imagePath}`);
+        }
+        // Read the file but only use its MIME for diagnostics; the endpoint takes
+        // raw base64 and infers the format itself.
+        const mimeType = this.mimeFromExt(imagePath);
+        console.error(`图片 ${imagePath} 转换完成 - 大小: ${sizeMB.toFixed(2)}MB, 格式: ${mimeType}`);
+        return buf.toString('base64');
+    }
+    /** Sleep helper for retry backoff. */
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    /**
+     * POST to /chat/completions with retry on 429/5xx. The Zhipu free vision
+     * model throttles aggressively under load, so callers benefit from backoff.
+     */
+    async chatCompletions(body, apiKey, endpoint) {
+        let lastError = null;
+        for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+            let response;
+            try {
+                response = await fetch(`${endpoint}/chat/completions`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        Authorization: `Bearer ${apiKey}`,
+                    },
+                    body: JSON.stringify(body),
+                });
+            }
+            catch (err) {
+                lastError = err instanceof Error ? err : new Error(String(err));
+                console.error(`网络请求失败 (尝试 ${attempt}/${MAX_RETRIES}): ${lastError.message}`);
+                if (attempt < MAX_RETRIES) {
+                    await this.sleep(RETRY_BASE_MS * attempt);
+                    continue;
+                }
+                throw new Error(`视觉模型网络请求失败 (${endpoint}): ${lastError.message}\n请检查网络连接和端点配置。`);
+            }
+            // Throttled — back off and retry.
+            if (response.status === 429) {
+                const wait = RETRY_BASE_MS * attempt;
+                console.error(`模型访问量过大 (429)，${wait}ms 后重试 (${attempt}/${MAX_RETRIES})`);
+                lastError = new Error('模型当前访问量过大，请稍后再试');
+                if (attempt < MAX_RETRIES) {
+                    await this.sleep(wait);
+                    continue;
+                }
+            }
+            // Transient server error — back off and retry.
+            if (response.status >= 500) {
+                const wait = RETRY_BASE_MS * attempt;
+                const errText = await response.text().catch(() => '');
+                console.error(`服务端错误 (HTTP ${response.status})，${wait}ms 后重试 (${attempt}/${MAX_RETRIES}): ${errText.slice(0, 120)}`);
+                lastError = new Error(`视觉模型服务端错误 (HTTP ${response.status})`);
+                if (attempt < MAX_RETRIES) {
+                    await this.sleep(wait);
+                    continue;
+                }
+            }
+            const raw = await response.text();
+            if (!response.ok) {
+                // Non-retriable error: surface a readable message.
+                let detail = raw;
+                try {
+                    const parsed = JSON.parse(raw);
+                    detail = parsed.error?.message || parsed.message || raw;
+                }
+                catch {
+                    /* keep raw */
+                }
+                throw new Error(`视觉模型请求失败 (HTTP ${response.status} ${response.statusText}):\n${detail}\n\n请检查:\n1. API Key 是否正确且未过期\n2. 模型名称是否正确 (视觉: ${body.model})\n3. 端点配置 (${endpoint})\n4. 账户余额/套餐是否包含该模型`);
+            }
+            let parsed;
+            try {
+                parsed = JSON.parse(raw);
+            }
+            catch {
+                throw new Error(`视觉模型返回了非 JSON 响应: ${raw.slice(0, 200)}`);
+            }
+            if (parsed.error) {
+                throw new Error(`视觉模型错误: ${JSON.stringify(parsed.error)}`);
+            }
+            return parsed;
+        }
+        throw lastError || new Error('视觉模型请求失败');
+    }
+    /**
+     * Analyze a single image. The backend is the OpenAI-compatible Chat
+     * Completions API with an `image_url` content part.
+     */
+    async analyzeImage(imagePath, prompt = '请描述这张图片的内容。', apiKeyOverride) {
+        console.error(`开始分析图片: ${imagePath}`);
+        const apiKey = this.resolveApiKey(apiKeyOverride);
+        const endpoint = this.resolveEndpoint();
+        const model = this.resolveVisionModel();
+        const base64 = await this.imageToBase64(imagePath);
+        const result = await this.chatCompletions({
+            model,
+            // Disable reasoning/thinking so we get a direct answer cheaply.
+            thinking: { type: 'disabled' },
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        { type: 'text', text: prompt },
+                        { type: 'image_url', image_url: { url: base64 } },
+                    ],
+                },
+            ],
+            max_tokens: 1024,
+        }, apiKey, endpoint);
+        const content = result.choices?.[0]?.message?.content ?? '';
+        const usage = result.usage ?? {};
+        const out = {
+            content,
+            usage: {
+                promptTokens: usage.prompt_tokens ?? 0,
+                completionTokens: usage.completion_tokens ?? 0,
+                totalTokens: usage.total_tokens ?? 0,
+            },
+        };
+        console.error(`图片分析完成 - Token 使用: ${out.usage.totalTokens} (提示: ${out.usage.promptTokens}, 回复: ${out.usage.completionTokens})`);
+        return out;
+    }
+    /**
+     * Analyze each image in sequence. Serial processing keeps us under the free
+     * model's QPS limit; failed images do not abort the batch.
+     */
+    async analyzeImageBatch(imagePaths, prompt = '请描述这张图片的内容。', apiKeyOverride) {
+        const results = [];
+        const total = imagePaths.length;
+        console.error(`开始批量分析 ${total} 张图片`);
+        for (let i = 0; i < total; i++) {
+            const imagePath = imagePaths[i];
+            try {
+                console.error(`分析进度: ${i + 1}/${total} - ${imagePath}`);
+                const result = await this.analyzeImage(imagePath, prompt, apiKeyOverride);
+                results.push(result);
+                // Throttle to respect the free model's rate limit between images.
+                if (i < total - 1) {
+                    await this.sleep(1200);
+                }
+            }
+            catch (error) {
+                const msg = error instanceof Error ? error.message : String(error);
+                console.error(`图片分析失败 ${imagePath}:`, msg);
+                results.push({
+                    content: `❌ 分析失败: ${msg}`,
+                    usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
+                });
+                console.error(`跳过失败的图片，继续处理剩余图片...`);
+            }
+        }
+        const success = results.filter((r) => !r.content.startsWith('❌')).length;
+        const totalTokens = results.reduce((s, r) => s + r.usage.totalTokens, 0);
+        console.error(`批量分析完成 - 成功: ${success}/${total}, 总 Token: ${totalTokens}`);
+        return results;
+    }
+    /**
+     * Analyze multiple images in a single request. Useful for video frames so
+     * the model can reason about sequence/context. Limited to 4 images to keep
+     * request size and cost bounded (matches the original behavior).
+     */
+    async analyzeImagesInSingleRequest(imagePaths, prompt = '请描述这些图片的内容。', apiKeyOverride) {
+        const apiKey = this.resolveApiKey(apiKeyOverride);
+        const endpoint = this.resolveEndpoint();
+        const model = this.resolveVisionModel();
+        const limited = imagePaths.slice(0, 4);
+        const images = await Promise.all(limited.map((p) => this.imageToBase64(p)));
+        const content = [{ type: 'text', text: prompt }];
+        for (const b64 of images) {
+            content.push({ type: 'image_url', image_url: { url: b64 } });
+        }
+        const result = await this.chatCompletions({
+            model,
+            thinking: { type: 'disabled' },
+            messages: [{ role: 'user', content }],
+            max_tokens: 2048,
+        }, apiKey, endpoint);
+        const out = {
+            content: result.choices?.[0]?.message?.content ?? '',
+            usage: {
+                promptTokens: result.usage?.prompt_tokens ?? 0,
+                completionTokens: result.usage?.completion_tokens ?? 0,
+                totalTokens: result.usage?.total_tokens ?? 0,
+            },
+        };
+        return out;
+    }
+    /**
+     * Generate text from a prompt. Used for the second pass of script
+     * generation, where we don't need vision — a plain text model is cheaper.
+     */
+    async generateText(prompt, modelOverride, apiKeyOverride) {
+        const apiKey = this.resolveApiKey(apiKeyOverride);
+        const endpoint = this.resolveEndpoint();
+        const model = this.resolveTextModel(modelOverride);
+        console.error(`开始生成文本内容，使用模型: ${model}`);
+        const result = await this.chatCompletions({
+            model,
+            messages: [{ role: 'user', content: prompt }],
+            max_tokens: 4096,
+        }, apiKey, endpoint);
+        const out = {
+            content: result.choices?.[0]?.message?.content ?? '',
+            usage: {
+                promptTokens: result.usage?.prompt_tokens ?? 0,
+                completionTokens: result.usage?.completion_tokens ?? 0,
+                totalTokens: result.usage?.total_tokens ?? 0,
+            },
+        };
+        console.error(`文本生成完成 - Token 使用: ${out.usage.totalTokens} (提示: ${out.usage.promptTokens}, 回复: ${out.usage.completionTokens})`);
+        return out;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+declare class VideoMCPServer {
+    private server;
+    private videoProcessor;
+    private frameExtractor;
+    private secretId?;
+    private secretKey?;
+    private region?;
+    constructor(options?: {
+        secretId?: string;
+        secretKey?: string;
+        region?: string;
+    });
+    private setupToolHandlers;
+    private formatError;
+    private handleExtractFrames;
+    private handleAnalyzeVideo;
+    private handleAnalyzeImageBatch;
+    private handleGetVideoInfo;
+    private handleGenerateVideoScript;
+    private handleGenerateImageScript;
+    run(): Promise<void>;
+}
+export { VideoMCPServer };