@mengruo/dsh-vision-toolkit 0.1.5 → 0.1.6-beta.0

package/lib/runtime.js

@@ -16,12 +16,15 @@ import { BUILT_IN_FREE_VISION_KEY } from "./defaults.js";
 import { evidenceRuntimeFingerprint } from "./evidence-cache.js";
 import { VisionToolkitError } from "./errors.js";
 import { ObjectStorageClient, splitObjectStorageCredential } from "./object-storage.js";
-import { assertDistinctOutput, commitStagedDirectory, commitStagedOutput, createPathPolicy, createStagedDirectory, createStagedOutput, isWithin, resolveHtmlFile, resolveInputFile, resolveOutputDirectory, resolveOutputFile, seedStagedDirectory, } from "./paths.js";
+import { assertDistinctOutput, commitStagedDirectory, commitStagedOutput, createPathPolicy, createStagedDirectory, createStagedOutput, isWithin, resolveHtmlFile, resolveInputFile, resolveInputVideo, resolveOutputDirectory, resolveOutputFile, seedStagedDirectory, } from "./paths.js";
+import { extractChatAnswer, ffprobeBinaryPath, parseFfprobeOutput, videoMediaType, } from "./video.js";
 import { parseCropOutput, parseDominantColorsOutput, parseExtractForegroundOutput, parseHtmlScreenshotOutput, parseLocationOutput, parsePixelDiffOutput, parseTraceOutput, UpstreamAdapter, } from "./upstream.js";
 import { PLUGIN_VERSION } from "./version.js";
 const SVG_NAMESPACE = 'http://www.w3.org/2000/svg';
 const VISION_MODEL_TEST_IMAGE = fileURLToPath(new URL('../assets/vision-model-test.png', import.meta.url));
 const VISION_MODEL_TEST_PROMPT = 'This is an explicit service readiness test. Reply with one short sentence confirming that you received the image.';
+const VISION_MODEL_TEST_VIDEO = fileURLToPath(new URL('../assets/1.mp4', import.meta.url));
+const VISION_MODEL_TEST_VIDEO_PROMPT = '这个视频内容是什么？';
 /** Bump when the Pillow compression ladder changes so stale cache entries are ignored. */
 const COMPRESSED_IMAGE_CACHE_VERSION = 'v2';
 /** Cache keys carry 64-bit digests so Windows paths stay below MAX_PATH; the full file sha256 is computed on read and compared against this prefix. */
@@ -372,6 +375,16 @@ export class VisionToolkitRuntime {
     get storageDirectory() {
         return this.config.storageDir;
     }
+    /**
+     * Whether at least one enabled OpenAI-compatible provider has video support
+     * turned on. A capability probe only: tool visibility is governed by the
+     * Settings tool-visibility video bucket, not by this flag. `videoUnderstand`
+     * re-checks it at call time and reports "video understanding unavailable"
+     * when it is false.
+     */
+    get videoSupportEnabled() {
+        return this.videoProvider() !== undefined;
+    }
     /** Stable identity for persisted image descriptions produced by this runtime. */
     get evidenceFingerprint() {
         return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim());
@@ -492,6 +505,10 @@ export class VisionToolkitRuntime {
     get primaryProvider() {
         return this.config.providers.find(provider => provider.enabled) ?? this.config.providers[0];
     }
+    /** Highest-priority enabled OpenAI provider with video support enabled. */
+    videoProvider() {
+        return this.config.providers.find(provider => provider.enabled && provider.videoSupport && provider.protocol === 'openai');
+    }
     /** Build the upstream environment for one resolved provider. */
     providerEnv(provider, resolved) {
         const sslVerify = process.env.VISION_SSL_VERIFY?.trim();
@@ -520,6 +537,7 @@ export class VisionToolkitRuntime {
                 userAgent: provider.userAgent,
                 stream: provider.stream,
                 uploadViaUrl: provider.uploadViaUrl,
+                videoSupport: provider.videoSupport,
             })
                 ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
                 : await this.ctx.credentials.resolve(provider.credential);
@@ -850,6 +868,162 @@ export class VisionToolkitRuntime {
         }
         return client.test();
     }
+    /**
+     * Settings "test video call" probe: upload the bundled diagnostic video to
+     * object storage and send one OpenAI-compatible (Aliyun Qwen) video request.
+     * The wire content uses the Qwen video block shape (`video_url` + `fps`) plus
+     * the fixed test question. The object is deleted afterwards, best-effort.
+     */
+    async testVideoCall(options, provider) {
+        return this.runOperation('vision_toolkit_video_test', options, async (operation) => {
+            const target = provider ?? this.primaryProvider;
+            if (target.protocol !== 'openai') {
+                throw new VisionToolkitError('config', 'video support is only available for OpenAI-compatible providers');
+            }
+            const entry = await this.resolveProviderEnv(target);
+            if (entry === undefined) {
+                throw new VisionToolkitError('config', `credential ${String(target.credential)} is not configured`);
+            }
+            const client = await this.resolveObjectStorageClient();
+            if (client === undefined) {
+                throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)');
+            }
+            const uploaded = await client.uploadImage(VISION_MODEL_TEST_VIDEO, 'video/mp4');
+            try {
+                const answer = await this.requestVideoAnswer(target, entry.env.VISION_API_KEY, uploaded.url, VISION_MODEL_TEST_VIDEO_PROMPT, 2, operation);
+                const snippet = answer.trim().slice(0, 160);
+                return {
+                    detail: snippet.length === 0
+                        ? `Video call completed against ${target.model}`
+                        : `Video call completed against ${target.model}: ${snippet}`,
+                };
+            }
+            finally {
+                await client.deleteObject(uploaded.key);
+            }
+        });
+    }
+    /** Run the bundled ffprobe binary once and parse its JSON metadata. */
+    async runFfprobe(videoPath, operation) {
+        const binary = ffprobeBinaryPath();
+        if (binary === null) {
+            throw new VisionToolkitError('runtime', 'ffprobe is unavailable; the ffprobe-static package must be installed alongside the plugin');
+        }
+        const handle = this.ctx.subprocess.spawn({
+            argv: [binary, '-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', videoPath],
+            cwd: process.cwd(),
+            stdio: {
+                stdin: 'ignore',
+                stdout: { maxBytes: 512 * 1024 },
+                stderr: { maxBytes: 64 * 1024 },
+            },
+            graceMs: 2000,
+            signal: operation.signal,
+        });
+        const outcome = await handle.done;
+        const stdout = handle.collected.stdout?.readFrom(0);
+        const stderr = handle.collected.stderr?.readFrom(0);
+        if (outcome.exitCode !== 0) {
+            throw new VisionToolkitError('input', `cannot read video metadata: ${(stderr?.text ?? '').trim() || 'ffprobe failed'}`);
+        }
+        if (stdout?.lossy === true) {
+            throw new VisionToolkitError('output', 'video metadata output exceeded the capture limit');
+        }
+        try {
+            return parseFfprobeOutput(stdout?.text ?? '');
+        }
+        catch (error) {
+            throw new VisionToolkitError('output', 'video metadata output is invalid', { cause: error });
+        }
+    }
+    /**
+     * One OpenAI-compatible (Aliyun Qwen) video chat-completions request. The
+     * content array carries the video_url block (with `fps`) plus the prompt text.
+     */
+    async requestVideoAnswer(target, apiKey, videoUrl, prompt, fps, operation) {
+        operation.metrics.usedVisionService = true;
+        const started = Date.now();
+        const response = await fetch(`${target.baseUrl}/chat/completions`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${apiKey}`,
+                'User-Agent': target.userAgent,
+            },
+            body: JSON.stringify({
+                model: target.model,
+                messages: [{
+                        role: 'user',
+                        content: [
+                            { type: 'video_url', video_url: { url: videoUrl }, fps },
+                            { type: 'text', text: prompt },
+                        ],
+                    }],
+            }),
+            signal: operation.signal,
+        });
+        operation.metrics.upstreamMs += Date.now() - started;
+        const body = await response.text().catch(() => '');
+        if (!response.ok) {
+            throw new VisionToolkitError('service', `video request failed with HTTP ${response.status}: ${body.slice(0, 300)}`);
+        }
+        const answer = extractChatAnswer(body);
+        if (answer.trim().length === 0) {
+            throw new VisionToolkitError('output', 'vision API returned an empty answer');
+        }
+        return answer;
+    }
+    /** Local video basic-info probe: ffprobe metadata, no API or credential. */
+    async videoInfo(request, options) {
+        return this.runOperation('vision_video_info', options, async (operation) => {
+            const policy = await this.pathPolicy(options.workspace);
+            const resolved = await resolveInputVideo(request.video, policy);
+            const metadata = await this.runFfprobe(resolved.path, operation);
+            return { path: resolved.path, bytes: resolved.bytes, ...metadata };
+        });
+    }
+    /**
+     * Video understanding: upload the video to object storage and send it plus a
+     * prompt to the first enabled OpenAI provider with video support. Whether the
+     * tool appears in an Agent is governed by the Settings tool-visibility video
+     * bucket, not by provider capability; this method re-checks capability at
+     * call time and returns a "video understanding unavailable" config error when
+     * no enabled provider supports video.
+     */
+    async videoUnderstand(request, options) {
+        return this.runOperation('vision_video_understand', options, async (operation) => {
+            const target = this.videoProvider();
+            if (target === undefined) {
+                throw new VisionToolkitError('config', 'no enabled vision service has video support; enable it in Settings first');
+            }
+            const entry = await this.resolveProviderEnv(target);
+            if (entry === undefined) {
+                throw new VisionToolkitError('config', `credential ${String(target.credential)} is not configured`);
+            }
+            const prompt = request.prompt.trim();
+            if (prompt.length === 0) {
+                throw new VisionToolkitError('input', 'prompt must not be empty');
+            }
+            const fps = request.fps ?? 2;
+            if (!Number.isInteger(fps) || fps < 1 || fps > 120) {
+                throw new VisionToolkitError('input', 'fps must be an integer between 1 and 120');
+            }
+            const policy = await this.pathPolicy(options.workspace);
+            const resolved = await resolveInputVideo(request.video, policy);
+            const client = await this.resolveObjectStorageClient();
+            if (client === undefined) {
+                throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)');
+            }
+            const uploaded = await client.uploadImage(resolved.path, videoMediaType(extname(resolved.path).toLowerCase()));
+            try {
+                const answer = await this.requestVideoAnswer(target, entry.env.VISION_API_KEY, uploaded.url, prompt, fps, operation);
+                return { path: resolved.path, answer };
+            }
+            finally {
+                await client.deleteObject(uploaded.key);
+            }
+        });
+    }
     /** Stable gate key for one provider's in-flight request cap. */
     providerGate(provider) {
         const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`;