@mengruo/dsh-vision-toolkit 0.1.5 → 0.1.6-beta.0

package/src/runtime.ts

@@ -29,11 +29,23 @@ import {
   isWithin,
   resolveHtmlFile,
   resolveInputFile,
+  resolveInputVideo,
   resolveOutputDirectory,
   resolveOutputFile,
   seedStagedDirectory,
   type PathPolicy,
 } from './paths.ts'
+import {
+  extractChatAnswer,
+  ffprobeBinaryPath,
+  parseFfprobeOutput,
+  videoMediaType,
+  type VideoInfo,
+  type VideoInfoRequest,
+  type VideoMetadata,
+  type VideoUnderstandRequest,
+  type VideoUnderstandResult,
+} from './video.ts'
 import {
   parseCropOutput,
   parseDominantColorsOutput,
@@ -56,6 +68,8 @@ import { PLUGIN_VERSION } from './version.ts'
 const SVG_NAMESPACE = 'http://www.w3.org/2000/svg'
 const VISION_MODEL_TEST_IMAGE = fileURLToPath(new URL('../assets/vision-model-test.png', import.meta.url))
 const VISION_MODEL_TEST_PROMPT = 'This is an explicit service readiness test. Reply with one short sentence confirming that you received the image.'
+const VISION_MODEL_TEST_VIDEO = fileURLToPath(new URL('../assets/1.mp4', import.meta.url))
+const VISION_MODEL_TEST_VIDEO_PROMPT = '这个视频内容是什么？'
 
 /** Bump when the Pillow compression ladder changes so stale cache entries are ignored. */
 const COMPRESSED_IMAGE_CACHE_VERSION = 'v2'
@@ -813,6 +827,17 @@ export class VisionToolkitRuntime {
     return this.config.storageDir
   }
 
+  /**
+   * Whether at least one enabled OpenAI-compatible provider has video support
+   * turned on. A capability probe only: tool visibility is governed by the
+   * Settings tool-visibility video bucket, not by this flag. `videoUnderstand`
+   * re-checks it at call time and reports "video understanding unavailable"
+   * when it is false.
+   */
+  get videoSupportEnabled(): boolean {
+    return this.videoProvider() !== undefined
+  }
+
   /** Stable identity for persisted image descriptions produced by this runtime. */
   get evidenceFingerprint(): string {
     return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim())
@@ -970,6 +995,11 @@ export class VisionToolkitRuntime {
     return this.config.providers.find(provider => provider.enabled) ?? this.config.providers[0]!
   }
 
+  /** Highest-priority enabled OpenAI provider with video support enabled. */
+  private videoProvider(): ResolvedProvider | undefined {
+    return this.config.providers.find(provider => provider.enabled && provider.videoSupport && provider.protocol === 'openai')
+  }
+
   /** Build the upstream environment for one resolved provider. */
   private providerEnv(provider: ResolvedProvider, resolved: ResolvedCredential): UpstreamEnvironment {
     const sslVerify = process.env.VISION_SSL_VERIFY?.trim()
@@ -999,6 +1029,7 @@ export class VisionToolkitRuntime {
         userAgent: provider.userAgent,
         stream: provider.stream,
         uploadViaUrl: provider.uploadViaUrl,
+        videoSupport: provider.videoSupport,
       })
         ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
         : await this.ctx.credentials.resolve(provider.credential)
@@ -1346,6 +1377,171 @@ export class VisionToolkitRuntime {
     return client.test()
   }
 
+  /**
+   * Settings "test video call" probe: upload the bundled diagnostic video to
+   * object storage and send one OpenAI-compatible (Aliyun Qwen) video request.
+   * The wire content uses the Qwen video block shape (`video_url` + `fps`) plus
+   * the fixed test question. The object is deleted afterwards, best-effort.
+   */
+  async testVideoCall(options: ToolCallOptions, provider?: ResolvedProvider): Promise<{ detail: string }> {
+    return this.runOperation('vision_toolkit_video_test', options, async (operation) => {
+      const target = provider ?? this.primaryProvider
+      if (target.protocol !== 'openai') {
+        throw new VisionToolkitError('config', 'video support is only available for OpenAI-compatible providers')
+      }
+      const entry = await this.resolveProviderEnv(target)
+      if (entry === undefined) {
+        throw new VisionToolkitError('config', `credential ${String(target.credential)} is not configured`)
+      }
+      const client = await this.resolveObjectStorageClient()
+      if (client === undefined) {
+        throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)')
+      }
+      const uploaded = await client.uploadImage(VISION_MODEL_TEST_VIDEO, 'video/mp4')
+      try {
+        const answer = await this.requestVideoAnswer(target, entry.env.VISION_API_KEY, uploaded.url, VISION_MODEL_TEST_VIDEO_PROMPT, 2, operation)
+        const snippet = answer.trim().slice(0, 160)
+        return {
+          detail: snippet.length === 0
+            ? `Video call completed against ${target.model}`
+            : `Video call completed against ${target.model}: ${snippet}`,
+        }
+      } finally {
+        await client.deleteObject(uploaded.key)
+      }
+    })
+  }
+
+  /** Run the bundled ffprobe binary once and parse its JSON metadata. */
+  private async runFfprobe(videoPath: string, operation: OperationContext): Promise<VideoMetadata> {
+    const binary = ffprobeBinaryPath()
+    if (binary === null) {
+      throw new VisionToolkitError('runtime', 'ffprobe is unavailable; the ffprobe-static package must be installed alongside the plugin')
+    }
+    const handle = this.ctx.subprocess.spawn({
+      argv: [binary, '-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', videoPath],
+      cwd: process.cwd(),
+      stdio: {
+        stdin: 'ignore',
+        stdout: { maxBytes: 512 * 1024 },
+        stderr: { maxBytes: 64 * 1024 },
+      },
+      graceMs: 2000,
+      signal: operation.signal,
+    })
+    const outcome = await handle.done
+    const stdout = handle.collected.stdout?.readFrom(0)
+    const stderr = handle.collected.stderr?.readFrom(0)
+    if (outcome.exitCode !== 0) {
+      throw new VisionToolkitError('input', `cannot read video metadata: ${(stderr?.text ?? '').trim() || 'ffprobe failed'}`)
+    }
+    if (stdout?.lossy === true) {
+      throw new VisionToolkitError('output', 'video metadata output exceeded the capture limit')
+    }
+    try {
+      return parseFfprobeOutput(stdout?.text ?? '')
+    } catch (error) {
+      throw new VisionToolkitError('output', 'video metadata output is invalid', { cause: error })
+    }
+  }
+
+  /**
+   * One OpenAI-compatible (Aliyun Qwen) video chat-completions request. The
+   * content array carries the video_url block (with `fps`) plus the prompt text.
+   */
+  private async requestVideoAnswer(
+    target: ResolvedProvider,
+    apiKey: string,
+    videoUrl: string,
+    prompt: string,
+    fps: number,
+    operation: OperationContext,
+  ): Promise<string> {
+    operation.metrics.usedVisionService = true
+    const started = Date.now()
+    const response = await fetch(`${target.baseUrl}/chat/completions`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${apiKey}`,
+        'User-Agent': target.userAgent,
+      },
+      body: JSON.stringify({
+        model: target.model,
+        messages: [{
+          role: 'user',
+          content: [
+            { type: 'video_url', video_url: { url: videoUrl }, fps },
+            { type: 'text', text: prompt },
+          ],
+        }],
+      }),
+      signal: operation.signal,
+    })
+    operation.metrics.upstreamMs += Date.now() - started
+    const body = await response.text().catch(() => '')
+    if (!response.ok) {
+      throw new VisionToolkitError('service', `video request failed with HTTP ${response.status}: ${body.slice(0, 300)}`)
+    }
+    const answer = extractChatAnswer(body)
+    if (answer.trim().length === 0) {
+      throw new VisionToolkitError('output', 'vision API returned an empty answer')
+    }
+    return answer
+  }
+
+  /** Local video basic-info probe: ffprobe metadata, no API or credential. */
+  async videoInfo(request: VideoInfoRequest, options: ToolCallOptions): Promise<VideoInfo> {
+    return this.runOperation('vision_video_info', options, async (operation) => {
+      const policy = await this.pathPolicy(options.workspace)
+      const resolved = await resolveInputVideo(request.video, policy)
+      const metadata = await this.runFfprobe(resolved.path, operation)
+      return { path: resolved.path, bytes: resolved.bytes, ...metadata }
+    })
+  }
+
+  /**
+   * Video understanding: upload the video to object storage and send it plus a
+   * prompt to the first enabled OpenAI provider with video support. Whether the
+   * tool appears in an Agent is governed by the Settings tool-visibility video
+   * bucket, not by provider capability; this method re-checks capability at
+   * call time and returns a "video understanding unavailable" config error when
+   * no enabled provider supports video.
+   */
+  async videoUnderstand(request: VideoUnderstandRequest, options: ToolCallOptions): Promise<VideoUnderstandResult> {
+    return this.runOperation('vision_video_understand', options, async (operation) => {
+      const target = this.videoProvider()
+      if (target === undefined) {
+        throw new VisionToolkitError('config', 'no enabled vision service has video support; enable it in Settings first')
+      }
+      const entry = await this.resolveProviderEnv(target)
+      if (entry === undefined) {
+        throw new VisionToolkitError('config', `credential ${String(target.credential)} is not configured`)
+      }
+      const prompt = request.prompt.trim()
+      if (prompt.length === 0) {
+        throw new VisionToolkitError('input', 'prompt must not be empty')
+      }
+      const fps = request.fps ?? 2
+      if (!Number.isInteger(fps) || fps < 1 || fps > 120) {
+        throw new VisionToolkitError('input', 'fps must be an integer between 1 and 120')
+      }
+      const policy = await this.pathPolicy(options.workspace)
+      const resolved = await resolveInputVideo(request.video, policy)
+      const client = await this.resolveObjectStorageClient()
+      if (client === undefined) {
+        throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)')
+      }
+      const uploaded = await client.uploadImage(resolved.path, videoMediaType(extname(resolved.path).toLowerCase()))
+      try {
+        const answer = await this.requestVideoAnswer(target, entry.env.VISION_API_KEY, uploaded.url, prompt, fps, operation)
+        return { path: resolved.path, answer }
+      } finally {
+        await client.deleteObject(uploaded.key)
+      }
+    })
+  }
+
   /** Stable gate key for one provider's in-flight request cap. */
   private providerGate(provider: ResolvedProvider): Semaphore {
     const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`

package/src/tools.ts

@@ -22,6 +22,7 @@ import {
   type TraceRequest,
 } from './runtime.ts'
 import { platformTempDirectory } from './paths.ts'
+import type { VideoInfoRequest, VideoUnderstandRequest } from './video.ts'
 
 const renderJson = (_args: unknown, value: unknown): ContentBlock[] => [{
   type: 'text',
@@ -48,8 +49,55 @@ export const VISION_TOOL_NAMES = {
   dominantColors: 'vision_dominant_colors',
   htmlScreenshot: 'vision_html_screenshot',
   concurrency: 'vision_concurrency',
+  videoInfo: 'vision_video_info',
 } as const
 
+/**
+ * Opt-in video-understanding tool name. Deliberately outside {@link VISION_TOOL_NAMES}
+ * so the always-registered canonical set stays unconditional; this tool enters an
+ * Agent only when a vision service has video support enabled.
+ */
+export const VISION_VIDEO_UNDERSTAND_TOOL = 'vision_video_understand'
+
+/**
+ * Tool-group buckets used by the `toolVisibility` snapshot. A bucket that is
+ * off contributes none of its listed tools to an Agent's visible surface.
+ * - local: local-processing tools (no on-line fan-out, no concurrency charge).
+ * - online: on-line image tools plus the concurrency/status probe.
+ * - video: video-understanding tool (experimental).
+ */
+export interface ToolVisibility {
+  local: boolean
+  online: boolean
+  video: boolean
+}
+
+const TOOL_BUCKETS: Record<string, 'local' | 'online' | 'video'> = {
+  [VISION_TOOL_NAMES.glance]: 'online',
+  [VISION_TOOL_NAMES.ground]: 'online',
+  [VISION_TOOL_NAMES.detect]: 'online',
+  [VISION_TOOL_NAMES.longScreenshotOcr]: 'online',
+  [VISION_TOOL_NAMES.concurrency]: 'online',
+  [VISION_TOOL_NAMES.trace]: 'local',
+  [VISION_TOOL_NAMES.crop]: 'local',
+  [VISION_TOOL_NAMES.pixelDiff]: 'local',
+  [VISION_TOOL_NAMES.extractForeground]: 'local',
+  [VISION_TOOL_NAMES.dominantColors]: 'local',
+  [VISION_TOOL_NAMES.htmlScreenshot]: 'local',
+  [VISION_TOOL_NAMES.videoInfo]: 'local',
+  [VISION_VIDEO_UNDERSTAND_TOOL]: 'video',
+}
+
+/** Filter one tool definition by the visibility snapshot's matching bucket. */
+function visibleBySnapshot(definition: { name: string }, snapshot: ToolVisibility): boolean {
+  switch (TOOL_BUCKETS[definition.name]) {
+    case 'local': return snapshot.local
+    case 'online': return snapshot.online
+    case 'video': return snapshot.video
+    default: return true
+  }
+}
+
 /** Resolve the caller workspace exactly like first-party fs/bash tools. */
 function sessionWorkspace(exec: ToolRunContext): string {
   return exec.agent?.session.header.cwd ?? process.cwd()
@@ -122,6 +170,19 @@ const requiredBoxSchema = { ...boxSchema, required: true } as const
 const requiredImageInfoSchema = { ...imageInfoSchema, required: true } as const
 const requiredArtifactSchema = { ...artifactSchema, required: true } as const
 
+const nullableStringSchema = {
+  oneOf: [{ type: 'string' }, { type: 'null' }],
+} as const satisfies ValueSchemaSpec
+const nullableIntegerSchema = {
+  oneOf: [{ type: 'integer' }, { type: 'null' }],
+} as const satisfies ValueSchemaSpec
+const nullableNumberSchema = {
+  oneOf: [{ type: 'number' }, { type: 'null' }],
+} as const satisfies ValueSchemaSpec
+const requiredNullableStringSchema = { ...nullableStringSchema, required: true } as const
+const requiredNullableIntegerSchema = { ...nullableIntegerSchema, required: true } as const
+const requiredNullableNumberSchema = { ...nullableNumberSchema, required: true } as const
+
 const locatedMatchSchema = {
   type: 'object',
   additionalProperties: false,
@@ -181,17 +242,21 @@ function runtimeFrom(source: VisionToolkitRuntimeSource): VisionToolkitRuntime {
  * @param source - Current runtime or atomic runtime lookup.
  * @param projectPresentation - Browser-only projection for Artifact capabilities.
  * @param lifecycleSignal - Plugin lifetime; aborting it cancels every active tool call.
+ * @param toolVisibility - Session-head visibility snapshot; a bucket that is off
+ *   contributes none of its tools. Defaults to every bucket on.
  * @returns Native tool definitions registered as one lifecycle generation.
  */
 export function createVisionTools(
   source: VisionToolkitRuntimeSource,
   projectPresentation: VisionToolkitPresentationProjector = presentationIdentity,
   lifecycleSignal?: AbortSignal,
+  toolVisibility: ToolVisibility = { local: true, online: true, video: false },
 ): ReturnType<typeof defineTool>[] {
   const presentationMeta = (_args: unknown, value: JsonValue): JsonValue => projectPresentation(value)
-  const sessionMaxConcurrency = runtimeFrom(source).sessionMaxConcurrency
+  const runtime = runtimeFrom(source)
+  const sessionMaxConcurrency = runtime.sessionMaxConcurrency
   const concurrencyNote = `A single session runs at most ${sessionMaxConcurrency} concurrent vision calls; query vision_concurrency for the live available count. `
-  return [
+  const tools: ReturnType<typeof defineTool>[] = [
     defineTool({
       name: VISION_TOOL_NAMES.glance,
       description: 'Describe, answer a targeted question about, OCR, or compare one or more images with the configured vision model. '
@@ -591,7 +656,8 @@ export function createVisionTools(
     }),
     defineTool({
       name: VISION_TOOL_NAMES.concurrency,
-      description: 'Report the current available concurrency for vision tool calls in this session: the smaller of the remaining per-session slots and the total remaining model-request slots across enabled providers.',
+      description: 'Report the current available concurrency for ON-LINE vision tool calls (glance, ground, detect, long-screenshot OCR, video understanding) in this session: the smaller of the remaining per-session slots and the total remaining model-request slots across enabled providers. '
+        + 'Local-processing tools (trace, crop, pixel diff, extract foreground, colors, HTML screenshot, video info) are NOT throttled by this and run immediately.',
       parameters: {},
       output: {
         schema: {
@@ -621,7 +687,72 @@ export function createVisionTools(
       isConcurrencySafe: () => true,
       presentCall: () => ({ card: 'generic', title: 'Vision concurrency', kind: 'read', locations: [] }),
     }),
+    defineTool({
+      name: VISION_TOOL_NAMES.videoInfo,
+      description: 'Probe a local video file and return its basic metadata (container format, duration, resolution, frame rate, codecs, and stream counts) using the bundled ffprobe binary. '
+        + 'This is a local operation and never calls a vision API or needs a credential. ' + concurrencyNote + WORKSPACE_NOTE,
+      parameters: {
+        video: { type: 'string', required: true, description: 'Video file path.' },
+        timeoutSeconds: { type: 'integer', description: TIMEOUT_NOTE },
+      },
+      output: {
+        schema: {
+          type: 'object', additionalProperties: false, properties: {
+            path: { type: 'string', required: true },
+            bytes: { type: 'integer', required: true },
+            format: { type: 'string', required: true },
+            durationSeconds: requiredNullableNumberSchema,
+            width: requiredNullableIntegerSchema,
+            height: requiredNullableIntegerSchema,
+            frameRate: requiredNullableNumberSchema,
+            videoCodec: requiredNullableStringSchema,
+            audioCodec: requiredNullableStringSchema,
+            bitRate: requiredNullableIntegerSchema,
+            videoStreamCount: { type: 'integer', required: true },
+            audioStreamCount: { type: 'integer', required: true },
+          },
+        },
+        render: renderJson,
+      },
+      async execute(args: VideoInfoArgs, exec) {
+        const request: VideoInfoRequest = { video: args.video }
+        return runtimeFrom(source).videoInfo(request, callOptions(exec, args.timeoutSeconds, lifecycleSignal))
+      },
+      isConcurrencySafe: () => true,
+      presentCall: args => ({ card: 'generic', title: `Inspect ${args.video}`, kind: 'read', locations: [{ path: args.video }] }),
+    }),
+    defineTool({
+      name: VISION_VIDEO_UNDERSTAND_TOOL,
+      description: 'Send a local video plus a prompt to the configured vision service and return its answer text. '
+        + 'The video is uploaded to object storage and passed as a video_url block (Aliyun Qwen format). '
+        + `This tool exists only when the video tool bucket is enabled in Settings. Calling it without an enabled vision service that supports video returns a "video understanding unavailable" error. ${UNTRUSTED_EVIDENCE_NOTE} ` + concurrencyNote + WORKSPACE_NOTE,
+      parameters: {
+        video: { type: 'string', required: true, description: 'Video file path.' },
+        prompt: { type: 'string', required: true, description: 'Question or instruction about the video.' },
+        fps: { type: 'integer', description: 'Video sampling frame rate passed to the model; default 2.' },
+        timeoutSeconds: { type: 'integer', description: TIMEOUT_NOTE },
+      },
+      output: {
+        schema: {
+          type: 'object', additionalProperties: false, properties: {
+            path: { type: 'string', required: true },
+            answer: { type: 'string', required: true },
+          },
+        },
+        render: renderJson,
+      },
+      async execute(args: VideoUnderstandArgs, exec) {
+        const request: VideoUnderstandRequest = {
+          video: args.video,
+          prompt: args.prompt,
+          ...(args.fps === undefined ? {} : { fps: args.fps }),
+        }
+        return runtimeFrom(source).videoUnderstand(request, callOptions(exec, args.timeoutSeconds, lifecycleSignal))
+      },
+      presentCall: args => ({ card: 'generic', title: `Understand ${args.video}`, kind: 'read', locations: [{ path: args.video }] }),
+    }),
   ]
+  return tools.filter(definition => visibleBySnapshot(definition, toolVisibility))
 }
 
 interface GlanceArgs {
@@ -722,3 +853,13 @@ interface HtmlArgs {
   output?: string
   timeoutSeconds?: number
 }
+interface VideoInfoArgs {
+  video: string
+  timeoutSeconds?: number
+}
+interface VideoUnderstandArgs {
+  video: string
+  prompt: string
+  fps?: number
+  timeoutSeconds?: number
+}

package/src/video.ts

@@ -0,0 +1,222 @@
+/**
+ * Local video support built on the bundled `ffprobe-static` binary instead of
+ * the pinned Python vision pipeline: metadata probing stays a plain JS
+ * subprocess, and the model-facing video request reuses the existing object
+ * storage + OpenAI-compatible (Aliyun Qwen) chat-completions transport.
+ * @module dsh-vision-toolkit/video
+ */
+
+import { createRequire } from 'node:module'
+
+const require = createRequire(import.meta.url)
+
+/** Metadata derived from one ffprobe run (path/bytes are supplied by the runtime). */
+export interface VideoMetadata {
+  /** Container format name, e.g. "mov" or "matroska". */
+  format: string
+  /** Format duration in seconds, or null when the stream reports none. */
+  durationSeconds: number | null
+  /** Width of the first video stream, or null when the file carries no video. */
+  width: number | null
+  /** Height of the first video stream, or null when the file carries no video. */
+  height: number | null
+  /** Average frame rate of the first video stream, or null when unknown. */
+  frameRate: number | null
+  /** Codec of the first video stream, or null when the file carries no video. */
+  videoCodec: string | null
+  /** Codec of the first audio stream, or null when the file carries no audio. */
+  audioCodec: string | null
+  /** Format bit rate in bits per second, or null when unknown. */
+  bitRate: number | null
+  /** Number of video streams. */
+  videoStreamCount: number
+  /** Number of audio streams. */
+  audioStreamCount: number
+}
+
+/** Full structured result for one local video-info probe. */
+export interface VideoInfo extends VideoMetadata {
+  /** Fence-checked absolute input path. */
+  path: string
+  /** Input file size in bytes. */
+  bytes: number
+}
+
+/** Input for the local video-info tool (no API call). */
+export interface VideoInfoRequest {
+  video: string
+}
+
+/** Input for the video-understanding tool (video + prompt → vision API). */
+export interface VideoUnderstandRequest {
+  video: string
+  prompt: string
+  /** Video sampling frame rate passed to the model; default 2. */
+  fps?: number
+}
+
+/** Result of one video-understanding call. */
+export interface VideoUnderstandResult {
+  /** Fence-checked absolute input path. */
+  path: string
+  /** The vision model's answer text. */
+  answer: string
+}
+
+/**
+ * Resolve the bundled ffprobe executable. An explicit `FFPROBE_BIN` override
+ * wins (used by tests and unusual installs); otherwise the platform-correct
+ * binary shipped by `ffprobe-static` is returned. `null` means no binary is
+ * available and the caller must fail loud.
+ */
+export function ffprobeBinaryPath(): string | null {
+  const override = process.env.FFPROBE_BIN?.trim()
+  if (override !== undefined && override.length > 0) return override
+  try {
+    const mod = require('ffprobe-static') as { path?: unknown } | undefined
+    const candidate = mod?.path
+    return typeof candidate === 'string' && candidate.length > 0 ? candidate : null
+  } catch {
+    return null
+  }
+}
+
+/** S3 object Content-Type for one video extension (browser-probing convention). */
+export function videoMediaType(extension: string): string {
+  switch (extension) {
+    case '.mp4': return 'video/mp4'
+    case '.mov': return 'video/quicktime'
+    case '.webm': return 'video/webm'
+    case '.mkv': return 'video/x-matroska'
+    case '.avi': return 'video/x-msvideo'
+    case '.m4v': return 'video/x-m4v'
+    case '.mpeg':
+    case '.mpg': return 'video/mpeg'
+    case '.wmv': return 'video/x-ms-wmv'
+    case '.flv': return 'video/x-flv'
+    case '.ts':
+    case '.m2ts': return 'video/mp2t'
+    case '.3gp': return 'video/3gpp'
+    default: return 'application/octet-stream'
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+function stringValue(value: unknown): string | null {
+  return typeof value === 'string' && value.trim().length > 0 ? value : null
+}
+
+function integerValue(value: unknown): number | null {
+  return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : null
+}
+
+/** Parse "num/den" or a plain float string into a frame rate, else null. */
+function parseRate(raw: unknown): number | null {
+  const text = stringValue(raw)
+  if (text === null) return null
+  const slash = text.indexOf('/')
+  if (slash === -1) {
+    const value = Number(text)
+    return Number.isFinite(value) && value > 0 ? value : null
+  }
+  const numerator = Number(text.slice(0, slash))
+  const denominator = Number(text.slice(slash + 1))
+  if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator === 0) return null
+  const value = numerator / denominator
+  return Number.isFinite(value) && value > 0 ? value : null
+}
+
+/**
+ * Parse the JSON printed by `ffprobe -print_format json -show_format -show_streams`
+ * into a compact, model-facing metadata object. Missing or malformed fields
+ * degrade to null rather than throwing: metadata is advisory, and the probe
+ * only fails when the process itself failed or returned no JSON.
+ */
+export function parseFfprobeOutput(stdout: string): VideoMetadata {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(stdout)
+  } catch {
+    throw new Error('ffprobe returned invalid JSON')
+  }
+  if (!isRecord(parsed)) throw new Error('ffprobe returned an unexpected structure')
+
+  const streams = Array.isArray(parsed.streams) ? parsed.streams.filter(isRecord) : []
+  const format = isRecord(parsed.format) ? parsed.format : {}
+
+  let videoCodec: string | null = null
+  let audioCodec: string | null = null
+  let width: number | null = null
+  let height: number | null = null
+  let frameRate: number | null = null
+  let videoStreamCount = 0
+  let audioStreamCount = 0
+  for (const stream of streams) {
+    const codecType = stringValue(stream.codec_type)
+    const codecName = stringValue(stream.codec_name)
+    if (codecType === 'video') {
+      videoStreamCount += 1
+      if (videoCodec === null) {
+        videoCodec = codecName
+        width = integerValue(stream.width)
+        height = integerValue(stream.height)
+        frameRate = parseRate(stream.avg_frame_rate ?? stream.r_frame_rate)
+      }
+    } else if (codecType === 'audio') {
+      audioStreamCount += 1
+      if (audioCodec === null) audioCodec = codecName
+    }
+  }
+
+  const durationText = stringValue(format.duration)
+  const durationSeconds = durationText === null || durationText === 'N/A'
+    ? null
+    : (() => { const value = Number(durationText); return Number.isFinite(value) && value >= 0 ? value : null })()
+
+  const formatName = stringValue(format.format_name)?.split(',')[0]?.trim() ?? null
+
+  return {
+    format: formatName ?? 'unknown',
+    durationSeconds,
+    width,
+    height,
+    frameRate,
+    videoCodec,
+    audioCodec,
+    bitRate: integerValue(typeof format.bit_rate === 'string' ? Number(format.bit_rate) : format.bit_rate),
+    videoStreamCount,
+    audioStreamCount,
+  }
+}
+
+/**
+ * Extract the answer text from one OpenAI-compatible chat-completions response
+ * body. Handles both string content and array content (text parts joined).
+ * @returns the answer text, or an empty string when the shape is unrecognized.
+ */
+export function extractChatAnswer(body: string): string {
+  try {
+    const parsed = JSON.parse(body) as unknown
+    if (!isRecord(parsed)) return ''
+    const choices = parsed.choices
+    if (!Array.isArray(choices) || choices.length === 0) return ''
+    const first = choices[0]
+    if (!isRecord(first)) return ''
+    const message = first.message
+    if (!isRecord(message)) return ''
+    const content = message.content
+    if (typeof content === 'string') return content
+    if (Array.isArray(content)) {
+      return content
+        .filter(isRecord)
+        .map(part => (typeof part.text === 'string' ? part.text : ''))
+        .join('')
+    }
+    return ''
+  } catch {
+    return ''
+  }
+}