@mengruo/dsh-vision-toolkit 0.1.4 → 0.1.6-beta.0

package/src/runtime.ts

@@ -18,6 +18,7 @@ import { isBuiltInFreeVisionProvider, type ResolvedProvider, type ResolvedVision
 import { BUILT_IN_FREE_VISION_KEY } from './defaults.ts'
 import { evidenceRuntimeFingerprint } from './evidence-cache.ts'
 import { VisionToolkitError, type VisionToolkitErrorCode } from './errors.ts'
+import { ObjectStorageClient, splitObjectStorageCredential, type ObjectStorageSettings } from './object-storage.ts'
 import {
   assertDistinctOutput,
   commitStagedDirectory,
@@ -28,11 +29,23 @@ import {
   isWithin,
   resolveHtmlFile,
   resolveInputFile,
+  resolveInputVideo,
   resolveOutputDirectory,
   resolveOutputFile,
   seedStagedDirectory,
   type PathPolicy,
 } from './paths.ts'
+import {
+  extractChatAnswer,
+  ffprobeBinaryPath,
+  parseFfprobeOutput,
+  videoMediaType,
+  type VideoInfo,
+  type VideoInfoRequest,
+  type VideoMetadata,
+  type VideoUnderstandRequest,
+  type VideoUnderstandResult,
+} from './video.ts'
 import {
   parseCropOutput,
   parseDominantColorsOutput,
@@ -55,6 +68,8 @@ import { PLUGIN_VERSION } from './version.ts'
 const SVG_NAMESPACE = 'http://www.w3.org/2000/svg'
 const VISION_MODEL_TEST_IMAGE = fileURLToPath(new URL('../assets/vision-model-test.png', import.meta.url))
 const VISION_MODEL_TEST_PROMPT = 'This is an explicit service readiness test. Reply with one short sentence confirming that you received the image.'
+const VISION_MODEL_TEST_VIDEO = fileURLToPath(new URL('../assets/1.mp4', import.meta.url))
+const VISION_MODEL_TEST_VIDEO_PROMPT = '这个视频内容是什么？'
 
 /** Bump when the Pillow compression ladder changes so stale cache entries are ignored. */
 const COMPRESSED_IMAGE_CACHE_VERSION = 'v2'
@@ -540,6 +555,17 @@ const FORMAT_BY_EXTENSION = new Map([
 ])
 const HEX_COLOR_PATTERN = /^#[0-9A-F]{6}$/
 
+/** MIME type for one analyzed image format, used when uploading to object storage. */
+function imageMimeType(format: string): string {
+  switch (format) {
+    case 'png': return 'image/png'
+    case 'jpeg': return 'image/jpeg'
+    case 'gif': return 'image/gif'
+    case 'webp': return 'image/webp'
+    default: return 'application/octet-stream'
+  }
+}
+
 /**
  * Error codes a provider retries against the SAME provider within its
  * `attempts` budget. Only transient failures are worth re-requesting: a
@@ -801,6 +827,17 @@ export class VisionToolkitRuntime {
     return this.config.storageDir
   }
 
+  /**
+   * Whether at least one enabled OpenAI-compatible provider has video support
+   * turned on. A capability probe only: tool visibility is governed by the
+   * Settings tool-visibility video bucket, not by this flag. `videoUnderstand`
+   * re-checks it at call time and reports "video understanding unavailable"
+   * when it is false.
+   */
+  get videoSupportEnabled(): boolean {
+    return this.videoProvider() !== undefined
+  }
+
   /** Stable identity for persisted image descriptions produced by this runtime. */
   get evidenceFingerprint(): string {
     return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim())
@@ -958,6 +995,11 @@ export class VisionToolkitRuntime {
     return this.config.providers.find(provider => provider.enabled) ?? this.config.providers[0]!
   }
 
+  /** Highest-priority enabled OpenAI provider with video support enabled. */
+  private videoProvider(): ResolvedProvider | undefined {
+    return this.config.providers.find(provider => provider.enabled && provider.videoSupport && provider.protocol === 'openai')
+  }
+
   /** Build the upstream environment for one resolved provider. */
   private providerEnv(provider: ResolvedProvider, resolved: ResolvedCredential): UpstreamEnvironment {
     const sslVerify = process.env.VISION_SSL_VERIFY?.trim()
@@ -968,6 +1010,7 @@ export class VisionToolkitRuntime {
       VISION_API_PROTOCOL: provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
       VISION_ANTHROPIC_THINKING: provider.anthropicThinking,
       ...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
+      ...(provider.stream ? { VISION_STREAM: '1' } : {}),
       VISION_USER_AGENT: provider.userAgent,
       LANG: this.config.language,
     }
@@ -984,6 +1027,9 @@ export class VisionToolkitRuntime {
         protocol: provider.protocol,
         anthropicThinking: provider.anthropicThinking,
         userAgent: provider.userAgent,
+        stream: provider.stream,
+        uploadViaUrl: provider.uploadViaUrl,
+        videoSupport: provider.videoSupport,
       })
         ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
         : await this.ctx.credentials.resolve(provider.credential)
@@ -1253,6 +1299,249 @@ export class VisionToolkitRuntime {
     operation.metrics.imagePixels += image.width * image.height
   }
 
+  /** Resolve the configured object storage into a usable client, or undefined. */
+  private async resolveObjectStorageClient(): Promise<ObjectStorageClient | undefined> {
+    const objectStorage = this.config.objectStorage
+    if (objectStorage.credential === undefined || objectStorage.endpoint.length === 0 || objectStorage.bucket.length === 0) {
+      return undefined
+    }
+    let resolved: ResolvedCredential | undefined
+    try {
+      resolved = await this.ctx.credentials.resolve(objectStorage.credential)
+    } catch {
+      resolved = undefined
+    }
+    if (resolved === undefined) return undefined
+    let accessKeyId: string
+    let secretAccessKey: string
+    try {
+      const split = splitObjectStorageCredential(resolved.value)
+      accessKeyId = split.accessKeyId
+      secretAccessKey = split.secretAccessKey
+    } catch (error) {
+      throw new VisionToolkitError('config', 'object storage credential is malformed', { cause: error })
+    }
+    const settings: ObjectStorageSettings = {
+      endpoint: objectStorage.endpoint,
+      bucket: objectStorage.bucket,
+      accessKeyId,
+      secretAccessKey,
+      ...(objectStorage.publicBase === undefined ? {} : { publicBase: objectStorage.publicBase }),
+    }
+    return new ObjectStorageClient(settings)
+  }
+
+  /**
+   * Upload the prepared images to object storage and return their URLs plus a
+   * cleanup callback, when the primary provider opts into URL transfer and
+   * object storage is configured. Returns undefined otherwise (base64 path).
+   */
+  private async maybeTransferImages(
+    pool: readonly ResolvedProviderEnv[],
+    images: readonly ImageInfo[],
+    operation: OperationContext,
+  ): Promise<{ urls: string[]; cleanup: () => Promise<void> } | undefined> {
+    const primary = pool[0]
+    if (primary === undefined || primary.provider.uploadViaUrl !== true) return undefined
+    const client = await this.resolveObjectStorageClient()
+    if (client === undefined) {
+      throw new VisionToolkitError('config', 'uploadViaUrl is enabled but object storage is not configured')
+    }
+    const keys: string[] = []
+    const urls: string[] = []
+    try {
+      for (const image of images) {
+        if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision image upload cancelled')
+        const uploaded = await client.uploadImage(image.path, imageMimeType(image.format))
+        keys.push(uploaded.key)
+        urls.push(uploaded.url)
+      }
+    } catch (error) {
+      await Promise.allSettled(keys.map(key => client.deleteObject(key)))
+      throw error
+    }
+    return {
+      urls,
+      cleanup: async () => {
+        await Promise.allSettled(keys.map(key => client.deleteObject(key)))
+      },
+    }
+  }
+
+  /** Settings "test storage" probe: upload → head → delete a tiny marker object. */
+  async testObjectStorage(): Promise<{ detail: string }> {
+    const client = await this.resolveObjectStorageClient()
+    if (client === undefined) {
+      throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)')
+    }
+    return client.test()
+  }
+
+  /**
+   * Settings "test video call" probe: upload the bundled diagnostic video to
+   * object storage and send one OpenAI-compatible (Aliyun Qwen) video request.
+   * The wire content uses the Qwen video block shape (`video_url` + `fps`) plus
+   * the fixed test question. The object is deleted afterwards, best-effort.
+   */
+  async testVideoCall(options: ToolCallOptions, provider?: ResolvedProvider): Promise<{ detail: string }> {
+    return this.runOperation('vision_toolkit_video_test', options, async (operation) => {
+      const target = provider ?? this.primaryProvider
+      if (target.protocol !== 'openai') {
+        throw new VisionToolkitError('config', 'video support is only available for OpenAI-compatible providers')
+      }
+      const entry = await this.resolveProviderEnv(target)
+      if (entry === undefined) {
+        throw new VisionToolkitError('config', `credential ${String(target.credential)} is not configured`)
+      }
+      const client = await this.resolveObjectStorageClient()
+      if (client === undefined) {
+        throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)')
+      }
+      const uploaded = await client.uploadImage(VISION_MODEL_TEST_VIDEO, 'video/mp4')
+      try {
+        const answer = await this.requestVideoAnswer(target, entry.env.VISION_API_KEY, uploaded.url, VISION_MODEL_TEST_VIDEO_PROMPT, 2, operation)
+        const snippet = answer.trim().slice(0, 160)
+        return {
+          detail: snippet.length === 0
+            ? `Video call completed against ${target.model}`
+            : `Video call completed against ${target.model}: ${snippet}`,
+        }
+      } finally {
+        await client.deleteObject(uploaded.key)
+      }
+    })
+  }
+
+  /** Run the bundled ffprobe binary once and parse its JSON metadata. */
+  private async runFfprobe(videoPath: string, operation: OperationContext): Promise<VideoMetadata> {
+    const binary = ffprobeBinaryPath()
+    if (binary === null) {
+      throw new VisionToolkitError('runtime', 'ffprobe is unavailable; the ffprobe-static package must be installed alongside the plugin')
+    }
+    const handle = this.ctx.subprocess.spawn({
+      argv: [binary, '-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', videoPath],
+      cwd: process.cwd(),
+      stdio: {
+        stdin: 'ignore',
+        stdout: { maxBytes: 512 * 1024 },
+        stderr: { maxBytes: 64 * 1024 },
+      },
+      graceMs: 2000,
+      signal: operation.signal,
+    })
+    const outcome = await handle.done
+    const stdout = handle.collected.stdout?.readFrom(0)
+    const stderr = handle.collected.stderr?.readFrom(0)
+    if (outcome.exitCode !== 0) {
+      throw new VisionToolkitError('input', `cannot read video metadata: ${(stderr?.text ?? '').trim() || 'ffprobe failed'}`)
+    }
+    if (stdout?.lossy === true) {
+      throw new VisionToolkitError('output', 'video metadata output exceeded the capture limit')
+    }
+    try {
+      return parseFfprobeOutput(stdout?.text ?? '')
+    } catch (error) {
+      throw new VisionToolkitError('output', 'video metadata output is invalid', { cause: error })
+    }
+  }
+
+  /**
+   * One OpenAI-compatible (Aliyun Qwen) video chat-completions request. The
+   * content array carries the video_url block (with `fps`) plus the prompt text.
+   */
+  private async requestVideoAnswer(
+    target: ResolvedProvider,
+    apiKey: string,
+    videoUrl: string,
+    prompt: string,
+    fps: number,
+    operation: OperationContext,
+  ): Promise<string> {
+    operation.metrics.usedVisionService = true
+    const started = Date.now()
+    const response = await fetch(`${target.baseUrl}/chat/completions`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${apiKey}`,
+        'User-Agent': target.userAgent,
+      },
+      body: JSON.stringify({
+        model: target.model,
+        messages: [{
+          role: 'user',
+          content: [
+            { type: 'video_url', video_url: { url: videoUrl }, fps },
+            { type: 'text', text: prompt },
+          ],
+        }],
+      }),
+      signal: operation.signal,
+    })
+    operation.metrics.upstreamMs += Date.now() - started
+    const body = await response.text().catch(() => '')
+    if (!response.ok) {
+      throw new VisionToolkitError('service', `video request failed with HTTP ${response.status}: ${body.slice(0, 300)}`)
+    }
+    const answer = extractChatAnswer(body)
+    if (answer.trim().length === 0) {
+      throw new VisionToolkitError('output', 'vision API returned an empty answer')
+    }
+    return answer
+  }
+
+  /** Local video basic-info probe: ffprobe metadata, no API or credential. */
+  async videoInfo(request: VideoInfoRequest, options: ToolCallOptions): Promise<VideoInfo> {
+    return this.runOperation('vision_video_info', options, async (operation) => {
+      const policy = await this.pathPolicy(options.workspace)
+      const resolved = await resolveInputVideo(request.video, policy)
+      const metadata = await this.runFfprobe(resolved.path, operation)
+      return { path: resolved.path, bytes: resolved.bytes, ...metadata }
+    })
+  }
+
+  /**
+   * Video understanding: upload the video to object storage and send it plus a
+   * prompt to the first enabled OpenAI provider with video support. Whether the
+   * tool appears in an Agent is governed by the Settings tool-visibility video
+   * bucket, not by provider capability; this method re-checks capability at
+   * call time and returns a "video understanding unavailable" config error when
+   * no enabled provider supports video.
+   */
+  async videoUnderstand(request: VideoUnderstandRequest, options: ToolCallOptions): Promise<VideoUnderstandResult> {
+    return this.runOperation('vision_video_understand', options, async (operation) => {
+      const target = this.videoProvider()
+      if (target === undefined) {
+        throw new VisionToolkitError('config', 'no enabled vision service has video support; enable it in Settings first')
+      }
+      const entry = await this.resolveProviderEnv(target)
+      if (entry === undefined) {
+        throw new VisionToolkitError('config', `credential ${String(target.credential)} is not configured`)
+      }
+      const prompt = request.prompt.trim()
+      if (prompt.length === 0) {
+        throw new VisionToolkitError('input', 'prompt must not be empty')
+      }
+      const fps = request.fps ?? 2
+      if (!Number.isInteger(fps) || fps < 1 || fps > 120) {
+        throw new VisionToolkitError('input', 'fps must be an integer between 1 and 120')
+      }
+      const policy = await this.pathPolicy(options.workspace)
+      const resolved = await resolveInputVideo(request.video, policy)
+      const client = await this.resolveObjectStorageClient()
+      if (client === undefined) {
+        throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)')
+      }
+      const uploaded = await client.uploadImage(resolved.path, videoMediaType(extname(resolved.path).toLowerCase()))
+      try {
+        const answer = await this.requestVideoAnswer(target, entry.env.VISION_API_KEY, uploaded.url, prompt, fps, operation)
+        return { path: resolved.path, answer }
+      } finally {
+        await client.deleteObject(uploaded.key)
+      }
+    })
+  }
+
   /** Stable gate key for one provider's in-flight request cap. */
   private providerGate(provider: ResolvedProvider): Semaphore {
     const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`
@@ -1603,6 +1892,8 @@ export class VisionToolkitRuntime {
         protocol: env.VISION_API_PROTOCOL,
         anthropicThinking: env.VISION_ANTHROPIC_THINKING,
         sslVerify: env.VISION_SSL_VERIFY ?? null,
+        stream: env.VISION_STREAM === '1',
+        uploadViaUrl: provider.uploadViaUrl,
         userAgent: env.VISION_USER_AGENT,
         credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
         maxImageBytes: provider.maxImageBytes,
@@ -1733,24 +2024,31 @@ export class VisionToolkitRuntime {
           return cached.result
         }
       }
-      const result = await this.runVisionHedge('glance', [
-        ...images.map(image => image.path),
-        ...(request.region !== undefined ? ['--region', request.region] : []),
-        ...(request.ocr === true ? ['--ocr'] : []),
-        ...(request.query !== undefined ? ['-q', request.query] : []),
-      ], images, operation, pool)
-      const answer = result.stdout.trim()
-      if (answer.length === 0) throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
-      const value: GlanceResult = {
-        images,
-        mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
-        answer,
-        truncated: false,
-      }
-      if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
-        this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value })
+      const transfer = request.region === undefined
+        ? await this.maybeTransferImages(pool, images, operation)
+        : undefined
+      try {
+        const result = await this.runVisionHedge('glance', [
+          ...(transfer !== undefined ? transfer.urls : images.map(image => image.path)),
+          ...(transfer === undefined && request.region !== undefined ? ['--region', request.region] : []),
+          ...(request.ocr === true ? ['--ocr'] : []),
+          ...(request.query !== undefined ? ['-q', request.query] : []),
+        ], images, operation, pool)
+        const answer = result.stdout.trim()
+        if (answer.length === 0) throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
+        const value: GlanceResult = {
+          images,
+          mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
+          answer,
+          truncated: false,
+        }
+        if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
+          this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value })
+        }
+        return value
+      } finally {
+        if (transfer !== undefined) await transfer.cleanup()
       }
-      return value
     })
   }
 
@@ -1786,14 +2084,23 @@ export class VisionToolkitRuntime {
     }
     const image = await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation)
     this.accountImage(image, operation)
-    const result = await this.runVisionHedge(tool, [
-      image.path,
-      request.target,
-      ...(request.region !== undefined ? ['--region', request.region] : []),
-    ], [image], operation, pool)
-    const elements = parseLocationOutput(result.stdout)
-    this.validateLocations(elements, image.width, image.height)
-    return { image, elements }
+    const transfer = request.region === undefined
+      ? await this.maybeTransferImages(pool, [image], operation)
+      : undefined
+    try {
+      const result = await this.runVisionHedge(tool, transfer !== undefined
+        ? [transfer.urls[0]!, request.target, '--size', `${image.width}x${image.height}`]
+        : [
+          image.path,
+          request.target,
+          ...(request.region !== undefined ? ['--region', request.region] : []),
+        ], [image], operation, pool)
+      const elements = parseLocationOutput(result.stdout)
+      this.validateLocations(elements, image.width, image.height)
+      return { image, elements }
+    } finally {
+      if (transfer !== undefined) await transfer.cleanup()
+    }
   }
 
   /** ground: locate one named target and return pixel boxes. */

package/src/tools.ts

@@ -22,6 +22,7 @@ import {
   type TraceRequest,
 } from './runtime.ts'
 import { platformTempDirectory } from './paths.ts'
+import type { VideoInfoRequest, VideoUnderstandRequest } from './video.ts'
 
 const renderJson = (_args: unknown, value: unknown): ContentBlock[] => [{
   type: 'text',
@@ -48,8 +49,55 @@ export const VISION_TOOL_NAMES = {
   dominantColors: 'vision_dominant_colors',
   htmlScreenshot: 'vision_html_screenshot',
   concurrency: 'vision_concurrency',
+  videoInfo: 'vision_video_info',
 } as const
 
+/**
+ * Opt-in video-understanding tool name. Deliberately outside {@link VISION_TOOL_NAMES}
+ * so the always-registered canonical set stays unconditional; this tool enters an
+ * Agent only when a vision service has video support enabled.
+ */
+export const VISION_VIDEO_UNDERSTAND_TOOL = 'vision_video_understand'
+
+/**
+ * Tool-group buckets used by the `toolVisibility` snapshot. A bucket that is
+ * off contributes none of its listed tools to an Agent's visible surface.
+ * - local: local-processing tools (no on-line fan-out, no concurrency charge).
+ * - online: on-line image tools plus the concurrency/status probe.
+ * - video: video-understanding tool (experimental).
+ */
+export interface ToolVisibility {
+  local: boolean
+  online: boolean
+  video: boolean
+}
+
+const TOOL_BUCKETS: Record<string, 'local' | 'online' | 'video'> = {
+  [VISION_TOOL_NAMES.glance]: 'online',
+  [VISION_TOOL_NAMES.ground]: 'online',
+  [VISION_TOOL_NAMES.detect]: 'online',
+  [VISION_TOOL_NAMES.longScreenshotOcr]: 'online',
+  [VISION_TOOL_NAMES.concurrency]: 'online',
+  [VISION_TOOL_NAMES.trace]: 'local',
+  [VISION_TOOL_NAMES.crop]: 'local',
+  [VISION_TOOL_NAMES.pixelDiff]: 'local',
+  [VISION_TOOL_NAMES.extractForeground]: 'local',
+  [VISION_TOOL_NAMES.dominantColors]: 'local',
+  [VISION_TOOL_NAMES.htmlScreenshot]: 'local',
+  [VISION_TOOL_NAMES.videoInfo]: 'local',
+  [VISION_VIDEO_UNDERSTAND_TOOL]: 'video',
+}
+
+/** Filter one tool definition by the visibility snapshot's matching bucket. */
+function visibleBySnapshot(definition: { name: string }, snapshot: ToolVisibility): boolean {
+  switch (TOOL_BUCKETS[definition.name]) {
+    case 'local': return snapshot.local
+    case 'online': return snapshot.online
+    case 'video': return snapshot.video
+    default: return true
+  }
+}
+
 /** Resolve the caller workspace exactly like first-party fs/bash tools. */
 function sessionWorkspace(exec: ToolRunContext): string {
   return exec.agent?.session.header.cwd ?? process.cwd()
@@ -122,6 +170,19 @@ const requiredBoxSchema = { ...boxSchema, required: true } as const
 const requiredImageInfoSchema = { ...imageInfoSchema, required: true } as const
 const requiredArtifactSchema = { ...artifactSchema, required: true } as const
 
+const nullableStringSchema = {
+  oneOf: [{ type: 'string' }, { type: 'null' }],
+} as const satisfies ValueSchemaSpec
+const nullableIntegerSchema = {
+  oneOf: [{ type: 'integer' }, { type: 'null' }],
+} as const satisfies ValueSchemaSpec
+const nullableNumberSchema = {
+  oneOf: [{ type: 'number' }, { type: 'null' }],
+} as const satisfies ValueSchemaSpec
+const requiredNullableStringSchema = { ...nullableStringSchema, required: true } as const
+const requiredNullableIntegerSchema = { ...nullableIntegerSchema, required: true } as const
+const requiredNullableNumberSchema = { ...nullableNumberSchema, required: true } as const
+
 const locatedMatchSchema = {
   type: 'object',
   additionalProperties: false,
@@ -181,17 +242,21 @@ function runtimeFrom(source: VisionToolkitRuntimeSource): VisionToolkitRuntime {
  * @param source - Current runtime or atomic runtime lookup.
  * @param projectPresentation - Browser-only projection for Artifact capabilities.
  * @param lifecycleSignal - Plugin lifetime; aborting it cancels every active tool call.
+ * @param toolVisibility - Session-head visibility snapshot; a bucket that is off
+ *   contributes none of its tools. Defaults to every bucket on.
  * @returns Native tool definitions registered as one lifecycle generation.
  */
 export function createVisionTools(
   source: VisionToolkitRuntimeSource,
   projectPresentation: VisionToolkitPresentationProjector = presentationIdentity,
   lifecycleSignal?: AbortSignal,
+  toolVisibility: ToolVisibility = { local: true, online: true, video: false },
 ): ReturnType<typeof defineTool>[] {
   const presentationMeta = (_args: unknown, value: JsonValue): JsonValue => projectPresentation(value)
-  const sessionMaxConcurrency = runtimeFrom(source).sessionMaxConcurrency
+  const runtime = runtimeFrom(source)
+  const sessionMaxConcurrency = runtime.sessionMaxConcurrency
   const concurrencyNote = `A single session runs at most ${sessionMaxConcurrency} concurrent vision calls; query vision_concurrency for the live available count. `
-  return [
+  const tools: ReturnType<typeof defineTool>[] = [
     defineTool({
       name: VISION_TOOL_NAMES.glance,
       description: 'Describe, answer a targeted question about, OCR, or compare one or more images with the configured vision model. '
@@ -591,7 +656,8 @@ export function createVisionTools(
     }),
     defineTool({
       name: VISION_TOOL_NAMES.concurrency,
-      description: 'Report the current available concurrency for vision tool calls in this session: the smaller of the remaining per-session slots and the total remaining model-request slots across enabled providers.',
+      description: 'Report the current available concurrency for ON-LINE vision tool calls (glance, ground, detect, long-screenshot OCR, video understanding) in this session: the smaller of the remaining per-session slots and the total remaining model-request slots across enabled providers. '
+        + 'Local-processing tools (trace, crop, pixel diff, extract foreground, colors, HTML screenshot, video info) are NOT throttled by this and run immediately.',
       parameters: {},
       output: {
         schema: {
@@ -621,7 +687,72 @@ export function createVisionTools(
       isConcurrencySafe: () => true,
       presentCall: () => ({ card: 'generic', title: 'Vision concurrency', kind: 'read', locations: [] }),
     }),
+    defineTool({
+      name: VISION_TOOL_NAMES.videoInfo,
+      description: 'Probe a local video file and return its basic metadata (container format, duration, resolution, frame rate, codecs, and stream counts) using the bundled ffprobe binary. '
+        + 'This is a local operation and never calls a vision API or needs a credential. ' + concurrencyNote + WORKSPACE_NOTE,
+      parameters: {
+        video: { type: 'string', required: true, description: 'Video file path.' },
+        timeoutSeconds: { type: 'integer', description: TIMEOUT_NOTE },
+      },
+      output: {
+        schema: {
+          type: 'object', additionalProperties: false, properties: {
+            path: { type: 'string', required: true },
+            bytes: { type: 'integer', required: true },
+            format: { type: 'string', required: true },
+            durationSeconds: requiredNullableNumberSchema,
+            width: requiredNullableIntegerSchema,
+            height: requiredNullableIntegerSchema,
+            frameRate: requiredNullableNumberSchema,
+            videoCodec: requiredNullableStringSchema,
+            audioCodec: requiredNullableStringSchema,
+            bitRate: requiredNullableIntegerSchema,
+            videoStreamCount: { type: 'integer', required: true },
+            audioStreamCount: { type: 'integer', required: true },
+          },
+        },
+        render: renderJson,
+      },
+      async execute(args: VideoInfoArgs, exec) {
+        const request: VideoInfoRequest = { video: args.video }
+        return runtimeFrom(source).videoInfo(request, callOptions(exec, args.timeoutSeconds, lifecycleSignal))
+      },
+      isConcurrencySafe: () => true,
+      presentCall: args => ({ card: 'generic', title: `Inspect ${args.video}`, kind: 'read', locations: [{ path: args.video }] }),
+    }),
+    defineTool({
+      name: VISION_VIDEO_UNDERSTAND_TOOL,
+      description: 'Send a local video plus a prompt to the configured vision service and return its answer text. '
+        + 'The video is uploaded to object storage and passed as a video_url block (Aliyun Qwen format). '
+        + `This tool exists only when the video tool bucket is enabled in Settings. Calling it without an enabled vision service that supports video returns a "video understanding unavailable" error. ${UNTRUSTED_EVIDENCE_NOTE} ` + concurrencyNote + WORKSPACE_NOTE,
+      parameters: {
+        video: { type: 'string', required: true, description: 'Video file path.' },
+        prompt: { type: 'string', required: true, description: 'Question or instruction about the video.' },
+        fps: { type: 'integer', description: 'Video sampling frame rate passed to the model; default 2.' },
+        timeoutSeconds: { type: 'integer', description: TIMEOUT_NOTE },
+      },
+      output: {
+        schema: {
+          type: 'object', additionalProperties: false, properties: {
+            path: { type: 'string', required: true },
+            answer: { type: 'string', required: true },
+          },
+        },
+        render: renderJson,
+      },
+      async execute(args: VideoUnderstandArgs, exec) {
+        const request: VideoUnderstandRequest = {
+          video: args.video,
+          prompt: args.prompt,
+          ...(args.fps === undefined ? {} : { fps: args.fps }),
+        }
+        return runtimeFrom(source).videoUnderstand(request, callOptions(exec, args.timeoutSeconds, lifecycleSignal))
+      },
+      presentCall: args => ({ card: 'generic', title: `Understand ${args.video}`, kind: 'read', locations: [{ path: args.video }] }),
+    }),
   ]
+  return tools.filter(definition => visibleBySnapshot(definition, toolVisibility))
 }
 
 interface GlanceArgs {
@@ -722,3 +853,13 @@ interface HtmlArgs {
   output?: string
   timeoutSeconds?: number
 }
+interface VideoInfoArgs {
+  video: string
+  timeoutSeconds?: number
+}
+interface VideoUnderstandArgs {
+  video: string
+  prompt: string
+  fps?: number
+  timeoutSeconds?: number
+}

package/src/upstream.ts

@@ -41,6 +41,7 @@ export interface UpstreamEnvironment {
   VISION_API_PROTOCOL: 'chat_completions' | 'anthropic'
   VISION_ANTHROPIC_THINKING: 'omit' | 'disabled' | 'adaptive'
   VISION_SSL_VERIFY?: string
+  VISION_STREAM?: string
   VISION_USER_AGENT: string
   LANG: 'zh' | 'en'
 }
@@ -761,6 +762,9 @@ export class UpstreamAdapter {
           ...(options.env.VISION_SSL_VERIFY === undefined
             ? {}
             : { VISION_SSL_VERIFY: options.env.VISION_SSL_VERIFY }),
+          ...(options.env.VISION_STREAM === undefined
+            ? {}
+            : { VISION_STREAM: options.env.VISION_STREAM }),
           VISION_USER_AGENT: options.env.VISION_USER_AGENT,
           LANG: options.env.LANG,
           VISION_ENV_FILE: join(prepared.cleanHome, 'vision.env'),