@mengruo/dsh-vision-toolkit 0.1.4 → 0.1.6-beta.0

package/src/video.ts

@@ -0,0 +1,222 @@
+/**
+ * Local video support built on the bundled `ffprobe-static` binary instead of
+ * the pinned Python vision pipeline: metadata probing stays a plain JS
+ * subprocess, and the model-facing video request reuses the existing object
+ * storage + OpenAI-compatible (Aliyun Qwen) chat-completions transport.
+ * @module dsh-vision-toolkit/video
+ */
+
+import { createRequire } from 'node:module'
+
+const require = createRequire(import.meta.url)
+
+/** Metadata derived from one ffprobe run (path/bytes are supplied by the runtime). */
+export interface VideoMetadata {
+  /** Container format name, e.g. "mov" or "matroska". */
+  format: string
+  /** Format duration in seconds, or null when the stream reports none. */
+  durationSeconds: number | null
+  /** Width of the first video stream, or null when the file carries no video. */
+  width: number | null
+  /** Height of the first video stream, or null when the file carries no video. */
+  height: number | null
+  /** Average frame rate of the first video stream, or null when unknown. */
+  frameRate: number | null
+  /** Codec of the first video stream, or null when the file carries no video. */
+  videoCodec: string | null
+  /** Codec of the first audio stream, or null when the file carries no audio. */
+  audioCodec: string | null
+  /** Format bit rate in bits per second, or null when unknown. */
+  bitRate: number | null
+  /** Number of video streams. */
+  videoStreamCount: number
+  /** Number of audio streams. */
+  audioStreamCount: number
+}
+
+/** Full structured result for one local video-info probe. */
+export interface VideoInfo extends VideoMetadata {
+  /** Fence-checked absolute input path. */
+  path: string
+  /** Input file size in bytes. */
+  bytes: number
+}
+
+/** Input for the local video-info tool (no API call). */
+export interface VideoInfoRequest {
+  video: string
+}
+
+/** Input for the video-understanding tool (video + prompt → vision API). */
+export interface VideoUnderstandRequest {
+  video: string
+  prompt: string
+  /** Video sampling frame rate passed to the model; default 2. */
+  fps?: number
+}
+
+/** Result of one video-understanding call. */
+export interface VideoUnderstandResult {
+  /** Fence-checked absolute input path. */
+  path: string
+  /** The vision model's answer text. */
+  answer: string
+}
+
+/**
+ * Resolve the bundled ffprobe executable. An explicit `FFPROBE_BIN` override
+ * wins (used by tests and unusual installs); otherwise the platform-correct
+ * binary shipped by `ffprobe-static` is returned. `null` means no binary is
+ * available and the caller must fail loud.
+ */
+export function ffprobeBinaryPath(): string | null {
+  const override = process.env.FFPROBE_BIN?.trim()
+  if (override !== undefined && override.length > 0) return override
+  try {
+    const mod = require('ffprobe-static') as { path?: unknown } | undefined
+    const candidate = mod?.path
+    return typeof candidate === 'string' && candidate.length > 0 ? candidate : null
+  } catch {
+    return null
+  }
+}
+
+/** S3 object Content-Type for one video extension (browser-probing convention). */
+export function videoMediaType(extension: string): string {
+  switch (extension) {
+    case '.mp4': return 'video/mp4'
+    case '.mov': return 'video/quicktime'
+    case '.webm': return 'video/webm'
+    case '.mkv': return 'video/x-matroska'
+    case '.avi': return 'video/x-msvideo'
+    case '.m4v': return 'video/x-m4v'
+    case '.mpeg':
+    case '.mpg': return 'video/mpeg'
+    case '.wmv': return 'video/x-ms-wmv'
+    case '.flv': return 'video/x-flv'
+    case '.ts':
+    case '.m2ts': return 'video/mp2t'
+    case '.3gp': return 'video/3gpp'
+    default: return 'application/octet-stream'
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+function stringValue(value: unknown): string | null {
+  return typeof value === 'string' && value.trim().length > 0 ? value : null
+}
+
+function integerValue(value: unknown): number | null {
+  return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : null
+}
+
+/** Parse "num/den" or a plain float string into a frame rate, else null. */
+function parseRate(raw: unknown): number | null {
+  const text = stringValue(raw)
+  if (text === null) return null
+  const slash = text.indexOf('/')
+  if (slash === -1) {
+    const value = Number(text)
+    return Number.isFinite(value) && value > 0 ? value : null
+  }
+  const numerator = Number(text.slice(0, slash))
+  const denominator = Number(text.slice(slash + 1))
+  if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator === 0) return null
+  const value = numerator / denominator
+  return Number.isFinite(value) && value > 0 ? value : null
+}
+
+/**
+ * Parse the JSON printed by `ffprobe -print_format json -show_format -show_streams`
+ * into a compact, model-facing metadata object. Missing or malformed fields
+ * degrade to null rather than throwing: metadata is advisory, and the probe
+ * only fails when the process itself failed or returned no JSON.
+ */
+export function parseFfprobeOutput(stdout: string): VideoMetadata {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(stdout)
+  } catch {
+    throw new Error('ffprobe returned invalid JSON')
+  }
+  if (!isRecord(parsed)) throw new Error('ffprobe returned an unexpected structure')
+
+  const streams = Array.isArray(parsed.streams) ? parsed.streams.filter(isRecord) : []
+  const format = isRecord(parsed.format) ? parsed.format : {}
+
+  let videoCodec: string | null = null
+  let audioCodec: string | null = null
+  let width: number | null = null
+  let height: number | null = null
+  let frameRate: number | null = null
+  let videoStreamCount = 0
+  let audioStreamCount = 0
+  for (const stream of streams) {
+    const codecType = stringValue(stream.codec_type)
+    const codecName = stringValue(stream.codec_name)
+    if (codecType === 'video') {
+      videoStreamCount += 1
+      if (videoCodec === null) {
+        videoCodec = codecName
+        width = integerValue(stream.width)
+        height = integerValue(stream.height)
+        frameRate = parseRate(stream.avg_frame_rate ?? stream.r_frame_rate)
+      }
+    } else if (codecType === 'audio') {
+      audioStreamCount += 1
+      if (audioCodec === null) audioCodec = codecName
+    }
+  }
+
+  const durationText = stringValue(format.duration)
+  const durationSeconds = durationText === null || durationText === 'N/A'
+    ? null
+    : (() => { const value = Number(durationText); return Number.isFinite(value) && value >= 0 ? value : null })()
+
+  const formatName = stringValue(format.format_name)?.split(',')[0]?.trim() ?? null
+
+  return {
+    format: formatName ?? 'unknown',
+    durationSeconds,
+    width,
+    height,
+    frameRate,
+    videoCodec,
+    audioCodec,
+    bitRate: integerValue(typeof format.bit_rate === 'string' ? Number(format.bit_rate) : format.bit_rate),
+    videoStreamCount,
+    audioStreamCount,
+  }
+}
+
+/**
+ * Extract the answer text from one OpenAI-compatible chat-completions response
+ * body. Handles both string content and array content (text parts joined).
+ * @returns the answer text, or an empty string when the shape is unrecognized.
+ */
+export function extractChatAnswer(body: string): string {
+  try {
+    const parsed = JSON.parse(body) as unknown
+    if (!isRecord(parsed)) return ''
+    const choices = parsed.choices
+    if (!Array.isArray(choices) || choices.length === 0) return ''
+    const first = choices[0]
+    if (!isRecord(first)) return ''
+    const message = first.message
+    if (!isRecord(message)) return ''
+    const content = message.content
+    if (typeof content === 'string') return content
+    if (Array.isArray(content)) {
+      return content
+        .filter(isRecord)
+        .map(part => (typeof part.text === 'string' ? part.text : ''))
+        .join('')
+    }
+    return ''
+  } catch {
+    return ''
+  }
+}

package/src/web.ts

@@ -75,6 +75,12 @@ export interface VisionToolkitSettingsSnapshot {
     source?: string
     writable: boolean
   }>
+  /** Object-storage credential state; `ref` is empty when object storage is unset. */
+  objectStorageCredential: {
+    ref: string
+    configured: boolean
+    writable: boolean
+  }
   runtime: RuntimeManagerStatus
   release: {
     pluginVersion: string
@@ -121,7 +127,17 @@ interface ApplyUpdateRequest {
   expectedVersion: string
 }
 
-type SettingsRequest = SaveRequest | HealthRequest | CredentialRequest | DeleteCredentialRequest | CheckUpdateRequest | ApplyUpdateRequest
+interface StorageTestRequest {
+  action: 'test-storage'
+}
+
+interface VideoTestRequest {
+  action: 'test-video'
+  /** 0-based provider index; when absent, the primary provider is tested. */
+  providerIndex?: number
+}
+
+type SettingsRequest = SaveRequest | HealthRequest | CredentialRequest | DeleteCredentialRequest | CheckUpdateRequest | ApplyUpdateRequest | StorageTestRequest | VideoTestRequest
 
 interface JsonError {
   ok: false
@@ -261,6 +277,17 @@ function parseRequest(value: unknown): SettingsRequest {
     }
     return { action: 'apply-update', expectedVersion: value.expectedVersion.trim() }
   }
+  if (value.action === 'test-storage') return { action: 'test-storage' }
+  if (value.action === 'test-video') {
+    const providerIndex = value.providerIndex
+    if (providerIndex !== undefined && (!Number.isSafeInteger(providerIndex) || (providerIndex as number) < 0)) {
+      throw new TypeError('test-video.providerIndex must be a non-negative integer')
+    }
+    return {
+      action: 'test-video',
+      ...(providerIndex === undefined ? {} : { providerIndex: providerIndex as number }),
+    }
+  }
   throw new TypeError(`unsupported action: ${value.action}`)
 }
 
@@ -315,6 +342,14 @@ export class VisionToolkitWebBackend {
         writable: info.writable,
       }
     }))
+    const objectStorageRef = resolved.objectStorage.credential === undefined ? '' : String(resolved.objectStorage.credential)
+    const objectStorageCredential = objectStorageRef === ''
+      ? { ref: '', configured: false, writable: this.ctx.settings.writable }
+      : await this.ctx.credentials.describe(credentialRef(objectStorageRef)).then(info => ({
+        ref: objectStorageRef,
+        configured: info.configured,
+        writable: info.writable,
+      }))
     const update = await this.updater.capability()
     return {
       schemaVersion: 1,
@@ -333,6 +368,7 @@ export class VisionToolkitWebBackend {
         writable: credential.writable,
       },
       credentials,
+      objectStorageCredential,
       runtime: this.manager.status(),
       release: {
         pluginVersion: PLUGIN_VERSION,
@@ -380,10 +416,18 @@ export class VisionToolkitWebBackend {
       )
     }
     const resolved = resolveConfig(descriptor.value as VisionToolkitConfig)
-    const provider = resolved.providers.find(entry => String(entry.credential) === String(request.ref))
+    const ref = String(request.ref)
+    if (resolved.objectStorage.credential !== undefined && String(resolved.objectStorage.credential) === ref) {
+      if (!request.value.includes(':')) {
+        throw new Error('object storage credential must be "accessKeyId:secretAccessKey"')
+      }
+      await this.ctx.credentials.set(request.ref, request.value)
+      return this.snapshot()
+    }
+    const provider = resolved.providers.find(entry => String(entry.credential) === ref)
     if (provider === undefined) {
       throw new CredentialReferenceConflictError(
-        `credential reference "${String(request.ref)}" does not match any configured vision provider; reload Settings and try again`,
+        `credential reference "${ref}" does not match any configured vision provider; reload Settings and try again`,
       )
     }
     if (isBuiltInFreeVisionProvider(provider)) {
@@ -426,6 +470,38 @@ export class VisionToolkitWebBackend {
     }
   }
 
+  private async testStorage(): Promise<{ detail: string }> {
+    if (!this.manager.ready) throw new Error('runtime is not ready; fix Settings and save a valid configuration first')
+    return this.manager.current().testObjectStorage()
+  }
+
+  private async testVideo(request: VideoTestRequest, req: IncomingMessage): Promise<{ detail: string }> {
+    if (!this.manager.ready) throw new Error('runtime is not ready; fix Settings and save a valid configuration first')
+    const controller = new AbortController()
+    const abort = (): void => { controller.abort() }
+    req.once('aborted', abort)
+    req.socket.once('close', abort)
+    try {
+      const runtime = this.manager.current()
+      let provider: ResolvedProvider | undefined
+      if (request.providerIndex !== undefined) {
+        const resolved = resolveConfig(descriptorOf(this.ctx).value as VisionToolkitConfig)
+        provider = resolved.providers[request.providerIndex]
+        if (provider === undefined) {
+          throw new Error(`provider index ${request.providerIndex} is out of range`)
+        }
+      }
+      return await runtime.testVideoCall({
+        signal: controller.signal,
+        workspace: runtime.upstreamVersion.runtimeHome,
+        sessionId: 'vision-toolkit-settings',
+      }, provider)
+    } finally {
+      req.off('aborted', abort)
+      req.socket.off('close', abort)
+    }
+  }
+
   /** Handle the exact Settings route. */
   async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
     if (req.method === 'GET') {
@@ -458,6 +534,12 @@ export class VisionToolkitWebBackend {
         case 'health':
           responseJson(res, 200, { ok: true, value: await this.health(parsed, req) })
           break
+        case 'test-storage':
+          responseJson(res, 200, { ok: true, value: await this.testStorage() })
+          break
+        case 'test-video':
+          responseJson(res, 200, { ok: true, value: await this.testVideo(parsed, req) })
+          break
         case 'save':
           responseJson(res, 200, { ok: true, value: await this.save(parsed) })
           break
@@ -486,14 +568,18 @@ export class VisionToolkitWebBackend {
             ? error.code
           : parsed.action === 'health'
             ? 'health-failed'
-            : parsed.action === 'credential'
-              ? 'credential-rejected'
-              : 'settings-rejected'
+            : parsed.action === 'test-storage'
+              ? 'storage-test-failed'
+              : parsed.action === 'test-video'
+                ? 'video-test-failed'
+                : parsed.action === 'credential'
+                  ? 'credential-rejected'
+                  : 'settings-rejected'
       const updateConflict = updateError && ['update-in-progress', 'update-stale', 'update-unavailable', 'already-current'].includes(error.code)
       const updateGateway = updateError && error.code === 'update-check-failed'
       const status = settingsConflict || credentialConflict || updateConflict
         ? 409
-        : parsed.action === 'health'
+        : parsed.action === 'health' || parsed.action === 'test-storage' || parsed.action === 'test-video'
           ? 503
           : updateGateway
             ? 502

package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json

@@ -3,7 +3,7 @@
   "repository": "https://github.com/Anionex/agent-vision-toolkit",
   "version": "v0.1.0+snapshot.bc9803d",
   "commit": "bc9803d7d6300c864d17460ecbb33540b26638e0",
-  "contentSha256": "0eaa22a0d1d0dd6d6523a0ddd3fa8197246005042bc164a703fb05102b287a38",
+  "contentSha256": "09bedf007e1469a28981227def5aa33e03667ea21fe2972937f0359898e2928e",
   "files": [
     {
       "path": "CHANGELOG.md",
@@ -32,8 +32,8 @@
     },
     {
       "path": "bin/glance",
-      "bytes": 3742,
-      "sha256": "fa4ba52e8e180475b948daec817151d893d957f973f7d9df1b8bbf201051cefc"
+      "bytes": 3941,
+      "sha256": "7552bef0ba396162f64e8fc4c4a53404dfe2d3149ca3d1c5c8bd717facf82af6"
     },
     {
       "path": "bin/ground",
@@ -47,13 +47,13 @@
     },
     {
       "path": "detect.py",
-      "bytes": 2218,
-      "sha256": "48a7070084f5b23b1477fa9a690ef1e679da8a03228e64a1ac583de633040bfc"
+      "bytes": 2532,
+      "sha256": "cac2135f9d835215f3ba90bfd70fd84dd02b0e88299f2b60e109e006b4f3a4d9"
     },
     {
       "path": "ground.py",
-      "bytes": 10117,
-      "sha256": "845e56dbdf92f2c79495170f5d215de49985b3099012b4d398231d3c78bd0090"
+      "bytes": 11289,
+      "sha256": "6d3175d3f5a7d561d55b2a967c5ca3d49940e80014de658ed9a3e5d45a233a2f"
     },
     {
       "path": "skills/vision-tools/scripts/dominant_colors.py",
@@ -82,13 +82,13 @@
     },
     {
       "path": "tests/test_vision_client.py",
-      "bytes": 20481,
-      "sha256": "b019f0da28b7567226292044f791059ee34d7f0088bd9ceb328cfa1bab250a48"
+      "bytes": 24882,
+      "sha256": "87e39429766b38caed6a4ccb1ac8b193094c69858a0f7541940a005f66421e45"
     },
     {
       "path": "vision_client.py",
-      "bytes": 11838,
-      "sha256": "c6a48048c864e99513faf90ba17c4ac0d9462bb718683682ed8eda12c0e2cfe0"
+      "bytes": 15053,
+      "sha256": "6800843b99df500a476b1dd6ff5a4c05aa8122ed990f6f91b16a11ee07bdebe6"
     }
   ]
 }

package/vendor/agent-vision-toolkit/bin/glance

@@ -40,6 +40,13 @@ def region_data_url(path, region):
     return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
 
 
+def image_url(path):
+    """Pass http(s) URLs through unchanged; encode local files as data URLs."""
+    if path.startswith(("http://", "https://")):
+        return path
+    return image_path_to_data_url(path)
+
+
 def build_prompt(args, count):
     if args.ocr is not None:
         extra = f" Additional requirements: {args.ocr}" if args.ocr else ""
@@ -77,7 +84,7 @@ def main():
         if args.region and len(args.images) > 1:
             raise VisionError("--region works with exactly one image")
         urls = ([region_data_url(args.images[0], args.region)] if args.region
-                else [image_path_to_data_url(path) for path in args.images])
+                else [image_url(path) for path in args.images])
         answer = describe_image(
             urls,
             build_prompt(args, len(urls)),

package/vendor/agent-vision-toolkit/detect.py

@@ -8,7 +8,7 @@ try:
 except ImportError:
     Image = None
 
-from ground import GroundError, _position, locate
+from ground import GroundError, _parse_size, _position, locate
 from vision_client import VisionError
 
 DEFAULT_CATEGORY = ("UI element (buttons, links, inputs, icons, labels, "
@@ -38,18 +38,24 @@ def main() -> None:
         prog="detect",
         description="Inventory the elements in an image (or a region) with pixel bounding boxes",
     )
-    parser.add_argument("image", type=Path, help="path to the image")
+    parser.add_argument("image", help="path or http(s) URL to the image")
     parser.add_argument("category", nargs="?",
                         help='restrict to a category, e.g. "buttons" or "icons" (default: all UI elements)')
     parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
                         help="inventory only this pixel box; output stays in original-image coordinates")
+    parser.add_argument("--size", metavar="WxH",
+                        help="analyzed image dimensions when the image is an http(s) URL")
     args = parser.parse_args()
     try:
-        matches = locate(args.image.expanduser(), build_target(args.category), region=args.region)
-        if Image is None:
-            raise GroundError("detect requires Pillow; install the optional dependency pillow first")
-        with Image.open(args.image.expanduser()) as image:
-            width, height = image.size
+        size = _parse_size(args.size) if args.size else None
+        matches = locate(args.image, build_target(args.category), region=args.region, size=size)
+        if size is not None:
+            width, height = size
+        else:
+            if Image is None:
+                raise GroundError("detect requires Pillow; install the optional dependency pillow first")
+            with Image.open(Path(args.image).expanduser()) as image:
+                width, height = image.size
     except (GroundError, VisionError) as exc:
         parser.exit(1, f"detect: {exc}\n")
     for line in format_inventory(matches, width, height):

package/vendor/agent-vision-toolkit/ground.py

@@ -184,23 +184,32 @@ def _parse_region(region: str, width: int, height: int) -> tuple[int, int, int,
     return box
 
 
-def locate(image_path: Path, target: str, region: str | None = None) -> list[Match]:
+def locate(image_path: str, target: str, region: str | None = None,
+           size: tuple[int, int] | None = None) -> list[Match]:
     if Image is None:
         raise GroundError("ground requires Pillow; install the optional dependency pillow first")
     load_default_env()
     box = None
-    try:
-        with Image.open(image_path) as image:
-            width, height = image.size
-            if region:
-                box = _parse_region(region, width, height)
-                buffer = io.BytesIO()
-                image.crop(box).save(buffer, format="PNG")
-                url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
-    except (OSError, ValueError) as exc:
-        raise GroundError(f"Cannot read image: {image_path}") from exc
+    if size is not None:
+        # URL transfer: the image was uploaded to object storage and this caller
+        # supplies the analyzed dimensions, so no local file access is needed.
+        width, height = size
+        url = image_path if image_path.startswith(("http://", "https://")) else image_path_to_data_url(image_path)
+    else:
+        local = Path(image_path).expanduser()
+        try:
+            with Image.open(local) as image:
+                width, height = image.size
+                if region:
+                    box = _parse_region(region, width, height)
+                    buffer = io.BytesIO()
+                    image.crop(box).save(buffer, format="PNG")
+                    url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+        except (OSError, ValueError) as exc:
+            raise GroundError(f"Cannot read image: {image_path}") from exc
+        if box is None:
+            url = image_path_to_data_url(local)
     if box is None:
-        url = image_path_to_data_url(image_path)
         width_used, height_used = width, height
     else:
         width_used, height_used = box[2] - box[0], box[3] - box[1]
@@ -216,6 +225,16 @@ def locate(image_path: Path, target: str, region: str | None = None) -> list[Mat
                             m.bbox[2] + box[0], m.bbox[3] + box[1])) for m in matches]
 
 
+def _parse_size(value: str) -> tuple[int, int]:
+    try:
+        width, height = (int(part) for part in value.lower().split("x"))
+    except (ValueError, AttributeError):
+        raise GroundError("--size expects WIDTHxHEIGHT (e.g. 1024x768)") from None
+    if width <= 0 or height <= 0:
+        raise GroundError("--size expects positive WIDTHxHEIGHT")
+    return width, height
+
+
 def _position(box: tuple[int, int, int, int], width: int, height: int) -> str:
     x1, y1, x2, y2 = box
     x = (x1 + x2) / 2
@@ -246,17 +265,23 @@ def main() -> None:
         prog="ground",
         description="Locate targets in an image with natural language and output pixel coordinates",
     )
-    parser.add_argument("image", type=Path, help="path to the image")
+    parser.add_argument("image", help="path or http(s) URL to the image")
     parser.add_argument("target", help="target object or region to locate")
     parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
                         help="search only this pixel box; output stays in original-image coordinates")
+    parser.add_argument("--size", metavar="WxH",
+                        help="analyzed image dimensions when the image is an http(s) URL")
     args = parser.parse_args()
     try:
-        matches = locate(args.image.expanduser(), args.target, region=args.region)
-        if Image is None:
-            raise GroundError("ground requires Pillow; install the optional dependency pillow first")
-        with Image.open(args.image.expanduser()) as image:
-            width, height = image.size
+        size = _parse_size(args.size) if args.size else None
+        matches = locate(args.image, args.target, region=args.region, size=size)
+        if size is not None:
+            width, height = size
+        else:
+            if Image is None:
+                raise GroundError("ground requires Pillow; install the optional dependency pillow first")
+            with Image.open(Path(args.image).expanduser()) as image:
+                width, height = image.size
     except (GroundError, VisionError) as exc:
         parser.exit(1, f"ground: {exc}\n")
     for line in format_matches(matches, width, height):