@sylphx/video-reader-mcp 0.1.2 → 0.1.5

package/README.md

@@ -43,6 +43,8 @@ Cue is **local-first timeline architecture**: streams, dialogue, scene cuts, **s
 
 Spec: [docs/specs/agent-video-read-contract.md](docs/specs/agent-video-read-contract.md)
 
+**Local-first frontier:** ffmpeg/ffprobe + structural keyframes, optional local whisper ASR. No cloud required.
+
 ## Product docs
 
 | Doc | Purpose |

package/bin/video-reader-mcp

@@ -1,48 +1,119 @@
 #!/usr/bin/env bash
-# Video Reader MCP launcher — Rust rmcp only (fail-closed, no TS stdio fallback).
+# Cue MCP launcher — Rust rmcp only (fail-closed).
+# Resolves native via optionalDependencies, staged bin/native, or cargo target.
 set -euo pipefail
 
-ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+PACKAGE_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+host_platform_key() {
+	local os arch
+	os="$(uname -s 2>/dev/null || echo unknown)"
+	arch="$(uname -m 2>/dev/null || echo unknown)"
+	case "$os" in
+		Darwin) os="darwin" ;;
+		Linux) os="linux" ;;
+		*) os="$(printf '%s' "$os" | tr '[:upper:]' '[:lower:]')" ;;
+	esac
+	case "$arch" in
+		x86_64|amd64) arch="x64" ;;
+		aarch64|arm64) arch="arm64" ;;
+	esac
+	printf '%s-%s\n' "$os" "$arch"
+}
+
+platform_package_name() {
+	case "$(host_platform_key)" in
+		darwin-arm64) printf '%s\n' '@sylphx/video-reader-mcp-darwin-arm64' ;;
+		darwin-x64) printf '%s\n' '@sylphx/video-reader-mcp-darwin-x64' ;;
+		linux-x64) printf '%s\n' '@sylphx/video-reader-mcp-linux-x64-gnu' ;;
+		linux-arm64) printf '%s\n' '@sylphx/video-reader-mcp-linux-arm64-gnu' ;;
+		*) return 1 ;;
+	esac
+}
+
+is_runnable_native() {
+	local candidate="$1"
+	[[ -n "$candidate" && -f "$candidate" && -x "$candidate" ]] || return 1
+	return 0
+}
+
+resolve_from_optional_dep() {
+	local pkg_name candidate
+	pkg_name="$(platform_package_name 2>/dev/null || true)"
+	[[ -n "$pkg_name" ]] || return 1
+	local search_roots=(
+		"$PACKAGE_ROOT/node_modules"
+		"$PACKAGE_ROOT/../node_modules"
+		"$PACKAGE_ROOT/../../node_modules"
+	)
+	for root in "${search_roots[@]}"; do
+		candidate="$root/$pkg_name/video-reader-mcp-server"
+		if is_runnable_native "$candidate"; then
+			printf '%s\n' "$candidate"
+			return 0
+		fi
+	done
+	if command -v node >/dev/null 2>&1; then
+		candidate="$(
+			node -e "
+        try {
+          const p = require('node:path');
+          const { createRequire } = require('node:module');
+          const r = createRequire(p.join(process.argv[1], 'package.json'));
+          const pkg = r.resolve(process.argv[2] + '/package.json');
+          process.stdout.write(p.join(p.dirname(pkg), 'video-reader-mcp-server'));
+        } catch { process.exit(1); }
+      " "$PACKAGE_ROOT" "$pkg_name" 2>/dev/null || true
+		)"
+		if is_runnable_native "${candidate:-}"; then
+			printf '%s\n' "$candidate"
+			return 0
+		fi
+	fi
+	return 1
+}
 
 resolve_rust_bin() {
-  if [[ -n "${VIDEO_READER_MCP_RUST_BIN:-}" && -x "${VIDEO_READER_MCP_RUST_BIN}" ]]; then
-    printf '%s\n' "${VIDEO_READER_MCP_RUST_BIN}"
-    return 0
-  fi
-
-  for candidate in \
-    "$ROOT/bin/native/video-reader-mcp-server" \
-    "$ROOT/target/release/video-reader-mcp-server" \
-    "$ROOT/target/debug/video-reader-mcp-server"; do
-    if [[ -x "$candidate" ]]; then
-      printf '%s\n' "$candidate"
-      return 0
-    fi
-  done
-
-  return 1
+	if [[ -n "${VIDEO_READER_MCP_RUST_BIN:-}" && -x "${VIDEO_READER_MCP_RUST_BIN}" ]]; then
+		printf '%s\n' "${VIDEO_READER_MCP_RUST_BIN}"
+		return 0
+	fi
+	if bin="$(resolve_from_optional_dep)"; then
+		printf '%s\n' "$bin"
+		return 0
+	fi
+	for candidate in \
+		"$PACKAGE_ROOT/bin/native/video-reader-mcp-server" \
+		"$PACKAGE_ROOT/target/release/video-reader-mcp-server" \
+		"$PACKAGE_ROOT/target/debug/video-reader-mcp-server"; do
+		if is_runnable_native "$candidate"; then
+			printf '%s\n' "$candidate"
+			return 0
+		fi
+	done
+	return 1
 }
 
 resolve_transport() {
-  if [[ -n "${VIDEO_READER_MCP_TRANSPORT:-}" ]]; then
-    printf '%s\n' "${VIDEO_READER_MCP_TRANSPORT}"
-    return 0
-  fi
-  if [[ -n "${MCP_TRANSPORT:-}" ]]; then
-    printf '%s\n' "${MCP_TRANSPORT}"
-    return 0
-  fi
-  printf '%s\n' "stdio"
+	if [[ -n "${VIDEO_READER_MCP_TRANSPORT:-}" ]]; then
+		printf '%s\n' "${VIDEO_READER_MCP_TRANSPORT}"
+		return 0
+	fi
+	if [[ -n "${MCP_TRANSPORT:-}" ]]; then
+		printf '%s\n' "${MCP_TRANSPORT}"
+		return 0
+	fi
+	printf '%s\n' "stdio"
 }
 
 if bin="$(resolve_rust_bin)"; then
-  transport="$(resolve_transport)"
-  if [[ "$transport" == "http" ]]; then
-    export MCP_TRANSPORT=http
-    export VIDEO_READER_MCP_TRANSPORT=http
-  fi
-  exec "$bin" "$@"
+	transport="$(resolve_transport)"
+	if [[ "$transport" == "http" ]]; then
+		export MCP_TRANSPORT=http
+		export VIDEO_READER_MCP_TRANSPORT=http
+	fi
+	exec "$bin" "$@"
 fi
 
-echo "[video-reader-mcp] Rust MCP server (rmcp) is not built. Run: bun run build:rust" >&2
+echo "[cue] Rust MCP server not found. Install matching optional native package or run: bun run build:rust" >&2
 exit 1

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@sylphx/video-reader-mcp",
-  "version": "0.1.2",
+  "version": "0.1.5",
   "mcpName": "io.github.SylphxAI/video-reader-mcp",
-  "description": "Cue \u2014 Evidence-first video reading for AI agents \u2014 ffprobe, subtitles, scenes, transcripts, and timelines without frame-by-frame LLM vision.",
+  "description": "Cue — Evidence-first video reading for AI agents — ffprobe, subtitles, scenes, transcripts, and timelines without frame-by-frame LLM vision.",
   "type": "module",
   "publishConfig": {
     "access": "public"
@@ -13,7 +13,7 @@
     "cue": "./bin/video-reader-mcp"
   },
   "files": [
-    "bin/",
+    "bin/video-reader-mcp",
     "dist/",
     "README.md",
     "LICENSE",
@@ -76,5 +76,11 @@
     "typescript": "^7.0.2"
   },
   "packageManager": "bun@1.3.12",
-  "private": false
+  "private": false,
+  "optionalDependencies": {
+    "@sylphx/video-reader-mcp-darwin-arm64": "0.1.5",
+    "@sylphx/video-reader-mcp-darwin-x64": "0.1.5",
+    "@sylphx/video-reader-mcp-linux-x64-gnu": "0.1.5",
+    "@sylphx/video-reader-mcp-linux-arm64-gnu": "0.1.5"
+  }
 }

package/src/utils/asr.ts

@@ -1,6 +1,10 @@
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { shouldUseRustAsrEngine, transcribeViaRustEngine } from '../engine/rust-asr.js';
 import type { TranscriptSegment } from '../types/timeline.js';
-import { isBinaryAvailable } from './exec.js';
+import { execBinary, isBinaryAvailable } from './exec.js';
 
 const ASR_CANDIDATES = ['whisper-cli', 'whisper-cpp', 'whisper', 'vosk-transcriber'] as const;
 
@@ -13,6 +17,124 @@ export const detectAsrAdapter = async (): Promise<string | null> => {
   return null;
 };
 
+/** Parse whisper.txt style lines: [00:00.000 --> 00:01.000] text */
+export function parseWhisperTxt(raw: string): TranscriptSegment[] {
+  const segments: TranscriptSegment[] = [];
+  const re =
+    /\[(\d{1,2}:)?(\d{1,2}):(\d{1,2})(?:\.(\d{1,3}))?\s*-->\s*(\d{1,2}:)?(\d{1,2}):(\d{1,2})(?:\.(\d{1,3}))?\]\s*(.+)/g;
+  const toMs = (h: string | undefined, min: string, sec: string, ms: string | undefined) => {
+    const hours = h ? Number.parseInt(h.replace(':', ''), 10) : 0;
+    const minutes = Number.parseInt(min, 10);
+    const seconds = Number.parseInt(sec, 10);
+    const millis = ms ? Number.parseInt(ms.padEnd(3, '0').slice(0, 3), 10) : 0;
+    return ((hours * 60 + minutes) * 60 + seconds) * 1000 + millis;
+  };
+  let m = re.exec(raw);
+  while (m !== null) {
+    const start = toMs(m[1], m[2] ?? '0', m[3] ?? '0', m[4]);
+    const end = toMs(m[5], m[6] ?? '0', m[7] ?? '0', m[8]);
+    const text = (m[9] ?? '').trim();
+    if (text) {
+      segments.push({
+        start_ms: start,
+        end_ms: end,
+        text,
+        provenance: { method: 'asr_adapter', adapter: 'whisper-cli' },
+      });
+    }
+    m = re.exec(raw);
+  }
+  if (segments.length === 0) {
+    const lines = raw
+      .split(/\r?\n/)
+      .map((l) => l.trim())
+      .filter((l) => l && !l.startsWith('['));
+    lines.forEach((text, i) => {
+      segments.push({
+        start_ms: i * 1000,
+        end_ms: (i + 1) * 1000,
+        text,
+        provenance: { method: 'asr_adapter', adapter: 'whisper-cli' },
+      });
+    });
+  }
+  return segments;
+}
+
+async function extractWavMono16k(
+  videoPath: string
+): Promise<{ wavPath: string; dir: string } | { error: string }> {
+  const ffmpegOk = await isBinaryAvailable('ffmpeg');
+  if (!ffmpegOk) {
+    return { error: 'ffmpeg required to extract audio for local whisper ASR' };
+  }
+  const dir = mkdtempSync(join(tmpdir(), 'cue-asr-'));
+  const wavPath = join(dir, 'audio.wav');
+  try {
+    await execBinary(
+      'ffmpeg',
+      ['-hide_banner', '-y', '-i', videoPath, '-ac', '1', '-ar', '16000', '-f', 'wav', wavPath],
+      { timeoutMs: 300_000 }
+    );
+    return { wavPath, dir };
+  } catch (error: unknown) {
+    rmSync(dir, { recursive: true, force: true });
+    const message = error instanceof Error ? error.message : String(error);
+    return { error: message };
+  }
+}
+
+/** Local-first frontier ASR: whisper-cli/whisper.cpp on PATH (no cloud). */
+export async function runLocalWhisperTranscript(
+  videoPath: string,
+  adapter: string
+): Promise<{ transcript: TranscriptSegment[]; warning?: string }> {
+  const wav = await extractWavMono16k(videoPath);
+  if ('error' in wav) {
+    return { transcript: [], warning: `ASR audio extract failed: ${wav.error}` };
+  }
+
+  const outBase = join(wav.dir, 'out');
+  const model = process.env.CUE_WHISPER_MODEL ?? process.env.WHISPER_MODEL ?? '';
+  const args: string[] = ['-f', wav.wavPath, '-otxt', '-of', outBase];
+  if (model) args.push('-m', model);
+
+  try {
+    const result = spawnSync(adapter, args, {
+      encoding: 'utf8',
+      timeout: 600_000,
+      windowsHide: true,
+      maxBuffer: 20 * 1024 * 1024,
+    });
+    if (result.status !== 0) {
+      const stderr = typeof result.stderr === 'string' ? result.stderr.trim() : '';
+      return {
+        transcript: [],
+        warning: `Local ASR (${adapter}) failed: ${stderr || `exit ${String(result.status)}`}`,
+      };
+    }
+    let raw = '';
+    try {
+      raw = readFileSync(`${outBase}.txt`, 'utf8');
+    } catch {
+      raw = typeof result.stdout === 'string' ? result.stdout : '';
+    }
+    const transcript = parseWhisperTxt(raw).map((seg) => ({
+      ...seg,
+      provenance: { method: 'asr_adapter' as const, adapter },
+    }));
+    if (transcript.length === 0) {
+      return { transcript: [], warning: `Local ASR (${adapter}) produced empty transcript` };
+    }
+    return { transcript };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { transcript: [], warning: `Local ASR (${adapter}) error: ${message}` };
+  } finally {
+    rmSync(wav.dir, { recursive: true, force: true });
+  }
+}
+
 export const tryAsrTranscript = async (
   videoPath: string,
   enabled: boolean
@@ -29,18 +151,12 @@ export const tryAsrTranscript = async (
         ...(response.result.warning ? { warning: response.result.warning } : {}),
       };
     }
-
-    if (response.code === 'ADAPTER_UNAVAILABLE') {
+    if (response.code !== 'ADAPTER_UNAVAILABLE') {
       return {
         transcript: [],
-        warning: `${response.message} Checked whisper-cli and whisper-cpp.`,
+        warning: `ASR transcription failed: ${response.message}`,
       };
     }
-
-    return {
-      transcript: [],
-      warning: `ASR transcription failed: ${response.message}`,
-    };
   }
 
   const adapter = await detectAsrAdapter();
@@ -48,12 +164,16 @@ export const tryAsrTranscript = async (
     return {
       transcript: [],
       warning:
-        'ASR requested but no local adapter found (checked whisper-cli, whisper-cpp, whisper, vosk-transcriber); transcript skipped.',
+        'ASR requested but no local adapter found (checked whisper-cli, whisper-cpp, whisper, vosk-transcriber). Install whisper.cpp for local-first frontier ASR.',
     };
   }
 
+  if (adapter === 'whisper-cli' || adapter === 'whisper-cpp' || adapter === 'whisper') {
+    return runLocalWhisperTranscript(videoPath, adapter);
+  }
+
   return {
     transcript: [],
-    warning: `ASR adapter "${adapter}" detected but Rust ASR engine is not enabled. Build video-reader-cli or set VIDEO_READER_USE_RUST_ASR=1.`,
+    warning: `ASR adapter "${adapter}" detected but no local runner implemented for it yet.`,
   };
 };