@sylphx/video-reader-mcp 0.1.0 → 0.1.1

package/src/doctor.ts

@@ -0,0 +1,132 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { resolveRustCliBinary } from './engine/rust-timeline.js';
+import { isBinaryAvailable } from './utils/exec.js';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+export type DoctorStatus = 'ok' | 'warn' | 'fail';
+
+export interface DoctorCheck {
+  id: string;
+  status: DoctorStatus;
+  message: string;
+}
+
+export interface DoctorReport {
+  profile: 'video_reader_doctor';
+  version: string;
+  status: 'ready' | 'degraded' | 'unavailable';
+  checks: DoctorCheck[];
+}
+
+const probeBinary = async (id: string, binary: string, required: boolean): Promise<DoctorCheck> => {
+  const available = await isBinaryAvailable(binary);
+  if (available) {
+    return {
+      id,
+      status: 'ok',
+      message: `${binary} is installed and responds to -version.`,
+    };
+  }
+
+  return {
+    id,
+    status: required ? 'fail' : 'warn',
+    message: required
+      ? `${binary} is required for read_video but was not found on PATH.`
+      : `${binary} is optional for some extraction paths.`,
+  };
+};
+
+const probeRustTimelineCli = (): DoctorCheck => {
+  const binary = resolveRustCliBinary();
+  if (binary !== 'video-reader-cli' && existsSync(binary)) {
+    const probe = spawnSync(binary, [], {
+      input: JSON.stringify({
+        tool: 'build_cache_key',
+        input: {
+          source_hash: 'abc123',
+          options: { include_streams: true, include_chapters: true },
+        },
+      }),
+      encoding: 'utf8',
+      timeout: 5_000,
+    });
+
+    if (probe.status === 0) {
+      return {
+        id: 'rust_timeline_cli',
+        status: 'ok',
+        message: `Rust timeline CLI is available at ${binary}.`,
+      };
+    }
+  }
+
+  const release = path.join(here, '../target/release/video-reader-cli');
+  const debug = path.join(here, '../target/debug/video-reader-cli');
+  if (existsSync(release) || existsSync(debug)) {
+    return {
+      id: 'rust_timeline_cli',
+      status: 'ok',
+      message: 'Rust timeline CLI is built locally.',
+    };
+  }
+
+  return {
+    id: 'rust_timeline_cli',
+    status: 'warn',
+    message:
+      'Rust timeline CLI is not built. Run `cargo build --release` to enable VIDEO_READER_USE_RUST_TIMELINE=1.',
+  };
+};
+
+const probeNode = (): DoctorCheck => {
+  const version = process.versions.node;
+  const major = Number.parseInt(version.split('.')[0] ?? '0', 10);
+  if (major >= 22) {
+    return {
+      id: 'node',
+      status: 'ok',
+      message: `Node.js ${version} meets the >=22.13 requirement.`,
+    };
+  }
+
+  return {
+    id: 'node',
+    status: 'warn',
+    message: `Node.js ${version} is below the recommended >=22.13 runtime.`,
+  };
+};
+
+const aggregateStatus = (checks: DoctorCheck[]): DoctorReport['status'] => {
+  if (checks.some((check) => check.status === 'fail')) {
+    return 'unavailable';
+  }
+  if (checks.some((check) => check.status === 'warn')) {
+    return 'degraded';
+  }
+  return 'ready';
+};
+
+export async function runDoctor(version: string): Promise<DoctorReport> {
+  const checks = [
+    probeNode(),
+    probeRustTimelineCli(),
+    await probeBinary('ffprobe', 'ffprobe', true),
+    await probeBinary('ffmpeg', 'ffmpeg', false),
+  ];
+
+  return {
+    profile: 'video_reader_doctor',
+    version,
+    status: aggregateStatus(checks),
+    checks,
+  };
+}
+
+export function formatDoctorReport(report: DoctorReport): string {
+  return JSON.stringify(report, null, 2);
+}

package/src/engine/rust-asr.ts

@@ -0,0 +1,98 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import type { TranscriptSegment } from '../types/timeline.js';
+
+export type RustAsrResult = {
+  transcript: TranscriptSegment[];
+  route: string;
+  adapter?: string;
+  warning?: string;
+};
+
+type RustAsrEnvelope =
+  | { status: 'ok'; asr: RustAsrResult }
+  | { status: 'error'; code: string; message: string };
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+export function resolveRustCliBinary(): string {
+  const env = process.env.VIDEO_READER_CLI;
+  if (env && existsSync(env)) {
+    return env;
+  }
+
+  const release = path.join(here, '../../target/release/video-reader-cli');
+  if (existsSync(release)) {
+    return release;
+  }
+
+  const debug = path.join(here, '../../target/debug/video-reader-cli');
+  if (existsSync(debug)) {
+    return debug;
+  }
+
+  return 'video-reader-cli';
+}
+
+export function isRustCliAvailable(): boolean {
+  return resolveRustCliBinary() !== 'video-reader-cli';
+}
+
+export function shouldUseRustAsrEngine(): boolean {
+  if (process.env.VIDEO_READER_USE_RUST_ASR === '0') {
+    return false;
+  }
+  if (process.env.VIDEO_READER_USE_RUST_ASR === '1') {
+    return true;
+  }
+  return isRustCliAvailable();
+}
+
+export function transcribeViaRustEngine(
+  videoPath: string,
+  maxAudioSeconds = 300
+): { ok: true; result: RustAsrResult } | { ok: false; code: string; message: string } {
+  const binary = resolveRustCliBinary();
+  const payload = JSON.stringify({
+    tool: 'transcribe_asr',
+    input: {
+      path: videoPath,
+      max_audio_seconds: maxAudioSeconds,
+    },
+  });
+
+  const response = spawnSync(binary, [], {
+    input: payload,
+    encoding: 'utf8',
+    maxBuffer: 16 * 1024 * 1024,
+  });
+
+  if (response.error) {
+    return {
+      ok: false,
+      code: 'ENGINE_UNAVAILABLE',
+      message: response.error.message,
+    };
+  }
+
+  if (response.status !== 0) {
+    return {
+      ok: false,
+      code: 'ENGINE_FAILED',
+      message: response.stderr || `Rust ASR engine exited with status ${response.status}`,
+    };
+  }
+
+  const envelope = JSON.parse(response.stdout) as RustAsrEnvelope;
+  if (envelope.status !== 'ok') {
+    return {
+      ok: false,
+      code: envelope.code,
+      message: envelope.message,
+    };
+  }
+
+  return { ok: true, result: envelope.asr };
+}

package/src/engine/rust-frames.ts

@@ -0,0 +1,95 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import type { FrameEvidence } from '../types/timeline.js';
+
+type RustKeyframeEnvelope =
+  | { status: 'ok'; keyframes: FrameEvidence[] }
+  | { status: 'error'; code: string; message: string };
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+export function resolveRustCliBinary(): string {
+  const env = process.env.VIDEO_READER_CLI;
+  if (env && existsSync(env)) {
+    return env;
+  }
+
+  const release = path.join(here, '../../target/release/video-reader-cli');
+  if (existsSync(release)) {
+    return release;
+  }
+
+  const debug = path.join(here, '../../target/debug/video-reader-cli');
+  if (existsSync(debug)) {
+    return debug;
+  }
+
+  return 'video-reader-cli';
+}
+
+export function isRustCliAvailable(): boolean {
+  return resolveRustCliBinary() !== 'video-reader-cli';
+}
+
+export function shouldUseRustFramesEngine(): boolean {
+  if (process.env.VIDEO_READER_USE_RUST_FRAMES === '0') {
+    return false;
+  }
+  if (process.env.VIDEO_READER_USE_RUST_FRAMES === '1') {
+    return true;
+  }
+  return isRustCliAvailable();
+}
+
+export function extractKeyframesViaRustEngine(input: {
+  videoPath: string;
+  limit: number;
+  includeImages: boolean;
+  maxDimension?: number | undefined;
+}): { ok: true; keyframes: FrameEvidence[] } | { ok: false; code: string; message: string } {
+  const binary = resolveRustCliBinary();
+  const payload = JSON.stringify({
+    tool: 'extract_keyframes',
+    input: {
+      path: input.videoPath,
+      limit: input.limit,
+      include_images: input.includeImages,
+      ...(input.maxDimension !== undefined ? { max_dimension: input.maxDimension } : {}),
+    },
+  });
+
+  const response = spawnSync(binary, [], {
+    input: payload,
+    encoding: 'utf8',
+    maxBuffer: 32 * 1024 * 1024,
+  });
+
+  if (response.error) {
+    return {
+      ok: false,
+      code: 'ENGINE_UNAVAILABLE',
+      message: response.error.message,
+    };
+  }
+
+  if (response.status !== 0) {
+    return {
+      ok: false,
+      code: 'ENGINE_FAILED',
+      message: response.stderr || `Rust frames engine exited with status ${response.status}`,
+    };
+  }
+
+  const envelope = JSON.parse(response.stdout) as RustKeyframeEnvelope;
+  if (envelope.status !== 'ok') {
+    return {
+      ok: false,
+      code: envelope.code,
+      message: envelope.message,
+    };
+  }
+
+  return { ok: true, keyframes: envelope.keyframes };
+}

package/src/engine/rust-timeline.ts

@@ -0,0 +1,164 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import type { FfprobeResult } from '../utils/ffprobe.js';
+
+export type RustProbeTimeline = {
+  format: {
+    format_name?: string;
+    duration_ms: number;
+    bit_rate?: number;
+    size_bytes?: number;
+    tags?: Record<string, string>;
+  };
+  streams: Array<{
+    index: number;
+    codec_type: string;
+    codec_name?: string;
+    language?: string;
+    channels?: number;
+    sample_rate?: number;
+    width?: number;
+    height?: number;
+    avg_frame_rate?: string;
+    r_frame_rate?: string;
+    bit_rate?: number;
+    disposition?: Record<string, number>;
+    tags?: Record<string, string>;
+  }>;
+  chapters: Array<{
+    id: number;
+    start_ms: number;
+    end_ms: number;
+    title?: string;
+  }>;
+  warnings: string[];
+  route: string;
+};
+
+type RustTimelineEnvelope =
+  | { status: 'ok'; timeline: RustProbeTimeline }
+  | { status: 'error'; code: string; message: string };
+
+type RustHashEnvelope =
+  | { status: 'ok'; source_hash: string }
+  | { status: 'error'; code: string; message: string };
+
+type RustCacheKeyEnvelope =
+  | { status: 'ok'; cache_key: string }
+  | { status: 'error'; code: string; message: string };
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+export function resolveRustCliBinary(): string {
+  const env = process.env.VIDEO_READER_CLI;
+  if (env && existsSync(env)) {
+    return env;
+  }
+
+  const release = path.join(here, '../../target/release/video-reader-cli');
+  if (existsSync(release)) {
+    return release;
+  }
+
+  const debug = path.join(here, '../../target/debug/video-reader-cli');
+  if (existsSync(debug)) {
+    return debug;
+  }
+
+  return 'video-reader-cli';
+}
+
+export function shouldUseRustTimelineEngine(): boolean {
+  return process.env.VIDEO_READER_USE_RUST_TIMELINE === '1';
+}
+
+const invokeRustCli = (tool: string, input: Record<string, unknown>): unknown => {
+  const binary = resolveRustCliBinary();
+  const payload = JSON.stringify({ tool, input });
+
+  const result = spawnSync(binary, [], {
+    input: payload,
+    encoding: 'utf8',
+    maxBuffer: 4 * 1024 * 1024,
+  });
+
+  if (result.error) {
+    throw new Error(`Failed to launch video timeline engine: ${result.error.message}`);
+  }
+
+  if (result.status !== 0) {
+    throw new Error(result.stderr || `Video timeline engine exited with status ${result.status}`);
+  }
+
+  return JSON.parse(result.stdout) as unknown;
+};
+
+export function assembleProbeTimelineViaRustEngine(
+  probe: FfprobeResult,
+  options: { includeStreams: boolean; includeChapters: boolean }
+): RustProbeTimeline {
+  const envelope = invokeRustCli('assemble_probe_timeline', {
+    ffprobe: probe,
+    options: {
+      include_streams: options.includeStreams,
+      include_chapters: options.includeChapters,
+    },
+  }) as RustTimelineEnvelope;
+
+  if (envelope.status !== 'ok') {
+    throw new Error(envelope.message);
+  }
+
+  return envelope.timeline;
+}
+
+export function hashSourceViaRustEngine(filePath: string): string {
+  const envelope = invokeRustCli('hash_source', { path: filePath }) as RustHashEnvelope;
+  if (envelope.status !== 'ok') {
+    throw new Error(envelope.message);
+  }
+
+  return envelope.source_hash;
+}
+
+export function buildCacheKeyViaRustEngine(
+  sourceHash: string,
+  options: {
+    includeStreams: boolean;
+    includeChapters: boolean;
+    includeSubtitles: boolean;
+    includeScenes: boolean;
+    includeTranscript: boolean;
+    includeKeyframes: boolean;
+    includeKeyframeImages: boolean;
+    keyframeLimit: number;
+    keyframeMaxDimension?: number | undefined;
+    sceneThreshold: number;
+  }
+): string {
+  const envelope = invokeRustCli('build_cache_key', {
+    source_hash: sourceHash,
+    options: {
+      include_streams: options.includeStreams,
+      include_chapters: options.includeChapters,
+      include_subtitles: options.includeSubtitles,
+      include_scenes: options.includeScenes,
+      include_transcript: options.includeTranscript,
+      include_keyframes: options.includeKeyframes,
+      include_keyframe_images: options.includeKeyframeImages,
+      keyframe_limit: options.keyframeLimit,
+      ...(options.keyframeMaxDimension !== undefined
+        ? { keyframe_max_dimension: options.keyframeMaxDimension }
+        : {}),
+      scene_threshold: options.sceneThreshold,
+    },
+  }) as RustCacheKeyEnvelope;
+
+  if (envelope.status !== 'ok') {
+    throw new Error(envelope.message);
+  }
+
+  return envelope.cache_key;
+}

package/src/engine/rust-video-evidence.ts

@@ -0,0 +1,132 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export type CropRegion = {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+};
+
+export type FrameRenderEvidence = {
+  time_ms: number;
+  route: string;
+  frame_hash: string;
+  mime: string;
+  width: number;
+  height: number;
+  image_base64: string;
+  provenance: {
+    method: string;
+    time_ms: number;
+  };
+  crop?: CropRegion | undefined;
+};
+
+type FrameRenderEnvelope =
+  | { status: 'ok'; frame: FrameRenderEvidence }
+  | { status: 'error'; code: string; message: string };
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+export function resolveRustCliBinary(): string {
+  const env = process.env.VIDEO_READER_CLI;
+  if (env && existsSync(env)) {
+    return env;
+  }
+
+  const release = path.join(here, '../../target/release/video-reader-cli');
+  if (existsSync(release)) {
+    return release;
+  }
+
+  const debug = path.join(here, '../../target/debug/video-reader-cli');
+  if (existsSync(debug)) {
+    return debug;
+  }
+
+  return 'video-reader-cli';
+}
+
+export function isRustCliAvailable(): boolean {
+  return resolveRustCliBinary() !== 'video-reader-cli';
+}
+
+export function shouldUseRustVideoEvidenceEngine(): boolean {
+  if (process.env.VIDEO_READER_USE_RUST_FRAMES === '0') {
+    return false;
+  }
+  if (process.env.VIDEO_READER_USE_RUST_FRAMES === '1') {
+    return true;
+  }
+  return isRustCliAvailable();
+}
+
+function invokeRustFrameTool(
+  tool: 'render_frame' | 'crop_frame',
+  input: Record<string, unknown>
+): { ok: true; frame: FrameRenderEvidence } | { ok: false; code: string; message: string } {
+  const binary = resolveRustCliBinary();
+  const payload = JSON.stringify({ tool, input });
+  const response = spawnSync(binary, [], {
+    input: payload,
+    encoding: 'utf8',
+    maxBuffer: 32 * 1024 * 1024,
+  });
+
+  if (response.error) {
+    return {
+      ok: false,
+      code: 'ENGINE_UNAVAILABLE',
+      message: response.error.message,
+    };
+  }
+
+  if (response.status !== 0) {
+    return {
+      ok: false,
+      code: 'ENGINE_FAILED',
+      message:
+        response.stderr || `Rust video evidence engine exited with status ${response.status}`,
+    };
+  }
+
+  const envelope = JSON.parse(response.stdout) as FrameRenderEnvelope;
+  if (envelope.status !== 'ok') {
+    return {
+      ok: false,
+      code: envelope.code,
+      message: envelope.message,
+    };
+  }
+
+  return { ok: true, frame: envelope.frame };
+}
+
+export function renderFrameViaRustEngine(input: {
+  videoPath: string;
+  timeMs: number;
+  maxDimension?: number | undefined;
+}) {
+  return invokeRustFrameTool('render_frame', {
+    path: input.videoPath,
+    time_ms: input.timeMs,
+    ...(input.maxDimension !== undefined ? { max_dimension: input.maxDimension } : {}),
+  });
+}
+
+export function cropFrameViaRustEngine(input: {
+  videoPath: string;
+  timeMs: number;
+  crop: CropRegion;
+  maxDimension?: number | undefined;
+}) {
+  return invokeRustFrameTool('crop_frame', {
+    path: input.videoPath,
+    time_ms: input.timeMs,
+    crop: input.crop,
+    ...(input.maxDimension !== undefined ? { max_dimension: input.maxDimension } : {}),
+  });
+}

package/src/handlers/readVideo.ts

@@ -0,0 +1,41 @@
+import { text, tool, toolError } from '../mcp.js';
+import { type ReadVideoArgs, readVideoArgsSchema } from '../schemas/readVideo.js';
+import { processVideoSource } from '../video/readCoordinator.js';
+
+const MAX_CONCURRENT_SOURCES = 2;
+
+export const createReadVideoHandler = (version: string) =>
+  tool()
+    .description(
+      'Primary video reader. Returns a timeline document with ffprobe metadata, embedded subtitles, optional scene boundaries, and warnings — no per-frame vision LLM.'
+    )
+    .input(readVideoArgsSchema)
+    .handler(async ({ input }: { input: ReadVideoArgs }) => {
+      const results = [];
+
+      for (let i = 0; i < input.sources.length; i += MAX_CONCURRENT_SOURCES) {
+        const batch = input.sources.slice(i, i + MAX_CONCURRENT_SOURCES);
+        const batchResults = await Promise.all(
+          batch.map((source) => processVideoSource(source.path, input, version))
+        );
+        results.push(...batchResults);
+      }
+
+      const allFailed = results.every((result) => !result.success);
+      if (allFailed) {
+        const errorMessages = results.map((result) => result.error).join('; ');
+        return toolError(`All video sources failed to process: ${errorMessages}`);
+      }
+
+      return text(
+        JSON.stringify(
+          {
+            results,
+          },
+          null,
+          2
+        )
+      );
+    });
+
+export const readVideo = createReadVideoHandler('0.1.0');