@sylphx/video-reader-mcp 0.1.0 → 0.1.1

package/src/types/timeline.ts

@@ -0,0 +1,106 @@
+export interface Provenance {
+  source: string;
+  tool: 'read_video';
+  version: string;
+  extracted_at: string;
+  source_hash?: string;
+  cache_key?: string;
+  assembly_route?: string;
+}
+
+export interface StreamInfo {
+  index: number;
+  codec_type: string;
+  codec_name?: string;
+  language?: string;
+  channels?: number;
+  sample_rate?: number;
+  width?: number;
+  height?: number;
+  avg_frame_rate?: string;
+  r_frame_rate?: string;
+  bit_rate?: number;
+  disposition?: Record<string, number>;
+  tags?: Record<string, string>;
+}
+
+export interface ChapterInfo {
+  id: number;
+  start_ms: number;
+  end_ms: number;
+  title?: string;
+}
+
+export interface SceneInfo {
+  index: number;
+  time_ms: number;
+  provenance: {
+    method: 'ffmpeg_scene_filter';
+    threshold: number;
+  };
+}
+
+export interface SubtitleCue {
+  index: number;
+  start_ms: number;
+  end_ms: number;
+  text: string;
+  stream_index?: number;
+  language?: string;
+  provenance: {
+    method: 'ffmpeg_extract' | 'embedded_parse';
+    format: 'srt' | 'vtt' | 'webvtt';
+  };
+}
+
+export interface TranscriptSegment {
+  start_ms: number;
+  end_ms: number;
+  text: string;
+  provenance: {
+    method: 'asr_adapter';
+    adapter: string;
+  };
+}
+
+export interface FrameEvidence {
+  index: number;
+  time_ms: number;
+  provenance: {
+    method: 'ffmpeg_keyframe_select';
+    pict_type: 'I';
+  };
+  route?: string;
+  frame_hash?: string;
+  mime?: string;
+  width?: number;
+  height?: number;
+  image_base64?: string;
+}
+
+export interface FormatInfo {
+  format_name?: string;
+  duration_ms: number;
+  bit_rate?: number;
+  size_bytes?: number;
+  tags?: Record<string, string>;
+}
+
+export interface TimelineDocument {
+  provenance: Provenance;
+  format: FormatInfo;
+  streams: StreamInfo[];
+  chapters: ChapterInfo[];
+  scenes: SceneInfo[];
+  subtitles: SubtitleCue[];
+  transcript: TranscriptSegment[];
+  keyframes: FrameEvidence[];
+  warnings: string[];
+}
+
+export interface VideoSourceResult {
+  source: string;
+  success: boolean;
+  data?: TimelineDocument;
+  error?: string;
+}

package/src/utils/asr.ts

@@ -0,0 +1,59 @@
+import { shouldUseRustAsrEngine, transcribeViaRustEngine } from '../engine/rust-asr.js';
+import type { TranscriptSegment } from '../types/timeline.js';
+import { isBinaryAvailable } from './exec.js';
+
+const ASR_CANDIDATES = ['whisper-cli', 'whisper-cpp', 'whisper', 'vosk-transcriber'] as const;
+
+export const detectAsrAdapter = async (): Promise<string | null> => {
+  for (const candidate of ASR_CANDIDATES) {
+    if (await isBinaryAvailable(candidate)) {
+      return candidate;
+    }
+  }
+  return null;
+};
+
+export const tryAsrTranscript = async (
+  videoPath: string,
+  enabled: boolean
+): Promise<{ transcript: TranscriptSegment[]; warning?: string }> => {
+  if (!enabled) {
+    return { transcript: [] };
+  }
+
+  if (shouldUseRustAsrEngine()) {
+    const response = transcribeViaRustEngine(videoPath);
+    if (response.ok) {
+      return {
+        transcript: response.result.transcript,
+        ...(response.result.warning ? { warning: response.result.warning } : {}),
+      };
+    }
+
+    if (response.code === 'ADAPTER_UNAVAILABLE') {
+      return {
+        transcript: [],
+        warning: `${response.message} Checked whisper-cli and whisper-cpp.`,
+      };
+    }
+
+    return {
+      transcript: [],
+      warning: `ASR transcription failed: ${response.message}`,
+    };
+  }
+
+  const adapter = await detectAsrAdapter();
+  if (!adapter) {
+    return {
+      transcript: [],
+      warning:
+        'ASR requested but no local adapter found (checked whisper-cli, whisper-cpp, whisper, vosk-transcriber); transcript skipped.',
+    };
+  }
+
+  return {
+    transcript: [],
+    warning: `ASR adapter "${adapter}" detected but Rust ASR engine is not enabled. Build video-reader-cli or set VIDEO_READER_USE_RUST_ASR=1.`,
+  };
+};

package/src/utils/errors.ts

@@ -0,0 +1,16 @@
+export enum ErrorCode {
+  InvalidParams = -32602,
+  InvalidRequest = -32600,
+  MethodNotFound = -32601,
+}
+
+export class VideoError extends Error {
+  constructor(
+    public code: ErrorCode,
+    message: string,
+    options?: { cause?: Error | undefined }
+  ) {
+    super(message, options?.cause ? { cause: options.cause } : undefined);
+    this.name = 'VideoError';
+  }
+}

package/src/utils/exec.ts

@@ -0,0 +1,76 @@
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
+
+export interface ExecResult {
+  stdout: string;
+  stderr: string;
+}
+
+export interface ExecOptions {
+  timeoutMs?: number;
+  maxBuffer?: number;
+}
+
+const DEFAULT_TIMEOUT_MS = 120_000;
+const DEFAULT_MAX_BUFFER = 16 * 1024 * 1024;
+
+export const execBinary = async (
+  binary: string,
+  args: readonly string[],
+  options: ExecOptions = {}
+): Promise<ExecResult> => {
+  const timeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxBuffer = options.maxBuffer ?? DEFAULT_MAX_BUFFER;
+
+  try {
+    const { stdout, stderr } = await execFileAsync(binary, [...args], {
+      timeout,
+      maxBuffer,
+      encoding: 'utf8',
+    });
+    return {
+      stdout: typeof stdout === 'string' ? stdout : String(stdout),
+      stderr: typeof stderr === 'string' ? stderr : String(stderr),
+    };
+  } catch (error: unknown) {
+    if (
+      typeof error === 'object' &&
+      error !== null &&
+      'killed' in error &&
+      error.killed === true &&
+      'signal' in error &&
+      error.signal === 'SIGTERM'
+    ) {
+      throw new Error(`${binary} timed out after ${timeout}ms`);
+    }
+    throw error;
+  }
+};
+
+const binaryCache = new Map<string, boolean>();
+
+export const isBinaryAvailable = async (binary: string): Promise<boolean> => {
+  const cached = binaryCache.get(binary);
+  if (cached !== undefined) return cached;
+
+  try {
+    await execFileAsync(binary, ['-version'], { timeout: 5_000, encoding: 'utf8' });
+    binaryCache.set(binary, true);
+    return true;
+  } catch {
+    try {
+      await execFileAsync(binary, ['--version'], { timeout: 5_000, encoding: 'utf8' });
+      binaryCache.set(binary, true);
+      return true;
+    } catch {
+      binaryCache.set(binary, false);
+      return false;
+    }
+  }
+};
+
+export const clearBinaryCache = (): void => {
+  binaryCache.clear();
+};

package/src/utils/ffmpeg.ts

@@ -0,0 +1,81 @@
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import type { SubtitleCue } from '../types/timeline.js';
+import { execBinary, isBinaryAvailable } from './exec.js';
+import type { FfprobeStream } from './ffprobe.js';
+import { parseSubtitleContent } from './subtitles.js';
+
+const subtitleFormatForStream = (stream: FfprobeStream): 'srt' | 'vtt' | 'webvtt' => {
+  const codec = stream.codec_name?.toLowerCase() ?? '';
+  if (codec.includes('webvtt') || codec === 'vtt') return 'vtt';
+  return 'srt';
+};
+
+export const extractSubtitles = async (
+  videoPath: string,
+  subtitleStreams: FfprobeStream[]
+): Promise<{ subtitles: SubtitleCue[]; warnings: string[] }> => {
+  const warnings: string[] = [];
+  const subtitles: SubtitleCue[] = [];
+
+  if (subtitleStreams.length === 0) {
+    return { subtitles, warnings: ['No embedded subtitle streams found.'] };
+  }
+
+  const available = await isBinaryAvailable('ffmpeg');
+  if (!available) {
+    return {
+      subtitles,
+      warnings: ['ffmpeg is not installed; subtitle extraction skipped.'],
+    };
+  }
+
+  const tempDir = await mkdtemp(path.join(os.tmpdir(), 'video-reader-mcp-'));
+
+  try {
+    for (const stream of subtitleStreams) {
+      const format = subtitleFormatForStream(stream);
+      const extension = format === 'srt' ? 'srt' : 'vtt';
+      const outputPath = path.join(tempDir, `sub-${stream.index}.${extension}`);
+
+      try {
+        await execBinary(
+          'ffmpeg',
+          [
+            '-hide_banner',
+            '-y',
+            '-i',
+            videoPath,
+            '-map',
+            `0:${stream.index}`,
+            '-c:s',
+            format === 'srt' ? 'srt' : 'webvtt',
+            outputPath,
+          ],
+          { timeoutMs: 120_000 }
+        );
+
+        const content = await readFile(outputPath, 'utf8');
+        const cues = parseSubtitleContent(content, format).map((cue) => ({
+          ...cue,
+          index: subtitles.length + cue.index,
+          stream_index: stream.index,
+          ...(stream.tags?.language ? { language: stream.tags.language } : {}),
+          provenance: {
+            method: 'ffmpeg_extract' as const,
+            format,
+          },
+        }));
+        subtitles.push(...cues);
+      } catch (error: unknown) {
+        const message = error instanceof Error ? error.message : String(error);
+        warnings.push(`Subtitle stream ${stream.index} extraction failed: ${message}`);
+      }
+    }
+  } finally {
+    await rm(tempDir, { recursive: true, force: true });
+  }
+
+  return { subtitles, warnings };
+};

package/src/utils/ffprobe.ts

@@ -0,0 +1,137 @@
+import { execBinary, isBinaryAvailable } from './exec.js';
+
+export interface FfprobeChapter {
+  id: number;
+  start: number;
+  end: number;
+  tags?: Record<string, string>;
+}
+
+export interface FfprobeStream {
+  index: number;
+  codec_type: string;
+  codec_name?: string;
+  tags?: Record<string, string>;
+  channels?: number;
+  sample_rate?: string;
+  width?: number;
+  height?: number;
+  avg_frame_rate?: string;
+  r_frame_rate?: string;
+  bit_rate?: string;
+  disposition?: Record<string, number>;
+}
+
+export interface FfprobeFormat {
+  format_name?: string;
+  duration?: string;
+  bit_rate?: string;
+  size?: string;
+  tags?: Record<string, string>;
+}
+
+export interface FfprobeResult {
+  streams: FfprobeStream[];
+  format: FfprobeFormat;
+  chapters?: FfprobeChapter[];
+}
+
+export const parseFfprobeJson = (raw: string): FfprobeResult => {
+  const parsed = JSON.parse(raw) as Partial<FfprobeResult>;
+  return {
+    streams: Array.isArray(parsed.streams) ? parsed.streams : [],
+    format: parsed.format ?? {},
+    chapters: Array.isArray(parsed.chapters) ? parsed.chapters : [],
+  };
+};
+
+export const secondsToMs = (value: string | number | undefined): number => {
+  if (value === undefined) return 0;
+  const seconds = typeof value === 'number' ? value : Number.parseFloat(value);
+  if (!Number.isFinite(seconds)) return 0;
+  return Math.round(seconds * 1000);
+};
+
+export const mapStreams = (streams: FfprobeStream[]) =>
+  streams.map((stream) => ({
+    index: stream.index,
+    codec_type: stream.codec_type,
+    ...(stream.codec_name ? { codec_name: stream.codec_name } : {}),
+    ...(stream.tags?.language ? { language: stream.tags.language } : {}),
+    ...(stream.channels !== undefined ? { channels: stream.channels } : {}),
+    ...(stream.sample_rate ? { sample_rate: Number.parseInt(stream.sample_rate, 10) } : {}),
+    ...(stream.width !== undefined ? { width: stream.width } : {}),
+    ...(stream.height !== undefined ? { height: stream.height } : {}),
+    ...(stream.avg_frame_rate ? { avg_frame_rate: stream.avg_frame_rate } : {}),
+    ...(stream.r_frame_rate ? { r_frame_rate: stream.r_frame_rate } : {}),
+    ...(stream.bit_rate ? { bit_rate: Number.parseInt(stream.bit_rate, 10) } : {}),
+    ...(stream.disposition ? { disposition: stream.disposition } : {}),
+    ...(stream.tags ? { tags: stream.tags } : {}),
+  }));
+
+export const mapChapters = (chapters: FfprobeChapter[] | undefined) =>
+  (chapters ?? []).map((chapter) => ({
+    id: chapter.id,
+    start_ms: secondsToMs(chapter.start),
+    end_ms: secondsToMs(chapter.end),
+    ...(chapter.tags?.title ? { title: chapter.tags.title } : {}),
+  }));
+
+export const collectProbeWarnings = (probe: FfprobeResult, includeStreams: boolean): string[] => {
+  const warnings: string[] = [];
+  const videoStreams = probe.streams.filter((s) => s.codec_type === 'video');
+  const audioStreams = probe.streams.filter((s) => s.codec_type === 'audio');
+
+  if (includeStreams && videoStreams.length === 0) {
+    warnings.push('No video stream detected.');
+  }
+  if (includeStreams && audioStreams.length === 0) {
+    warnings.push('No audio stream detected.');
+  }
+
+  for (const stream of videoStreams) {
+    if (
+      stream.avg_frame_rate &&
+      stream.r_frame_rate &&
+      stream.avg_frame_rate !== stream.r_frame_rate
+    ) {
+      warnings.push(
+        `Stream ${stream.index}: variable frame rate suspected (avg_frame_rate=${stream.avg_frame_rate}, r_frame_rate=${stream.r_frame_rate}).`
+      );
+    }
+  }
+
+  const duration = probe.format.duration;
+  if (!duration || Number.parseFloat(duration) <= 0) {
+    warnings.push('Duration unavailable or zero; timeline bounds may be incomplete.');
+  }
+
+  return warnings;
+};
+
+export const runFfprobe = async (videoPath: string): Promise<FfprobeResult> => {
+  const available = await isBinaryAvailable('ffprobe');
+  if (!available) {
+    throw new Error('ffprobe is not installed or not on PATH');
+  }
+
+  const { stdout } = await execBinary(
+    'ffprobe',
+    [
+      '-v',
+      'quiet',
+      '-print_format',
+      'json',
+      '-show_format',
+      '-show_streams',
+      '-show_chapters',
+      videoPath,
+    ],
+    { timeoutMs: 60_000 }
+  );
+
+  return parseFfprobeJson(stdout);
+};
+
+export const findSubtitleStreams = (streams: FfprobeStream[]): FfprobeStream[] =>
+  streams.filter((stream) => stream.codec_type === 'subtitle');

package/src/utils/frameOcr.ts

@@ -0,0 +1,99 @@
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+export type FrameOcrLine = {
+  text: string;
+  confidence?: number;
+};
+
+export type FrameOcrResult = {
+  available: boolean;
+  route: string;
+  skipped_reason?: string;
+  languages: string[];
+  lines: FrameOcrLine[];
+  line_count: number;
+};
+
+export const isTesseractAvailable = (): boolean => {
+  const result = spawnSync('tesseract', ['--version'], {
+    timeout: 2_500,
+    windowsHide: true,
+    stdio: 'ignore',
+  });
+  return result.status === 0;
+};
+
+/** OCR a PNG buffer (base64) with local Tesseract; honest skip when missing. */
+export const ocrPngBase64 = (
+  imageBase64: string,
+  languages: string[] = ['eng']
+): FrameOcrResult => {
+  const langs = languages.length ? languages : ['eng'];
+  if (!isTesseractAvailable()) {
+    return {
+      available: false,
+      route: 'tesseract_frame',
+      skipped_reason: 'Tesseract is not installed or not available on PATH.',
+      languages: langs,
+      lines: [],
+      line_count: 0,
+    };
+  }
+
+  const dir = mkdtempSync(join(tmpdir(), 'cue-ocr-'));
+  const pngPath = join(dir, 'frame.png');
+  try {
+    writeFileSync(pngPath, Buffer.from(imageBase64, 'base64'));
+    const languageArg = langs.join('+');
+    const result = spawnSync('tesseract', [pngPath, 'stdout', '-l', languageArg, '--psm', '6'], {
+      encoding: 'utf8',
+      timeout: 60_000,
+      windowsHide: true,
+      maxBuffer: 5 * 1024 * 1024,
+    });
+    if (result.error) {
+      return {
+        available: false,
+        route: 'tesseract_frame',
+        skipped_reason: `Tesseract failed to start: ${result.error.message}`,
+        languages: langs,
+        lines: [],
+        line_count: 0,
+      };
+    }
+    if (result.status !== 0) {
+      const stderr = typeof result.stderr === 'string' ? result.stderr.trim() : '';
+      return {
+        available: false,
+        route: 'tesseract_frame',
+        skipped_reason:
+          stderr.length > 0 ? stderr : `Tesseract exited with status ${String(result.status)}.`,
+        languages: langs,
+        lines: [],
+        line_count: 0,
+      };
+    }
+    const text = typeof result.stdout === 'string' ? result.stdout : '';
+    const lines = text
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0)
+      .map((line) => ({ text: line }));
+    return {
+      available: true,
+      route: 'tesseract_frame',
+      languages: langs,
+      lines,
+      line_count: lines.length,
+    };
+  } finally {
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      // ignore cleanup errors
+    }
+  }
+};

package/src/utils/frames.ts

@@ -0,0 +1,93 @@
+import { extractKeyframesViaRustEngine, shouldUseRustFramesEngine } from '../engine/rust-frames.js';
+import type { FrameEvidence } from '../types/timeline.js';
+import { execBinary, isBinaryAvailable } from './exec.js';
+
+const KEYFRAME_PTS_TIME_RE = /pts_time:([0-9.]+)/g;
+
+export const parseKeyframeFilterOutput = (stderr: string): FrameEvidence[] => {
+  const keyframes: FrameEvidence[] = [];
+  let match: RegExpExecArray | null = KEYFRAME_PTS_TIME_RE.exec(stderr);
+
+  while (match !== null) {
+    const seconds = Number.parseFloat(match[1]);
+    if (!Number.isFinite(seconds)) continue;
+
+    keyframes.push({
+      index: keyframes.length,
+      time_ms: Math.round(seconds * 1000),
+      provenance: {
+        method: 'ffmpeg_keyframe_select',
+        pict_type: 'I',
+      },
+    });
+    match = KEYFRAME_PTS_TIME_RE.exec(stderr);
+  }
+
+  return keyframes;
+};
+
+export const extractKeyframes = async (
+  videoPath: string,
+  limit = 8,
+  options: {
+    includeImages?: boolean | undefined;
+    maxDimension?: number | undefined;
+  } = {}
+): Promise<{ keyframes: FrameEvidence[]; warning?: string }> => {
+  const boundedLimit = Math.max(1, Math.min(limit, 64));
+  const includeImages = options.includeImages ?? false;
+
+  if (shouldUseRustFramesEngine()) {
+    const response = extractKeyframesViaRustEngine({
+      videoPath,
+      limit: boundedLimit,
+      includeImages,
+      ...(options.maxDimension !== undefined ? { maxDimension: options.maxDimension } : {}),
+    });
+
+    if (response.ok) {
+      return { keyframes: response.keyframes };
+    }
+
+    return {
+      keyframes: [],
+      warning: `Keyframe extraction failed: ${response.message}`,
+    };
+  }
+
+  const available = await isBinaryAvailable('ffmpeg');
+  if (!available) {
+    return {
+      keyframes: [],
+      warning: 'ffmpeg is not installed; keyframe index extraction skipped.',
+    };
+  }
+
+  try {
+    const { stderr } = await execBinary(
+      'ffmpeg',
+      [
+        '-hide_banner',
+        '-i',
+        videoPath,
+        '-vf',
+        "select='eq(pict_type,I)',showinfo",
+        '-vsync',
+        'vfr',
+        '-f',
+        'null',
+        '-',
+      ],
+      { timeoutMs: 300_000 }
+    );
+
+    const keyframes = parseKeyframeFilterOutput(stderr).slice(0, boundedLimit);
+    return { keyframes };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      keyframes: [],
+      warning: `Keyframe extraction failed: ${message}`,
+    };
+  }
+};

package/src/utils/pathUtils.ts

@@ -0,0 +1,36 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { ErrorCode, VideoError } from './errors.js';
+
+export const PROJECT_ROOT = process.cwd();
+
+const canonicalize = (p: string): string => {
+  try {
+    return fs.realpathSync(p);
+  } catch (err: unknown) {
+    if (
+      typeof err === 'object' &&
+      err !== null &&
+      'code' in err &&
+      (err.code === 'ENOENT' || err.code === 'ENOTDIR')
+    ) {
+      const parent = path.dirname(p);
+      if (parent === p) return p;
+      return path.join(canonicalize(parent), path.basename(p));
+    }
+    throw err;
+  }
+};
+
+export const resolvePath = (userPath: string): string => {
+  if (typeof userPath !== 'string') {
+    throw new VideoError(ErrorCode.InvalidParams, 'Path must be a string.');
+  }
+
+  const normalizedUserPath = path.normalize(userPath);
+  const resolved = path.isAbsolute(normalizedUserPath)
+    ? normalizedUserPath
+    : path.resolve(PROJECT_ROOT, normalizedUserPath);
+
+  return canonicalize(resolved);
+};