@sylphx/video-reader-mcp 0.1.1 → 0.1.3

package/README.md

@@ -36,6 +36,15 @@ Each instrument is an independent repository (marketplace + stars).
 ---
 
 
+
+## Read video structure (not N-second frame spam)
+
+Cue is **local-first timeline architecture**: streams, dialogue, scene cuts, **structural keyframes**, and **agent_index** for text-only agents.
+
+Spec: [docs/specs/agent-video-read-contract.md](docs/specs/agent-video-read-contract.md)
+
+**Local-first frontier:** ffmpeg/ffprobe + structural keyframes, optional local whisper ASR. No cloud required.
+
 ## Product docs
 
 | Doc | Purpose |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sylphx/video-reader-mcp",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "mcpName": "io.github.SylphxAI/video-reader-mcp",
   "description": "Cue \u2014 Evidence-first video reading for AI agents \u2014 ffprobe, subtitles, scenes, transcripts, and timelines without frame-by-frame LLM vision.",
   "type": "module",

package/src/handlers/readVideo.ts

@@ -7,7 +7,7 @@ const MAX_CONCURRENT_SOURCES = 2;
 export const createReadVideoHandler = (version: string) =>
   tool()
     .description(
-      'Primary video reader. Returns a timeline document with ffprobe metadata, embedded subtitles, optional scene boundaries, and warnings — no per-frame vision LLM.'
+      'Primary video reader for agents (read film structure, not sample frames blindly). Timeline: streams, scenes, subtitles, optional ASR, structural keyframe locators, agent_index outline. Local-first; no required vision LLM.'
     )
     .input(readVideoArgsSchema)
     .handler(async ({ input }: { input: ReadVideoArgs }) => {

package/src/schemas/readVideo.ts

@@ -61,6 +61,18 @@ export const readVideoArgsSchema = z.object({
     .positive()
     .optional()
     .describe('Maximum width or height when resizing keyframe PNG evidence.'),
+  keyframe_policy: z
+    .enum(['structural', 'iframes'])
+    .optional()
+    .describe(
+      'structural (default when include_keyframes): scene starts/midpoints. iframes: raw I-frame index only.'
+    ),
+  include_agent_index: z
+    .boolean()
+    .optional()
+    .describe(
+      'Return agent_index outline so text-only agents can read film structure. Defaults to true.'
+    ),
 });
 
 export type ReadVideoArgs = z.infer<typeof readVideoArgsSchema>;

package/src/types/timeline.ts

@@ -96,6 +96,19 @@ export interface TimelineDocument {
   transcript: TranscriptSegment[];
   keyframes: FrameEvidence[];
   warnings: string[];
+  /** Text outline of film architecture for non-vision agents. */
+  agent_index?: {
+    policy: string;
+    source: string;
+    duration_ms?: number;
+    outline: string;
+    scene_count: number;
+    keyframe_count: number;
+    subtitle_cue_count: number;
+    transcript_segment_count: number;
+    audio_stream_count: number;
+    video_stream_count: number;
+  };
 }
 
 export interface VideoSourceResult {

package/src/utils/agentIndex.ts

@@ -0,0 +1,107 @@
+/**
+ * Agent Index: a text-first reformatting of video structure.
+ * Agents "read" the film (timeline architecture), not sample arbitrary frames.
+ */
+
+export type AgentVideoIndex = {
+  policy: 'agent_video_index_v1';
+  source: string;
+  duration_ms?: number;
+  outline: string;
+  scene_count: number;
+  keyframe_count: number;
+  subtitle_cue_count: number;
+  transcript_segment_count: number;
+  audio_stream_count: number;
+  video_stream_count: number;
+};
+
+export function buildAgentVideoIndex(doc: {
+  provenance: { source: string };
+  format?: { duration_ms?: number; format_name?: string };
+  streams?: Array<{ codec_type?: string; codec_name?: string; width?: number; height?: number }>;
+  scenes?: Array<{ time_ms?: number }>;
+  keyframes?: Array<{ time_ms?: number; frame_hash?: string }>;
+  subtitles?: Array<{ cues?: unknown[]; text?: string }>;
+  transcript?: Array<{ text?: string; start_ms?: number }>;
+  warnings?: string[];
+}): AgentVideoIndex {
+  const streams = doc.streams ?? [];
+  const audio = streams.filter((s) => s.codec_type === 'audio');
+  const video = streams.filter((s) => s.codec_type === 'video');
+  const scenes = doc.scenes ?? [];
+  const keyframes = doc.keyframes ?? [];
+  const subtitleCueCount = (doc.subtitles ?? []).reduce((n, s) => {
+    if (Array.isArray(s.cues)) return n + s.cues.length;
+    return n + (s.text ? 1 : 0);
+  }, 0);
+  const transcript = doc.transcript ?? [];
+  const duration = doc.format?.duration_ms;
+
+  const lines: string[] = [
+    `# Video index: ${doc.provenance.source}`,
+    `- duration_ms: ${duration ?? 'unknown'}`,
+    `- container: ${doc.format?.format_name ?? 'unknown'}`,
+    `- video_streams: ${video.length}${video[0] ? ` (${video[0].width ?? '?'}x${video[0].height ?? '?'} ${video[0].codec_name ?? ''})` : ''}`,
+    `- audio_streams: ${audio.length}${audio[0] ? ` (${audio[0].codec_name ?? ''})` : ''}`,
+    `- scenes: ${scenes.length}`,
+    `- structural_keyframes: ${keyframes.length}`,
+    `- subtitle_cues: ${subtitleCueCount}`,
+    `- transcript_segments: ${transcript.length}`,
+  ];
+
+  if (scenes.length > 0) {
+    lines.push('', '## Scene architecture (not fixed-interval sampling)');
+    scenes.slice(0, 40).forEach((s, i) => {
+      lines.push(`- scene cut ${i + 1}: ${s.time_ms ?? '?'}ms`);
+    });
+    if (scenes.length > 40) lines.push(`- … ${scenes.length - 40} more scenes`);
+  }
+
+  if (keyframes.length > 0) {
+    lines.push('', '## Structural keyframe locators');
+    keyframes.slice(0, 32).forEach((k, i) => {
+      lines.push(
+        `- kf ${i + 1}: t=${k.time_ms ?? '?'}ms hash=${String(k.frame_hash ?? '').slice(0, 12) || 'n/a'}`
+      );
+    });
+  }
+
+  if (subtitleCueCount > 0) {
+    lines.push('', '## Subtitles present (embedded) — prefer as dialogue evidence over vision');
+  }
+  if (transcript.length > 0) {
+    lines.push('', '## Transcript segments (optional ASR)');
+    transcript.slice(0, 12).forEach((tseg) => {
+      const text = String(tseg.text ?? '')
+        .replace(/\s+/g, ' ')
+        .slice(0, 120);
+      lines.push(`- ${tseg.start_ms ?? '?'}ms: ${text}`);
+    });
+  }
+
+  if ((doc.warnings ?? []).length > 0) {
+    lines.push('', '## Warnings', ...(doc.warnings ?? []).map((w) => `- ${w}`));
+  }
+
+  lines.push(
+    '',
+    '## Agent guidance',
+    '- Use scenes + keyframe locators for structure; do not invent uniform N-second sampling as architecture.',
+    '- For pixel-level claims, follow up with video_evidence render_frame/crop_frame at a locator time_ms.',
+    '- Optional LLM vision is non-authority; timeline evidence is the local-first truth.'
+  );
+
+  return {
+    policy: 'agent_video_index_v1',
+    source: doc.provenance.source,
+    ...(duration !== undefined ? { duration_ms: duration } : {}),
+    outline: lines.join('\n'),
+    scene_count: scenes.length,
+    keyframe_count: keyframes.length,
+    subtitle_cue_count: subtitleCueCount,
+    transcript_segment_count: transcript.length,
+    audio_stream_count: audio.length,
+    video_stream_count: video.length,
+  };
+}

package/src/utils/asr.ts

@@ -1,6 +1,10 @@
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { shouldUseRustAsrEngine, transcribeViaRustEngine } from '../engine/rust-asr.js';
 import type { TranscriptSegment } from '../types/timeline.js';
-import { isBinaryAvailable } from './exec.js';
+import { execBinary, isBinaryAvailable } from './exec.js';
 
 const ASR_CANDIDATES = ['whisper-cli', 'whisper-cpp', 'whisper', 'vosk-transcriber'] as const;
 
@@ -13,6 +17,124 @@ export const detectAsrAdapter = async (): Promise<string | null> => {
   return null;
 };
 
+/** Parse whisper.txt style lines: [00:00.000 --> 00:01.000] text */
+export function parseWhisperTxt(raw: string): TranscriptSegment[] {
+  const segments: TranscriptSegment[] = [];
+  const re =
+    /\[(\d{1,2}:)?(\d{1,2}):(\d{1,2})(?:\.(\d{1,3}))?\s*-->\s*(\d{1,2}:)?(\d{1,2}):(\d{1,2})(?:\.(\d{1,3}))?\]\s*(.+)/g;
+  const toMs = (h: string | undefined, min: string, sec: string, ms: string | undefined) => {
+    const hours = h ? Number.parseInt(h.replace(':', ''), 10) : 0;
+    const minutes = Number.parseInt(min, 10);
+    const seconds = Number.parseInt(sec, 10);
+    const millis = ms ? Number.parseInt(ms.padEnd(3, '0').slice(0, 3), 10) : 0;
+    return ((hours * 60 + minutes) * 60 + seconds) * 1000 + millis;
+  };
+  let m = re.exec(raw);
+  while (m !== null) {
+    const start = toMs(m[1], m[2] ?? '0', m[3] ?? '0', m[4]);
+    const end = toMs(m[5], m[6] ?? '0', m[7] ?? '0', m[8]);
+    const text = (m[9] ?? '').trim();
+    if (text) {
+      segments.push({
+        start_ms: start,
+        end_ms: end,
+        text,
+        provenance: { method: 'asr_adapter', adapter: 'whisper-cli' },
+      });
+    }
+    m = re.exec(raw);
+  }
+  if (segments.length === 0) {
+    const lines = raw
+      .split(/\r?\n/)
+      .map((l) => l.trim())
+      .filter((l) => l && !l.startsWith('['));
+    lines.forEach((text, i) => {
+      segments.push({
+        start_ms: i * 1000,
+        end_ms: (i + 1) * 1000,
+        text,
+        provenance: { method: 'asr_adapter', adapter: 'whisper-cli' },
+      });
+    });
+  }
+  return segments;
+}
+
+async function extractWavMono16k(
+  videoPath: string
+): Promise<{ wavPath: string; dir: string } | { error: string }> {
+  const ffmpegOk = await isBinaryAvailable('ffmpeg');
+  if (!ffmpegOk) {
+    return { error: 'ffmpeg required to extract audio for local whisper ASR' };
+  }
+  const dir = mkdtempSync(join(tmpdir(), 'cue-asr-'));
+  const wavPath = join(dir, 'audio.wav');
+  try {
+    await execBinary(
+      'ffmpeg',
+      ['-hide_banner', '-y', '-i', videoPath, '-ac', '1', '-ar', '16000', '-f', 'wav', wavPath],
+      { timeoutMs: 300_000 }
+    );
+    return { wavPath, dir };
+  } catch (error: unknown) {
+    rmSync(dir, { recursive: true, force: true });
+    const message = error instanceof Error ? error.message : String(error);
+    return { error: message };
+  }
+}
+
+/** Local-first frontier ASR: whisper-cli/whisper.cpp on PATH (no cloud). */
+export async function runLocalWhisperTranscript(
+  videoPath: string,
+  adapter: string
+): Promise<{ transcript: TranscriptSegment[]; warning?: string }> {
+  const wav = await extractWavMono16k(videoPath);
+  if ('error' in wav) {
+    return { transcript: [], warning: `ASR audio extract failed: ${wav.error}` };
+  }
+
+  const outBase = join(wav.dir, 'out');
+  const model = process.env.CUE_WHISPER_MODEL ?? process.env.WHISPER_MODEL ?? '';
+  const args: string[] = ['-f', wav.wavPath, '-otxt', '-of', outBase];
+  if (model) args.push('-m', model);
+
+  try {
+    const result = spawnSync(adapter, args, {
+      encoding: 'utf8',
+      timeout: 600_000,
+      windowsHide: true,
+      maxBuffer: 20 * 1024 * 1024,
+    });
+    if (result.status !== 0) {
+      const stderr = typeof result.stderr === 'string' ? result.stderr.trim() : '';
+      return {
+        transcript: [],
+        warning: `Local ASR (${adapter}) failed: ${stderr || `exit ${String(result.status)}`}`,
+      };
+    }
+    let raw = '';
+    try {
+      raw = readFileSync(`${outBase}.txt`, 'utf8');
+    } catch {
+      raw = typeof result.stdout === 'string' ? result.stdout : '';
+    }
+    const transcript = parseWhisperTxt(raw).map((seg) => ({
+      ...seg,
+      provenance: { method: 'asr_adapter' as const, adapter },
+    }));
+    if (transcript.length === 0) {
+      return { transcript: [], warning: `Local ASR (${adapter}) produced empty transcript` };
+    }
+    return { transcript };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { transcript: [], warning: `Local ASR (${adapter}) error: ${message}` };
+  } finally {
+    rmSync(wav.dir, { recursive: true, force: true });
+  }
+}
+
 export const tryAsrTranscript = async (
   videoPath: string,
   enabled: boolean
@@ -29,18 +151,12 @@ export const tryAsrTranscript = async (
         ...(response.result.warning ? { warning: response.result.warning } : {}),
       };
     }
-
-    if (response.code === 'ADAPTER_UNAVAILABLE') {
+    if (response.code !== 'ADAPTER_UNAVAILABLE') {
       return {
         transcript: [],
-        warning: `${response.message} Checked whisper-cli and whisper-cpp.`,
+        warning: `ASR transcription failed: ${response.message}`,
       };
     }
-
-    return {
-      transcript: [],
-      warning: `ASR transcription failed: ${response.message}`,
-    };
   }
 
   const adapter = await detectAsrAdapter();
@@ -48,12 +164,16 @@ export const tryAsrTranscript = async (
     return {
       transcript: [],
       warning:
-        'ASR requested but no local adapter found (checked whisper-cli, whisper-cpp, whisper, vosk-transcriber); transcript skipped.',
+        'ASR requested but no local adapter found (checked whisper-cli, whisper-cpp, whisper, vosk-transcriber). Install whisper.cpp for local-first frontier ASR.',
     };
   }
 
+  if (adapter === 'whisper-cli' || adapter === 'whisper-cpp' || adapter === 'whisper') {
+    return runLocalWhisperTranscript(videoPath, adapter);
+  }
+
   return {
     transcript: [],
-    warning: `ASR adapter "${adapter}" detected but Rust ASR engine is not enabled. Build video-reader-cli or set VIDEO_READER_USE_RUST_ASR=1.`,
+    warning: `ASR adapter "${adapter}" detected but no local runner implemented for it yet.`,
   };
 };

package/src/utils/structuralKeyframes.ts

@@ -0,0 +1,73 @@
+/**
+ * Structural keyframe policy: prefer scene-change timestamps over fixed-interval thumbs.
+ */
+
+export type StructuralKeyframePlan = {
+  policy: 'structural_v1' | 'iframes_v1';
+  times_ms: number[];
+  notes: string[];
+};
+
+export function planStructuralKeyframeTimes(input: {
+  durationMs?: number;
+  sceneTimesMs: number[];
+  iframeTimesMs: number[];
+  limit: number;
+}): StructuralKeyframePlan {
+  const notes: string[] = [];
+  const limit = Math.max(1, Math.min(64, input.limit));
+  const times = new Set<number>();
+
+  const sceneTimes = [...input.sceneTimesMs]
+    .filter((t) => Number.isFinite(t) && t >= 0)
+    .sort((a, b) => a - b);
+  if (sceneTimes.length > 0) {
+    times.add(0);
+    for (let i = 0; i < sceneTimes.length; i++) {
+      const st = sceneTimes[i];
+      if (st !== undefined) times.add(st);
+      const a = sceneTimes[i];
+      const b = sceneTimes[i + 1];
+      if (a !== undefined && b !== undefined) {
+        times.add(Math.round((a + b) / 2));
+      }
+    }
+    if (typeof input.durationMs === 'number' && input.durationMs > 0) {
+      times.add(Math.max(0, input.durationMs - 1));
+    }
+    notes.push(
+      'Keyframe times derived from scene-change architecture (starts + mid-gaps), not fixed N-second sampling.'
+    );
+    const sorted = [...times].sort((a, b) => a - b);
+    if (sorted.length <= limit) {
+      return { policy: 'structural_v1', times_ms: sorted, notes };
+    }
+    const picked: number[] = [];
+    for (let i = 0; i < limit; i++) {
+      const idx = Math.round((i * (sorted.length - 1)) / Math.max(1, limit - 1));
+      const v = sorted[idx];
+      if (v !== undefined) picked.push(v);
+    }
+    notes.push(`Downsampled structural candidates from ${sorted.length} to ${limit}.`);
+    return {
+      policy: 'structural_v1',
+      times_ms: [...new Set(picked)].sort((a, b) => a - b),
+      notes,
+    };
+  }
+
+  notes.push('No scene cuts available; falling back to I-frame timestamps.');
+  const iframes = input.iframeTimesMs.slice(0, limit);
+  if (iframes.length === 0 && typeof input.durationMs === 'number' && input.durationMs > 0) {
+    notes.push('No I-frames parsed; using start/mid/end anchors only.');
+    return {
+      policy: 'structural_v1',
+      times_ms: [0, Math.round(input.durationMs / 2), Math.max(0, input.durationMs - 1)].slice(
+        0,
+        limit
+      ),
+      notes,
+    };
+  }
+  return { policy: 'iframes_v1', times_ms: iframes, notes };
+}

package/src/video/readCoordinator.ts

@@ -7,6 +7,7 @@ import {
 } from '../engine/rust-timeline.js';
 import type { ReadVideoArgs } from '../schemas/readVideo.js';
 import type { TimelineDocument, VideoSourceResult } from '../types/timeline.js';
+import { buildAgentVideoIndex } from '../utils/agentIndex.js';
 import { tryAsrTranscript } from '../utils/asr.js';
 import { extractSubtitles } from '../utils/ffmpeg.js';
 import {
@@ -20,6 +21,7 @@ import {
 import { extractKeyframes } from '../utils/frames.js';
 import { resolvePath } from '../utils/pathUtils.js';
 import { detectScenes } from '../utils/scenes.js';
+import { planStructuralKeyframeTimes } from '../utils/structuralKeyframes.js';
 
 const DEFAULT_SCENE_THRESHOLD = 0.4;
 const DEFAULT_KEYFRAME_LIMIT = 8;
@@ -109,19 +111,81 @@ export const buildTimelineDocument = async (
   }
 
   let keyframes: TimelineDocument['keyframes'] = [];
+  const keyframePolicy = args.keyframe_policy ?? 'structural';
   if (includeKeyframes) {
-    const extracted = await extractKeyframes(sourcePath, keyframeLimit, {
-      includeImages: includeKeyframeImages,
+    const extracted = await extractKeyframes(sourcePath, Math.max(keyframeLimit, 32), {
+      includeImages: false,
       ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
     });
-    keyframes = extracted.keyframes;
     if (extracted.warning) warnings.push(extracted.warning);
+
+    if (keyframePolicy === 'iframes') {
+      keyframes = extracted.keyframes.slice(0, keyframeLimit);
+      if (includeKeyframeImages && extracted.keyframes.length > 0) {
+        const withImages = await extractKeyframes(sourcePath, keyframeLimit, {
+          includeImages: true,
+          ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
+        });
+        keyframes = withImages.keyframes;
+        if (withImages.warning) warnings.push(withImages.warning);
+      }
+    } else {
+      const plan = planStructuralKeyframeTimes({
+        durationMs: format.duration_ms,
+        sceneTimesMs: scenes.map((s) => s.time_ms),
+        iframeTimesMs: extracted.keyframes.map((k) => k.time_ms),
+        limit: keyframeLimit,
+      });
+      warnings.push(...plan.notes);
+      keyframes = plan.times_ms.map((time_ms, index) => {
+        const nearest = extracted.keyframes.reduce<(typeof extracted.keyframes)[number] | null>(
+          (best, kf) => {
+            if (!best) return kf;
+            return Math.abs(kf.time_ms - time_ms) < Math.abs(best.time_ms - time_ms) ? kf : best;
+          },
+          null
+        );
+        return {
+          index,
+          time_ms,
+          provenance: {
+            method: 'ffmpeg_keyframe_select' as const,
+            pict_type: 'I' as const,
+          },
+          ...(nearest?.frame_hash ? { frame_hash: nearest.frame_hash } : {}),
+          ...(nearest?.route ? { route: nearest.route } : {}),
+        };
+      });
+      // Optional images still use iframe extractor for true pixel evidence when requested.
+      if (includeKeyframeImages) {
+        const withImages = await extractKeyframes(sourcePath, keyframeLimit, {
+          includeImages: true,
+          ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
+        });
+        if (withImages.warning) warnings.push(withImages.warning);
+        // attach images from nearest iframe evidence when hashes/times align
+        keyframes = keyframes.map((kf) => {
+          const hit = withImages.keyframes.find((x) => Math.abs(x.time_ms - kf.time_ms) < 50);
+          if (!hit) return kf;
+          return {
+            ...kf,
+            ...(hit.frame_hash ? { frame_hash: hit.frame_hash } : {}),
+            ...(hit.image_base64 ? { image_base64: hit.image_base64 } : {}),
+            ...(hit.mime ? { mime: hit.mime } : {}),
+            ...(hit.width ? { width: hit.width } : {}),
+            ...(hit.height ? { height: hit.height } : {}),
+            ...(hit.route ? { route: hit.route } : {}),
+          };
+        });
+      }
+    }
   }
 
-  return {
+  const includeAgentIndex = args.include_agent_index ?? true;
+  const documentBase = {
     provenance: {
       source: sourcePath,
-      tool: 'read_video',
+      tool: 'read_video' as const,
       version,
       extracted_at: new Date().toISOString(),
       ...(sourceHash ? { source_hash: sourceHash } : {}),
@@ -137,6 +201,17 @@ export const buildTimelineDocument = async (
     keyframes,
     warnings,
   };
+
+  return {
+    ...documentBase,
+    ...(includeAgentIndex
+      ? {
+          agent_index: buildAgentVideoIndex(
+            documentBase as Parameters<typeof buildAgentVideoIndex>[0]
+          ),
+        }
+      : {}),
+  };
 };
 
 export const processVideoSource = async (