@sylphx/video-reader-mcp 0.1.1 → 0.1.2

package/README.md

@@ -36,6 +36,13 @@ Each instrument is an independent repository (marketplace + stars).
 ---
 
 
+
+## Read video structure (not N-second frame spam)
+
+Cue is **local-first timeline architecture**: streams, dialogue, scene cuts, **structural keyframes**, and **agent_index** for text-only agents.
+
+Spec: [docs/specs/agent-video-read-contract.md](docs/specs/agent-video-read-contract.md)
+
 ## Product docs
 
 | Doc | Purpose |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sylphx/video-reader-mcp",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "mcpName": "io.github.SylphxAI/video-reader-mcp",
   "description": "Cue \u2014 Evidence-first video reading for AI agents \u2014 ffprobe, subtitles, scenes, transcripts, and timelines without frame-by-frame LLM vision.",
   "type": "module",

package/src/handlers/readVideo.ts

@@ -7,7 +7,7 @@ const MAX_CONCURRENT_SOURCES = 2;
 export const createReadVideoHandler = (version: string) =>
   tool()
     .description(
-      'Primary video reader. Returns a timeline document with ffprobe metadata, embedded subtitles, optional scene boundaries, and warnings — no per-frame vision LLM.'
+      'Primary video reader for agents (read film structure, not sample frames blindly). Timeline: streams, scenes, subtitles, optional ASR, structural keyframe locators, agent_index outline. Local-first; no required vision LLM.'
     )
     .input(readVideoArgsSchema)
     .handler(async ({ input }: { input: ReadVideoArgs }) => {

package/src/schemas/readVideo.ts

@@ -61,6 +61,18 @@ export const readVideoArgsSchema = z.object({
     .positive()
     .optional()
     .describe('Maximum width or height when resizing keyframe PNG evidence.'),
+  keyframe_policy: z
+    .enum(['structural', 'iframes'])
+    .optional()
+    .describe(
+      'structural (default when include_keyframes): scene starts/midpoints. iframes: raw I-frame index only.'
+    ),
+  include_agent_index: z
+    .boolean()
+    .optional()
+    .describe(
+      'Return agent_index outline so text-only agents can read film structure. Defaults to true.'
+    ),
 });
 
 export type ReadVideoArgs = z.infer<typeof readVideoArgsSchema>;

package/src/types/timeline.ts

@@ -96,6 +96,19 @@ export interface TimelineDocument {
   transcript: TranscriptSegment[];
   keyframes: FrameEvidence[];
   warnings: string[];
+  /** Text outline of film architecture for non-vision agents. */
+  agent_index?: {
+    policy: string;
+    source: string;
+    duration_ms?: number;
+    outline: string;
+    scene_count: number;
+    keyframe_count: number;
+    subtitle_cue_count: number;
+    transcript_segment_count: number;
+    audio_stream_count: number;
+    video_stream_count: number;
+  };
 }
 
 export interface VideoSourceResult {

package/src/utils/agentIndex.ts

@@ -0,0 +1,107 @@
+/**
+ * Agent Index: a text-first reformatting of video structure.
+ * Agents "read" the film (timeline architecture), not sample arbitrary frames.
+ */
+
+export type AgentVideoIndex = {
+  policy: 'agent_video_index_v1';
+  source: string;
+  duration_ms?: number;
+  outline: string;
+  scene_count: number;
+  keyframe_count: number;
+  subtitle_cue_count: number;
+  transcript_segment_count: number;
+  audio_stream_count: number;
+  video_stream_count: number;
+};
+
+export function buildAgentVideoIndex(doc: {
+  provenance: { source: string };
+  format?: { duration_ms?: number; format_name?: string };
+  streams?: Array<{ codec_type?: string; codec_name?: string; width?: number; height?: number }>;
+  scenes?: Array<{ time_ms?: number }>;
+  keyframes?: Array<{ time_ms?: number; frame_hash?: string }>;
+  subtitles?: Array<{ cues?: unknown[]; text?: string }>;
+  transcript?: Array<{ text?: string; start_ms?: number }>;
+  warnings?: string[];
+}): AgentVideoIndex {
+  const streams = doc.streams ?? [];
+  const audio = streams.filter((s) => s.codec_type === 'audio');
+  const video = streams.filter((s) => s.codec_type === 'video');
+  const scenes = doc.scenes ?? [];
+  const keyframes = doc.keyframes ?? [];
+  const subtitleCueCount = (doc.subtitles ?? []).reduce((n, s) => {
+    if (Array.isArray(s.cues)) return n + s.cues.length;
+    return n + (s.text ? 1 : 0);
+  }, 0);
+  const transcript = doc.transcript ?? [];
+  const duration = doc.format?.duration_ms;
+
+  const lines: string[] = [
+    `# Video index: ${doc.provenance.source}`,
+    `- duration_ms: ${duration ?? 'unknown'}`,
+    `- container: ${doc.format?.format_name ?? 'unknown'}`,
+    `- video_streams: ${video.length}${video[0] ? ` (${video[0].width ?? '?'}x${video[0].height ?? '?'} ${video[0].codec_name ?? ''})` : ''}`,
+    `- audio_streams: ${audio.length}${audio[0] ? ` (${audio[0].codec_name ?? ''})` : ''}`,
+    `- scenes: ${scenes.length}`,
+    `- structural_keyframes: ${keyframes.length}`,
+    `- subtitle_cues: ${subtitleCueCount}`,
+    `- transcript_segments: ${transcript.length}`,
+  ];
+
+  if (scenes.length > 0) {
+    lines.push('', '## Scene architecture (not fixed-interval sampling)');
+    scenes.slice(0, 40).forEach((s, i) => {
+      lines.push(`- scene cut ${i + 1}: ${s.time_ms ?? '?'}ms`);
+    });
+    if (scenes.length > 40) lines.push(`- … ${scenes.length - 40} more scenes`);
+  }
+
+  if (keyframes.length > 0) {
+    lines.push('', '## Structural keyframe locators');
+    keyframes.slice(0, 32).forEach((k, i) => {
+      lines.push(
+        `- kf ${i + 1}: t=${k.time_ms ?? '?'}ms hash=${String(k.frame_hash ?? '').slice(0, 12) || 'n/a'}`
+      );
+    });
+  }
+
+  if (subtitleCueCount > 0) {
+    lines.push('', '## Subtitles present (embedded) — prefer as dialogue evidence over vision');
+  }
+  if (transcript.length > 0) {
+    lines.push('', '## Transcript segments (optional ASR)');
+    transcript.slice(0, 12).forEach((tseg) => {
+      const text = String(tseg.text ?? '')
+        .replace(/\s+/g, ' ')
+        .slice(0, 120);
+      lines.push(`- ${tseg.start_ms ?? '?'}ms: ${text}`);
+    });
+  }
+
+  if ((doc.warnings ?? []).length > 0) {
+    lines.push('', '## Warnings', ...(doc.warnings ?? []).map((w) => `- ${w}`));
+  }
+
+  lines.push(
+    '',
+    '## Agent guidance',
+    '- Use scenes + keyframe locators for structure; do not invent uniform N-second sampling as architecture.',
+    '- For pixel-level claims, follow up with video_evidence render_frame/crop_frame at a locator time_ms.',
+    '- Optional LLM vision is non-authority; timeline evidence is the local-first truth.'
+  );
+
+  return {
+    policy: 'agent_video_index_v1',
+    source: doc.provenance.source,
+    ...(duration !== undefined ? { duration_ms: duration } : {}),
+    outline: lines.join('\n'),
+    scene_count: scenes.length,
+    keyframe_count: keyframes.length,
+    subtitle_cue_count: subtitleCueCount,
+    transcript_segment_count: transcript.length,
+    audio_stream_count: audio.length,
+    video_stream_count: video.length,
+  };
+}

package/src/utils/structuralKeyframes.ts

@@ -0,0 +1,73 @@
+/**
+ * Structural keyframe policy: prefer scene-change timestamps over fixed-interval thumbs.
+ */
+
+export type StructuralKeyframePlan = {
+  policy: 'structural_v1' | 'iframes_v1';
+  times_ms: number[];
+  notes: string[];
+};
+
+export function planStructuralKeyframeTimes(input: {
+  durationMs?: number;
+  sceneTimesMs: number[];
+  iframeTimesMs: number[];
+  limit: number;
+}): StructuralKeyframePlan {
+  const notes: string[] = [];
+  const limit = Math.max(1, Math.min(64, input.limit));
+  const times = new Set<number>();
+
+  const sceneTimes = [...input.sceneTimesMs]
+    .filter((t) => Number.isFinite(t) && t >= 0)
+    .sort((a, b) => a - b);
+  if (sceneTimes.length > 0) {
+    times.add(0);
+    for (let i = 0; i < sceneTimes.length; i++) {
+      const st = sceneTimes[i];
+      if (st !== undefined) times.add(st);
+      const a = sceneTimes[i];
+      const b = sceneTimes[i + 1];
+      if (a !== undefined && b !== undefined) {
+        times.add(Math.round((a + b) / 2));
+      }
+    }
+    if (typeof input.durationMs === 'number' && input.durationMs > 0) {
+      times.add(Math.max(0, input.durationMs - 1));
+    }
+    notes.push(
+      'Keyframe times derived from scene-change architecture (starts + mid-gaps), not fixed N-second sampling.'
+    );
+    const sorted = [...times].sort((a, b) => a - b);
+    if (sorted.length <= limit) {
+      return { policy: 'structural_v1', times_ms: sorted, notes };
+    }
+    const picked: number[] = [];
+    for (let i = 0; i < limit; i++) {
+      const idx = Math.round((i * (sorted.length - 1)) / Math.max(1, limit - 1));
+      const v = sorted[idx];
+      if (v !== undefined) picked.push(v);
+    }
+    notes.push(`Downsampled structural candidates from ${sorted.length} to ${limit}.`);
+    return {
+      policy: 'structural_v1',
+      times_ms: [...new Set(picked)].sort((a, b) => a - b),
+      notes,
+    };
+  }
+
+  notes.push('No scene cuts available; falling back to I-frame timestamps.');
+  const iframes = input.iframeTimesMs.slice(0, limit);
+  if (iframes.length === 0 && typeof input.durationMs === 'number' && input.durationMs > 0) {
+    notes.push('No I-frames parsed; using start/mid/end anchors only.');
+    return {
+      policy: 'structural_v1',
+      times_ms: [0, Math.round(input.durationMs / 2), Math.max(0, input.durationMs - 1)].slice(
+        0,
+        limit
+      ),
+      notes,
+    };
+  }
+  return { policy: 'iframes_v1', times_ms: iframes, notes };
+}

package/src/video/readCoordinator.ts

@@ -7,6 +7,7 @@ import {
 } from '../engine/rust-timeline.js';
 import type { ReadVideoArgs } from '../schemas/readVideo.js';
 import type { TimelineDocument, VideoSourceResult } from '../types/timeline.js';
+import { buildAgentVideoIndex } from '../utils/agentIndex.js';
 import { tryAsrTranscript } from '../utils/asr.js';
 import { extractSubtitles } from '../utils/ffmpeg.js';
 import {
@@ -20,6 +21,7 @@ import {
 import { extractKeyframes } from '../utils/frames.js';
 import { resolvePath } from '../utils/pathUtils.js';
 import { detectScenes } from '../utils/scenes.js';
+import { planStructuralKeyframeTimes } from '../utils/structuralKeyframes.js';
 
 const DEFAULT_SCENE_THRESHOLD = 0.4;
 const DEFAULT_KEYFRAME_LIMIT = 8;
@@ -109,19 +111,81 @@ export const buildTimelineDocument = async (
   }
 
   let keyframes: TimelineDocument['keyframes'] = [];
+  const keyframePolicy = args.keyframe_policy ?? 'structural';
   if (includeKeyframes) {
-    const extracted = await extractKeyframes(sourcePath, keyframeLimit, {
-      includeImages: includeKeyframeImages,
+    const extracted = await extractKeyframes(sourcePath, Math.max(keyframeLimit, 32), {
+      includeImages: false,
       ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
     });
-    keyframes = extracted.keyframes;
     if (extracted.warning) warnings.push(extracted.warning);
+
+    if (keyframePolicy === 'iframes') {
+      keyframes = extracted.keyframes.slice(0, keyframeLimit);
+      if (includeKeyframeImages && extracted.keyframes.length > 0) {
+        const withImages = await extractKeyframes(sourcePath, keyframeLimit, {
+          includeImages: true,
+          ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
+        });
+        keyframes = withImages.keyframes;
+        if (withImages.warning) warnings.push(withImages.warning);
+      }
+    } else {
+      const plan = planStructuralKeyframeTimes({
+        durationMs: format.duration_ms,
+        sceneTimesMs: scenes.map((s) => s.time_ms),
+        iframeTimesMs: extracted.keyframes.map((k) => k.time_ms),
+        limit: keyframeLimit,
+      });
+      warnings.push(...plan.notes);
+      keyframes = plan.times_ms.map((time_ms, index) => {
+        const nearest = extracted.keyframes.reduce<(typeof extracted.keyframes)[number] | null>(
+          (best, kf) => {
+            if (!best) return kf;
+            return Math.abs(kf.time_ms - time_ms) < Math.abs(best.time_ms - time_ms) ? kf : best;
+          },
+          null
+        );
+        return {
+          index,
+          time_ms,
+          provenance: {
+            method: 'ffmpeg_keyframe_select' as const,
+            pict_type: 'I' as const,
+          },
+          ...(nearest?.frame_hash ? { frame_hash: nearest.frame_hash } : {}),
+          ...(nearest?.route ? { route: nearest.route } : {}),
+        };
+      });
+      // Optional images still use iframe extractor for true pixel evidence when requested.
+      if (includeKeyframeImages) {
+        const withImages = await extractKeyframes(sourcePath, keyframeLimit, {
+          includeImages: true,
+          ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
+        });
+        if (withImages.warning) warnings.push(withImages.warning);
+        // attach images from nearest iframe evidence when hashes/times align
+        keyframes = keyframes.map((kf) => {
+          const hit = withImages.keyframes.find((x) => Math.abs(x.time_ms - kf.time_ms) < 50);
+          if (!hit) return kf;
+          return {
+            ...kf,
+            ...(hit.frame_hash ? { frame_hash: hit.frame_hash } : {}),
+            ...(hit.image_base64 ? { image_base64: hit.image_base64 } : {}),
+            ...(hit.mime ? { mime: hit.mime } : {}),
+            ...(hit.width ? { width: hit.width } : {}),
+            ...(hit.height ? { height: hit.height } : {}),
+            ...(hit.route ? { route: hit.route } : {}),
+          };
+        });
+      }
+    }
   }
 
-  return {
+  const includeAgentIndex = args.include_agent_index ?? true;
+  const documentBase = {
     provenance: {
       source: sourcePath,
-      tool: 'read_video',
+      tool: 'read_video' as const,
       version,
       extracted_at: new Date().toISOString(),
       ...(sourceHash ? { source_hash: sourceHash } : {}),
@@ -137,6 +201,17 @@ export const buildTimelineDocument = async (
     keyframes,
     warnings,
   };
+
+  return {
+    ...documentBase,
+    ...(includeAgentIndex
+      ? {
+          agent_index: buildAgentVideoIndex(
+            documentBase as Parameters<typeof buildAgentVideoIndex>[0]
+          ),
+        }
+      : {}),
+  };
 };
 
 export const processVideoSource = async (