@sylphx/cue 0.1.2

package/src/utils/frames.ts

@@ -0,0 +1,93 @@
+import { extractKeyframesViaRustEngine, shouldUseRustFramesEngine } from '../engine/rust-frames.js';
+import type { FrameEvidence } from '../types/timeline.js';
+import { execBinary, isBinaryAvailable } from './exec.js';
+
+const KEYFRAME_PTS_TIME_RE = /pts_time:([0-9.]+)/g;
+
+export const parseKeyframeFilterOutput = (stderr: string): FrameEvidence[] => {
+  const keyframes: FrameEvidence[] = [];
+  let match: RegExpExecArray | null = KEYFRAME_PTS_TIME_RE.exec(stderr);
+
+  while (match !== null) {
+    const seconds = Number.parseFloat(match[1]);
+    if (!Number.isFinite(seconds)) continue;
+
+    keyframes.push({
+      index: keyframes.length,
+      time_ms: Math.round(seconds * 1000),
+      provenance: {
+        method: 'ffmpeg_keyframe_select',
+        pict_type: 'I',
+      },
+    });
+    match = KEYFRAME_PTS_TIME_RE.exec(stderr);
+  }
+
+  return keyframes;
+};
+
+export const extractKeyframes = async (
+  videoPath: string,
+  limit = 8,
+  options: {
+    includeImages?: boolean | undefined;
+    maxDimension?: number | undefined;
+  } = {}
+): Promise<{ keyframes: FrameEvidence[]; warning?: string }> => {
+  const boundedLimit = Math.max(1, Math.min(limit, 64));
+  const includeImages = options.includeImages ?? false;
+
+  if (shouldUseRustFramesEngine()) {
+    const response = extractKeyframesViaRustEngine({
+      videoPath,
+      limit: boundedLimit,
+      includeImages,
+      ...(options.maxDimension !== undefined ? { maxDimension: options.maxDimension } : {}),
+    });
+
+    if (response.ok) {
+      return { keyframes: response.keyframes };
+    }
+
+    return {
+      keyframes: [],
+      warning: `Keyframe extraction failed: ${response.message}`,
+    };
+  }
+
+  const available = await isBinaryAvailable('ffmpeg');
+  if (!available) {
+    return {
+      keyframes: [],
+      warning: 'ffmpeg is not installed; keyframe index extraction skipped.',
+    };
+  }
+
+  try {
+    const { stderr } = await execBinary(
+      'ffmpeg',
+      [
+        '-hide_banner',
+        '-i',
+        videoPath,
+        '-vf',
+        "select='eq(pict_type,I)',showinfo",
+        '-vsync',
+        'vfr',
+        '-f',
+        'null',
+        '-',
+      ],
+      { timeoutMs: 300_000 }
+    );
+
+    const keyframes = parseKeyframeFilterOutput(stderr).slice(0, boundedLimit);
+    return { keyframes };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      keyframes: [],
+      warning: `Keyframe extraction failed: ${message}`,
+    };
+  }
+};

package/src/utils/pathUtils.ts

@@ -0,0 +1,36 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { ErrorCode, VideoError } from './errors.js';
+
+export const PROJECT_ROOT = process.cwd();
+
+const canonicalize = (p: string): string => {
+  try {
+    return fs.realpathSync(p);
+  } catch (err: unknown) {
+    if (
+      typeof err === 'object' &&
+      err !== null &&
+      'code' in err &&
+      (err.code === 'ENOENT' || err.code === 'ENOTDIR')
+    ) {
+      const parent = path.dirname(p);
+      if (parent === p) return p;
+      return path.join(canonicalize(parent), path.basename(p));
+    }
+    throw err;
+  }
+};
+
+export const resolvePath = (userPath: string): string => {
+  if (typeof userPath !== 'string') {
+    throw new VideoError(ErrorCode.InvalidParams, 'Path must be a string.');
+  }
+
+  const normalizedUserPath = path.normalize(userPath);
+  const resolved = path.isAbsolute(normalizedUserPath)
+    ? normalizedUserPath
+    : path.resolve(PROJECT_ROOT, normalizedUserPath);
+
+  return canonicalize(resolved);
+};

package/src/utils/scenes.ts

@@ -0,0 +1,63 @@
+import type { SceneInfo } from '../types/timeline.js';
+import { execBinary, isBinaryAvailable } from './exec.js';
+
+const SCENE_PTS_TIME_RE = /pts_time:([0-9.]+)/g;
+
+export const parseSceneFilterOutput = (stderr: string, threshold: number): SceneInfo[] => {
+  const scenes: SceneInfo[] = [];
+  let match: RegExpExecArray | null;
+
+  while ((match = SCENE_PTS_TIME_RE.exec(stderr)) !== null) {
+    const seconds = Number.parseFloat(match[1]);
+    if (!Number.isFinite(seconds)) continue;
+
+    scenes.push({
+      index: scenes.length,
+      time_ms: Math.round(seconds * 1000),
+      provenance: {
+        method: 'ffmpeg_scene_filter',
+        threshold,
+      },
+    });
+  }
+
+  return scenes;
+};
+
+export const detectScenes = async (
+  videoPath: string,
+  threshold: number
+): Promise<{ scenes: SceneInfo[]; warning?: string }> => {
+  const available = await isBinaryAvailable('ffmpeg');
+  if (!available) {
+    return {
+      scenes: [],
+      warning: 'ffmpeg is not installed; scene detection skipped.',
+    };
+  }
+
+  try {
+    const { stderr } = await execBinary(
+      'ffmpeg',
+      [
+        '-hide_banner',
+        '-i',
+        videoPath,
+        '-vf',
+        `select='gt(scene,${threshold})',showinfo`,
+        '-f',
+        'null',
+        '-',
+      ],
+      { timeoutMs: 300_000 }
+    );
+
+    return { scenes: parseSceneFilterOutput(stderr, threshold) };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      scenes: [],
+      warning: `Scene detection failed: ${message}`,
+    };
+  }
+};

package/src/utils/structuralKeyframes.ts

@@ -0,0 +1,73 @@
+/**
+ * Structural keyframe policy: prefer scene-change timestamps over fixed-interval thumbs.
+ */
+
+export type StructuralKeyframePlan = {
+  policy: 'structural_v1' | 'iframes_v1';
+  times_ms: number[];
+  notes: string[];
+};
+
+export function planStructuralKeyframeTimes(input: {
+  durationMs?: number;
+  sceneTimesMs: number[];
+  iframeTimesMs: number[];
+  limit: number;
+}): StructuralKeyframePlan {
+  const notes: string[] = [];
+  const limit = Math.max(1, Math.min(64, input.limit));
+  const times = new Set<number>();
+
+  const sceneTimes = [...input.sceneTimesMs]
+    .filter((t) => Number.isFinite(t) && t >= 0)
+    .sort((a, b) => a - b);
+  if (sceneTimes.length > 0) {
+    times.add(0);
+    for (let i = 0; i < sceneTimes.length; i++) {
+      const st = sceneTimes[i];
+      if (st !== undefined) times.add(st);
+      const a = sceneTimes[i];
+      const b = sceneTimes[i + 1];
+      if (a !== undefined && b !== undefined) {
+        times.add(Math.round((a + b) / 2));
+      }
+    }
+    if (typeof input.durationMs === 'number' && input.durationMs > 0) {
+      times.add(Math.max(0, input.durationMs - 1));
+    }
+    notes.push(
+      'Keyframe times derived from scene-change architecture (starts + mid-gaps), not fixed N-second sampling.'
+    );
+    const sorted = [...times].sort((a, b) => a - b);
+    if (sorted.length <= limit) {
+      return { policy: 'structural_v1', times_ms: sorted, notes };
+    }
+    const picked: number[] = [];
+    for (let i = 0; i < limit; i++) {
+      const idx = Math.round((i * (sorted.length - 1)) / Math.max(1, limit - 1));
+      const v = sorted[idx];
+      if (v !== undefined) picked.push(v);
+    }
+    notes.push(`Downsampled structural candidates from ${sorted.length} to ${limit}.`);
+    return {
+      policy: 'structural_v1',
+      times_ms: [...new Set(picked)].sort((a, b) => a - b),
+      notes,
+    };
+  }
+
+  notes.push('No scene cuts available; falling back to I-frame timestamps.');
+  const iframes = input.iframeTimesMs.slice(0, limit);
+  if (iframes.length === 0 && typeof input.durationMs === 'number' && input.durationMs > 0) {
+    notes.push('No I-frames parsed; using start/mid/end anchors only.');
+    return {
+      policy: 'structural_v1',
+      times_ms: [0, Math.round(input.durationMs / 2), Math.max(0, input.durationMs - 1)].slice(
+        0,
+        limit
+      ),
+      notes,
+    };
+  }
+  return { policy: 'iframes_v1', times_ms: iframes, notes };
+}

package/src/utils/subtitles.ts

@@ -0,0 +1,101 @@
+import type { SubtitleCue } from '../types/timeline.js';
+
+const parseTimestampToMs = (timestamp: string): number => {
+  const normalized = timestamp.trim().replace(',', '.');
+  const parts = normalized.split(':');
+  if (parts.length !== 3) return 0;
+
+  const [hours, minutes, secondsPart] = parts;
+  const [seconds, millis = '0'] = secondsPart.split('.');
+  const h = Number.parseInt(hours, 10);
+  const m = Number.parseInt(minutes, 10);
+  const s = Number.parseInt(seconds, 10);
+  const ms = Number.parseInt(millis.padEnd(3, '0').slice(0, 3), 10);
+
+  return ((h * 60 + m) * 60 + s) * 1000 + ms;
+};
+
+const stripTags = (text: string): string =>
+  text
+    .replace(/<[^>]+>/g, '')
+    .replace(/\{[^}]+\}/g, '')
+    .trim();
+
+export const parseSrt = (content: string): SubtitleCue[] => {
+  const blocks = content
+    .replace(/\r\n/g, '\n')
+    .split(/\n{2,}/)
+    .map((block) => block.trim())
+    .filter(Boolean);
+
+  const cues: SubtitleCue[] = [];
+
+  for (const block of blocks) {
+    const lines = block.split('\n');
+    if (lines.length < 2) continue;
+
+    const timingLine = lines.find((line) => line.includes('-->'));
+    if (!timingLine) continue;
+
+    const [startRaw, endRaw] = timingLine.split('-->').map((part) => part.trim());
+    const textLines = lines.filter((line) => line !== lines[0] && !line.includes('-->'));
+    const text = stripTags(textLines.join('\n'));
+    if (!text) continue;
+
+    cues.push({
+      index: cues.length,
+      start_ms: parseTimestampToMs(startRaw),
+      end_ms: parseTimestampToMs(endRaw),
+      text,
+      provenance: {
+        method: 'embedded_parse',
+        format: 'srt',
+      },
+    });
+  }
+
+  return cues;
+};
+
+export const parseVtt = (content: string): SubtitleCue[] => {
+  const normalized = content.replace(/\r\n/g, '\n');
+  const blocks = normalized
+    .split(/\n{2,}/)
+    .map((block) => block.trim())
+    .filter((block) => block && !block.startsWith('WEBVTT') && !block.startsWith('NOTE'));
+
+  const cues: SubtitleCue[] = [];
+
+  for (const block of blocks) {
+    const lines = block.split('\n');
+    const timingLine = lines.find((line) => line.includes('-->'));
+    if (!timingLine) continue;
+
+    const timing = timingLine.split('-->')[0]?.trim() ?? '';
+    const endTiming = timingLine.split('-->')[1]?.trim().split(' ')[0] ?? '';
+    const textLines = lines.filter((line) => !line.includes('-->') && !/^\d+$/.test(line));
+    const text = stripTags(textLines.join('\n'));
+    if (!text) continue;
+
+    cues.push({
+      index: cues.length,
+      start_ms: parseTimestampToMs(timing),
+      end_ms: parseTimestampToMs(endTiming),
+      text,
+      provenance: {
+        method: 'embedded_parse',
+        format: 'vtt',
+      },
+    });
+  }
+
+  return cues;
+};
+
+export const parseSubtitleContent = (
+  content: string,
+  format: 'srt' | 'vtt' | 'webvtt'
+): SubtitleCue[] => {
+  if (format === 'srt') return parseSrt(content);
+  return parseVtt(content);
+};

package/src/video/readCoordinator.ts

@@ -0,0 +1,240 @@
+import { access } from 'node:fs/promises';
+import {
+  assembleProbeTimelineViaRustEngine,
+  buildCacheKeyViaRustEngine,
+  hashSourceViaRustEngine,
+  shouldUseRustTimelineEngine,
+} from '../engine/rust-timeline.js';
+import type { ReadVideoArgs } from '../schemas/readVideo.js';
+import type { TimelineDocument, VideoSourceResult } from '../types/timeline.js';
+import { buildAgentVideoIndex } from '../utils/agentIndex.js';
+import { tryAsrTranscript } from '../utils/asr.js';
+import { extractSubtitles } from '../utils/ffmpeg.js';
+import {
+  collectProbeWarnings,
+  findSubtitleStreams,
+  mapChapters,
+  mapStreams,
+  runFfprobe,
+  secondsToMs,
+} from '../utils/ffprobe.js';
+import { extractKeyframes } from '../utils/frames.js';
+import { resolvePath } from '../utils/pathUtils.js';
+import { detectScenes } from '../utils/scenes.js';
+import { planStructuralKeyframeTimes } from '../utils/structuralKeyframes.js';
+
+const DEFAULT_SCENE_THRESHOLD = 0.4;
+const DEFAULT_KEYFRAME_LIMIT = 8;
+
+export const buildTimelineDocument = async (
+  sourcePath: string,
+  args: ReadVideoArgs,
+  version: string
+): Promise<TimelineDocument> => {
+  const includeStreams = args.include_streams ?? true;
+  const includeChapters = args.include_chapters ?? true;
+  const includeSubtitles = args.include_subtitles ?? true;
+  const includeScenes = args.include_scenes ?? true;
+  const includeTranscript = args.include_transcript ?? false;
+  const includeKeyframes = args.include_keyframes ?? false;
+  const includeKeyframeImages = args.include_keyframe_images ?? false;
+  const keyframeLimit = args.keyframe_limit ?? DEFAULT_KEYFRAME_LIMIT;
+  const keyframeMaxDimension = args.keyframe_max_dimension;
+  const sceneThreshold = args.scene_threshold ?? DEFAULT_SCENE_THRESHOLD;
+
+  const warnings: string[] = [];
+  const probe = await runFfprobe(sourcePath);
+
+  let format: TimelineDocument['format'];
+  let streams: TimelineDocument['streams'];
+  let chapters: TimelineDocument['chapters'];
+  let assemblyRoute = 'typescript-timeline-v1';
+  let sourceHash: string | undefined;
+  let cacheKey: string | undefined;
+
+  if (shouldUseRustTimelineEngine()) {
+    const assembled = assembleProbeTimelineViaRustEngine(probe, {
+      includeStreams,
+      includeChapters,
+    });
+    format = assembled.format;
+    streams = assembled.streams;
+    chapters = assembled.chapters;
+    warnings.push(...assembled.warnings);
+    assemblyRoute = assembled.route;
+    sourceHash = hashSourceViaRustEngine(sourcePath);
+    cacheKey = buildCacheKeyViaRustEngine(sourceHash, {
+      includeStreams,
+      includeChapters,
+      includeSubtitles,
+      includeScenes,
+      includeTranscript,
+      includeKeyframes,
+      includeKeyframeImages,
+      keyframeLimit,
+      keyframeMaxDimension,
+      sceneThreshold,
+    });
+  } else {
+    warnings.push(...collectProbeWarnings(probe, includeStreams));
+    format = {
+      ...(probe.format.format_name ? { format_name: probe.format.format_name } : {}),
+      duration_ms: secondsToMs(probe.format.duration),
+      ...(probe.format.bit_rate ? { bit_rate: Number.parseInt(probe.format.bit_rate, 10) } : {}),
+      ...(probe.format.size ? { size_bytes: Number.parseInt(probe.format.size, 10) } : {}),
+      ...(probe.format.tags ? { tags: probe.format.tags } : {}),
+    };
+    streams = includeStreams ? mapStreams(probe.streams) : [];
+    chapters = includeChapters ? mapChapters(probe.chapters) : [];
+  }
+
+  let subtitles: TimelineDocument['subtitles'] = [];
+  if (includeSubtitles) {
+    const subtitleStreams = findSubtitleStreams(probe.streams);
+    const extracted = await extractSubtitles(sourcePath, subtitleStreams);
+    subtitles = extracted.subtitles;
+    warnings.push(...extracted.warnings);
+  }
+
+  let scenes: TimelineDocument['scenes'] = [];
+  if (includeScenes) {
+    const detected = await detectScenes(sourcePath, sceneThreshold);
+    scenes = detected.scenes;
+    if (detected.warning) warnings.push(detected.warning);
+  }
+
+  let transcript: TimelineDocument['transcript'] = [];
+  if (includeTranscript) {
+    const asr = await tryAsrTranscript(sourcePath, true);
+    transcript = asr.transcript;
+    if (asr.warning) warnings.push(asr.warning);
+  }
+
+  let keyframes: TimelineDocument['keyframes'] = [];
+  const keyframePolicy = args.keyframe_policy ?? 'structural';
+  if (includeKeyframes) {
+    const extracted = await extractKeyframes(sourcePath, Math.max(keyframeLimit, 32), {
+      includeImages: false,
+      ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
+    });
+    if (extracted.warning) warnings.push(extracted.warning);
+
+    if (keyframePolicy === 'iframes') {
+      keyframes = extracted.keyframes.slice(0, keyframeLimit);
+      if (includeKeyframeImages && extracted.keyframes.length > 0) {
+        const withImages = await extractKeyframes(sourcePath, keyframeLimit, {
+          includeImages: true,
+          ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
+        });
+        keyframes = withImages.keyframes;
+        if (withImages.warning) warnings.push(withImages.warning);
+      }
+    } else {
+      const plan = planStructuralKeyframeTimes({
+        durationMs: format.duration_ms,
+        sceneTimesMs: scenes.map((s) => s.time_ms),
+        iframeTimesMs: extracted.keyframes.map((k) => k.time_ms),
+        limit: keyframeLimit,
+      });
+      warnings.push(...plan.notes);
+      keyframes = plan.times_ms.map((time_ms, index) => {
+        const nearest = extracted.keyframes.reduce<(typeof extracted.keyframes)[number] | null>(
+          (best, kf) => {
+            if (!best) return kf;
+            return Math.abs(kf.time_ms - time_ms) < Math.abs(best.time_ms - time_ms) ? kf : best;
+          },
+          null
+        );
+        return {
+          index,
+          time_ms,
+          provenance: {
+            method: 'ffmpeg_keyframe_select' as const,
+            pict_type: 'I' as const,
+          },
+          ...(nearest?.frame_hash ? { frame_hash: nearest.frame_hash } : {}),
+          ...(nearest?.route ? { route: nearest.route } : {}),
+        };
+      });
+      // Optional images still use iframe extractor for true pixel evidence when requested.
+      if (includeKeyframeImages) {
+        const withImages = await extractKeyframes(sourcePath, keyframeLimit, {
+          includeImages: true,
+          ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
+        });
+        if (withImages.warning) warnings.push(withImages.warning);
+        // attach images from nearest iframe evidence when hashes/times align
+        keyframes = keyframes.map((kf) => {
+          const hit = withImages.keyframes.find((x) => Math.abs(x.time_ms - kf.time_ms) < 50);
+          if (!hit) return kf;
+          return {
+            ...kf,
+            ...(hit.frame_hash ? { frame_hash: hit.frame_hash } : {}),
+            ...(hit.image_base64 ? { image_base64: hit.image_base64 } : {}),
+            ...(hit.mime ? { mime: hit.mime } : {}),
+            ...(hit.width ? { width: hit.width } : {}),
+            ...(hit.height ? { height: hit.height } : {}),
+            ...(hit.route ? { route: hit.route } : {}),
+          };
+        });
+      }
+    }
+  }
+
+  const includeAgentIndex = args.include_agent_index ?? true;
+  const documentBase = {
+    provenance: {
+      source: sourcePath,
+      tool: 'read_video' as const,
+      version,
+      extracted_at: new Date().toISOString(),
+      ...(sourceHash ? { source_hash: sourceHash } : {}),
+      ...(cacheKey ? { cache_key: cacheKey } : {}),
+      ...(shouldUseRustTimelineEngine() ? { assembly_route: assemblyRoute } : {}),
+    },
+    format,
+    streams,
+    chapters,
+    scenes,
+    subtitles,
+    transcript,
+    keyframes,
+    warnings,
+  };
+
+  return {
+    ...documentBase,
+    ...(includeAgentIndex
+      ? {
+          agent_index: buildAgentVideoIndex(
+            documentBase as Parameters<typeof buildAgentVideoIndex>[0]
+          ),
+        }
+      : {}),
+  };
+};
+
+export const processVideoSource = async (
+  userPath: string,
+  args: ReadVideoArgs,
+  version: string
+): Promise<VideoSourceResult> => {
+  try {
+    const sourcePath = resolvePath(userPath);
+    await access(sourcePath);
+
+    const data = await buildTimelineDocument(sourcePath, args, version);
+    return {
+      source: userPath,
+      success: true,
+      data,
+    };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      source: userPath,
+      success: false,
+      error: message,
+    };
+  }
+};