@sylphx/video-reader-mcp 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +229 -41
  2. package/bin/native/video-reader-mcp-server +0 -0
  3. package/bin/video-reader-mcp +48 -0
  4. package/dist/doctor-cli.js +167 -0
  5. package/package.json +23 -10
  6. package/src/doctor-cli.ts +18 -0
  7. package/src/doctor.ts +132 -0
  8. package/src/engine/rust-asr.ts +98 -0
  9. package/src/engine/rust-frames.ts +95 -0
  10. package/src/engine/rust-timeline.ts +164 -0
  11. package/src/engine/rust-video-evidence.ts +132 -0
  12. package/src/handlers/readVideo.ts +41 -0
  13. package/src/handlers/videoEvidence.ts +161 -0
  14. package/src/mcp.ts +302 -0
  15. package/src/schemas/readVideo.ts +79 -0
  16. package/src/schemas/videoEvidence.ts +53 -0
  17. package/src/sdk.ts +44 -0
  18. package/src/types/timeline.ts +119 -0
  19. package/src/utils/agentIndex.ts +107 -0
  20. package/src/utils/asr.ts +59 -0
  21. package/src/utils/errors.ts +16 -0
  22. package/src/utils/exec.ts +76 -0
  23. package/src/utils/ffmpeg.ts +81 -0
  24. package/src/utils/ffprobe.ts +137 -0
  25. package/src/utils/frameOcr.ts +99 -0
  26. package/src/utils/frames.ts +93 -0
  27. package/src/utils/pathUtils.ts +36 -0
  28. package/src/utils/scenes.ts +63 -0
  29. package/src/utils/structuralKeyframes.ts +73 -0
  30. package/src/utils/subtitles.ts +101 -0
  31. package/src/video/readCoordinator.ts +240 -0
  32. package/dist/handlers/readVideo.js +0 -24
  33. package/dist/index.js +0 -59
  34. package/dist/mcp.js +0 -181
  35. package/dist/schemas/readVideo.js +0 -33
  36. package/dist/types/timeline.js +0 -1
  37. package/dist/utils/asr.js +0 -26
  38. package/dist/utils/errors.js +0 -14
  39. package/dist/utils/exec.js +0 -56
  40. package/dist/utils/ffmpeg.js +0 -66
  41. package/dist/utils/ffprobe.js +0 -79
  42. package/dist/utils/pathUtils.js +0 -31
  43. package/dist/utils/scenes.js +0 -49
  44. package/dist/utils/subtitles.js +0 -85
  45. package/dist/video/readCoordinator.js +0 -81
@@ -0,0 +1,93 @@
1
+ import { extractKeyframesViaRustEngine, shouldUseRustFramesEngine } from '../engine/rust-frames.js';
2
+ import type { FrameEvidence } from '../types/timeline.js';
3
+ import { execBinary, isBinaryAvailable } from './exec.js';
4
+
5
+ const KEYFRAME_PTS_TIME_RE = /pts_time:([0-9.]+)/g;
6
+
7
+ export const parseKeyframeFilterOutput = (stderr: string): FrameEvidence[] => {
8
+ const keyframes: FrameEvidence[] = [];
9
+ let match: RegExpExecArray | null = KEYFRAME_PTS_TIME_RE.exec(stderr);
10
+
11
+ while (match !== null) {
12
+ const seconds = Number.parseFloat(match[1]);
13
+ if (!Number.isFinite(seconds)) continue;
14
+
15
+ keyframes.push({
16
+ index: keyframes.length,
17
+ time_ms: Math.round(seconds * 1000),
18
+ provenance: {
19
+ method: 'ffmpeg_keyframe_select',
20
+ pict_type: 'I',
21
+ },
22
+ });
23
+ match = KEYFRAME_PTS_TIME_RE.exec(stderr);
24
+ }
25
+
26
+ return keyframes;
27
+ };
28
+
29
+ export const extractKeyframes = async (
30
+ videoPath: string,
31
+ limit = 8,
32
+ options: {
33
+ includeImages?: boolean | undefined;
34
+ maxDimension?: number | undefined;
35
+ } = {}
36
+ ): Promise<{ keyframes: FrameEvidence[]; warning?: string }> => {
37
+ const boundedLimit = Math.max(1, Math.min(limit, 64));
38
+ const includeImages = options.includeImages ?? false;
39
+
40
+ if (shouldUseRustFramesEngine()) {
41
+ const response = extractKeyframesViaRustEngine({
42
+ videoPath,
43
+ limit: boundedLimit,
44
+ includeImages,
45
+ ...(options.maxDimension !== undefined ? { maxDimension: options.maxDimension } : {}),
46
+ });
47
+
48
+ if (response.ok) {
49
+ return { keyframes: response.keyframes };
50
+ }
51
+
52
+ return {
53
+ keyframes: [],
54
+ warning: `Keyframe extraction failed: ${response.message}`,
55
+ };
56
+ }
57
+
58
+ const available = await isBinaryAvailable('ffmpeg');
59
+ if (!available) {
60
+ return {
61
+ keyframes: [],
62
+ warning: 'ffmpeg is not installed; keyframe index extraction skipped.',
63
+ };
64
+ }
65
+
66
+ try {
67
+ const { stderr } = await execBinary(
68
+ 'ffmpeg',
69
+ [
70
+ '-hide_banner',
71
+ '-i',
72
+ videoPath,
73
+ '-vf',
74
+ "select='eq(pict_type,I)',showinfo",
75
+ '-vsync',
76
+ 'vfr',
77
+ '-f',
78
+ 'null',
79
+ '-',
80
+ ],
81
+ { timeoutMs: 300_000 }
82
+ );
83
+
84
+ const keyframes = parseKeyframeFilterOutput(stderr).slice(0, boundedLimit);
85
+ return { keyframes };
86
+ } catch (error: unknown) {
87
+ const message = error instanceof Error ? error.message : String(error);
88
+ return {
89
+ keyframes: [],
90
+ warning: `Keyframe extraction failed: ${message}`,
91
+ };
92
+ }
93
+ };
@@ -0,0 +1,36 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import { ErrorCode, VideoError } from './errors.js';
4
+
5
+ export const PROJECT_ROOT = process.cwd();
6
+
7
+ const canonicalize = (p: string): string => {
8
+ try {
9
+ return fs.realpathSync(p);
10
+ } catch (err: unknown) {
11
+ if (
12
+ typeof err === 'object' &&
13
+ err !== null &&
14
+ 'code' in err &&
15
+ (err.code === 'ENOENT' || err.code === 'ENOTDIR')
16
+ ) {
17
+ const parent = path.dirname(p);
18
+ if (parent === p) return p;
19
+ return path.join(canonicalize(parent), path.basename(p));
20
+ }
21
+ throw err;
22
+ }
23
+ };
24
+
25
+ export const resolvePath = (userPath: string): string => {
26
+ if (typeof userPath !== 'string') {
27
+ throw new VideoError(ErrorCode.InvalidParams, 'Path must be a string.');
28
+ }
29
+
30
+ const normalizedUserPath = path.normalize(userPath);
31
+ const resolved = path.isAbsolute(normalizedUserPath)
32
+ ? normalizedUserPath
33
+ : path.resolve(PROJECT_ROOT, normalizedUserPath);
34
+
35
+ return canonicalize(resolved);
36
+ };
@@ -0,0 +1,63 @@
1
+ import type { SceneInfo } from '../types/timeline.js';
2
+ import { execBinary, isBinaryAvailable } from './exec.js';
3
+
4
+ const SCENE_PTS_TIME_RE = /pts_time:([0-9.]+)/g;
5
+
6
+ export const parseSceneFilterOutput = (stderr: string, threshold: number): SceneInfo[] => {
7
+ const scenes: SceneInfo[] = [];
8
+ let match: RegExpExecArray | null;
9
+
10
+ while ((match = SCENE_PTS_TIME_RE.exec(stderr)) !== null) {
11
+ const seconds = Number.parseFloat(match[1]);
12
+ if (!Number.isFinite(seconds)) continue;
13
+
14
+ scenes.push({
15
+ index: scenes.length,
16
+ time_ms: Math.round(seconds * 1000),
17
+ provenance: {
18
+ method: 'ffmpeg_scene_filter',
19
+ threshold,
20
+ },
21
+ });
22
+ }
23
+
24
+ return scenes;
25
+ };
26
+
27
+ export const detectScenes = async (
28
+ videoPath: string,
29
+ threshold: number
30
+ ): Promise<{ scenes: SceneInfo[]; warning?: string }> => {
31
+ const available = await isBinaryAvailable('ffmpeg');
32
+ if (!available) {
33
+ return {
34
+ scenes: [],
35
+ warning: 'ffmpeg is not installed; scene detection skipped.',
36
+ };
37
+ }
38
+
39
+ try {
40
+ const { stderr } = await execBinary(
41
+ 'ffmpeg',
42
+ [
43
+ '-hide_banner',
44
+ '-i',
45
+ videoPath,
46
+ '-vf',
47
+ `select='gt(scene,${threshold})',showinfo`,
48
+ '-f',
49
+ 'null',
50
+ '-',
51
+ ],
52
+ { timeoutMs: 300_000 }
53
+ );
54
+
55
+ return { scenes: parseSceneFilterOutput(stderr, threshold) };
56
+ } catch (error: unknown) {
57
+ const message = error instanceof Error ? error.message : String(error);
58
+ return {
59
+ scenes: [],
60
+ warning: `Scene detection failed: ${message}`,
61
+ };
62
+ }
63
+ };
@@ -0,0 +1,73 @@
1
+ /**
2
+ * Structural keyframe policy: prefer scene-change timestamps over fixed-interval thumbs.
3
+ */
4
+
5
+ export type StructuralKeyframePlan = {
6
+ policy: 'structural_v1' | 'iframes_v1';
7
+ times_ms: number[];
8
+ notes: string[];
9
+ };
10
+
11
+ export function planStructuralKeyframeTimes(input: {
12
+ durationMs?: number;
13
+ sceneTimesMs: number[];
14
+ iframeTimesMs: number[];
15
+ limit: number;
16
+ }): StructuralKeyframePlan {
17
+ const notes: string[] = [];
18
+ const limit = Math.max(1, Math.min(64, input.limit));
19
+ const times = new Set<number>();
20
+
21
+ const sceneTimes = [...input.sceneTimesMs]
22
+ .filter((t) => Number.isFinite(t) && t >= 0)
23
+ .sort((a, b) => a - b);
24
+ if (sceneTimes.length > 0) {
25
+ times.add(0);
26
+ for (let i = 0; i < sceneTimes.length; i++) {
27
+ const st = sceneTimes[i];
28
+ if (st !== undefined) times.add(st);
29
+ const a = sceneTimes[i];
30
+ const b = sceneTimes[i + 1];
31
+ if (a !== undefined && b !== undefined) {
32
+ times.add(Math.round((a + b) / 2));
33
+ }
34
+ }
35
+ if (typeof input.durationMs === 'number' && input.durationMs > 0) {
36
+ times.add(Math.max(0, input.durationMs - 1));
37
+ }
38
+ notes.push(
39
+ 'Keyframe times derived from scene-change architecture (starts + mid-gaps), not fixed N-second sampling.'
40
+ );
41
+ const sorted = [...times].sort((a, b) => a - b);
42
+ if (sorted.length <= limit) {
43
+ return { policy: 'structural_v1', times_ms: sorted, notes };
44
+ }
45
+ const picked: number[] = [];
46
+ for (let i = 0; i < limit; i++) {
47
+ const idx = Math.round((i * (sorted.length - 1)) / Math.max(1, limit - 1));
48
+ const v = sorted[idx];
49
+ if (v !== undefined) picked.push(v);
50
+ }
51
+ notes.push(`Downsampled structural candidates from ${sorted.length} to ${limit}.`);
52
+ return {
53
+ policy: 'structural_v1',
54
+ times_ms: [...new Set(picked)].sort((a, b) => a - b),
55
+ notes,
56
+ };
57
+ }
58
+
59
+ notes.push('No scene cuts available; falling back to I-frame timestamps.');
60
+ const iframes = input.iframeTimesMs.slice(0, limit);
61
+ if (iframes.length === 0 && typeof input.durationMs === 'number' && input.durationMs > 0) {
62
+ notes.push('No I-frames parsed; using start/mid/end anchors only.');
63
+ return {
64
+ policy: 'structural_v1',
65
+ times_ms: [0, Math.round(input.durationMs / 2), Math.max(0, input.durationMs - 1)].slice(
66
+ 0,
67
+ limit
68
+ ),
69
+ notes,
70
+ };
71
+ }
72
+ return { policy: 'iframes_v1', times_ms: iframes, notes };
73
+ }
@@ -0,0 +1,101 @@
1
+ import type { SubtitleCue } from '../types/timeline.js';
2
+
3
+ const parseTimestampToMs = (timestamp: string): number => {
4
+ const normalized = timestamp.trim().replace(',', '.');
5
+ const parts = normalized.split(':');
6
+ if (parts.length !== 3) return 0;
7
+
8
+ const [hours, minutes, secondsPart] = parts;
9
+ const [seconds, millis = '0'] = secondsPart.split('.');
10
+ const h = Number.parseInt(hours, 10);
11
+ const m = Number.parseInt(minutes, 10);
12
+ const s = Number.parseInt(seconds, 10);
13
+ const ms = Number.parseInt(millis.padEnd(3, '0').slice(0, 3), 10);
14
+
15
+ return ((h * 60 + m) * 60 + s) * 1000 + ms;
16
+ };
17
+
18
+ const stripTags = (text: string): string =>
19
+ text
20
+ .replace(/<[^>]+>/g, '')
21
+ .replace(/\{[^}]+\}/g, '')
22
+ .trim();
23
+
24
+ export const parseSrt = (content: string): SubtitleCue[] => {
25
+ const blocks = content
26
+ .replace(/\r\n/g, '\n')
27
+ .split(/\n{2,}/)
28
+ .map((block) => block.trim())
29
+ .filter(Boolean);
30
+
31
+ const cues: SubtitleCue[] = [];
32
+
33
+ for (const block of blocks) {
34
+ const lines = block.split('\n');
35
+ if (lines.length < 2) continue;
36
+
37
+ const timingLine = lines.find((line) => line.includes('-->'));
38
+ if (!timingLine) continue;
39
+
40
+ const [startRaw, endRaw] = timingLine.split('-->').map((part) => part.trim());
41
+ const textLines = lines.filter((line) => line !== lines[0] && !line.includes('-->'));
42
+ const text = stripTags(textLines.join('\n'));
43
+ if (!text) continue;
44
+
45
+ cues.push({
46
+ index: cues.length,
47
+ start_ms: parseTimestampToMs(startRaw),
48
+ end_ms: parseTimestampToMs(endRaw),
49
+ text,
50
+ provenance: {
51
+ method: 'embedded_parse',
52
+ format: 'srt',
53
+ },
54
+ });
55
+ }
56
+
57
+ return cues;
58
+ };
59
+
60
+ export const parseVtt = (content: string): SubtitleCue[] => {
61
+ const normalized = content.replace(/\r\n/g, '\n');
62
+ const blocks = normalized
63
+ .split(/\n{2,}/)
64
+ .map((block) => block.trim())
65
+ .filter((block) => block && !block.startsWith('WEBVTT') && !block.startsWith('NOTE'));
66
+
67
+ const cues: SubtitleCue[] = [];
68
+
69
+ for (const block of blocks) {
70
+ const lines = block.split('\n');
71
+ const timingLine = lines.find((line) => line.includes('-->'));
72
+ if (!timingLine) continue;
73
+
74
+ const timing = timingLine.split('-->')[0]?.trim() ?? '';
75
+ const endTiming = timingLine.split('-->')[1]?.trim().split(' ')[0] ?? '';
76
+ const textLines = lines.filter((line) => !line.includes('-->') && !/^\d+$/.test(line));
77
+ const text = stripTags(textLines.join('\n'));
78
+ if (!text) continue;
79
+
80
+ cues.push({
81
+ index: cues.length,
82
+ start_ms: parseTimestampToMs(timing),
83
+ end_ms: parseTimestampToMs(endTiming),
84
+ text,
85
+ provenance: {
86
+ method: 'embedded_parse',
87
+ format: 'vtt',
88
+ },
89
+ });
90
+ }
91
+
92
+ return cues;
93
+ };
94
+
95
+ export const parseSubtitleContent = (
96
+ content: string,
97
+ format: 'srt' | 'vtt' | 'webvtt'
98
+ ): SubtitleCue[] => {
99
+ if (format === 'srt') return parseSrt(content);
100
+ return parseVtt(content);
101
+ };
@@ -0,0 +1,240 @@
1
+ import { access } from 'node:fs/promises';
2
+ import {
3
+ assembleProbeTimelineViaRustEngine,
4
+ buildCacheKeyViaRustEngine,
5
+ hashSourceViaRustEngine,
6
+ shouldUseRustTimelineEngine,
7
+ } from '../engine/rust-timeline.js';
8
+ import type { ReadVideoArgs } from '../schemas/readVideo.js';
9
+ import type { TimelineDocument, VideoSourceResult } from '../types/timeline.js';
10
+ import { buildAgentVideoIndex } from '../utils/agentIndex.js';
11
+ import { tryAsrTranscript } from '../utils/asr.js';
12
+ import { extractSubtitles } from '../utils/ffmpeg.js';
13
+ import {
14
+ collectProbeWarnings,
15
+ findSubtitleStreams,
16
+ mapChapters,
17
+ mapStreams,
18
+ runFfprobe,
19
+ secondsToMs,
20
+ } from '../utils/ffprobe.js';
21
+ import { extractKeyframes } from '../utils/frames.js';
22
+ import { resolvePath } from '../utils/pathUtils.js';
23
+ import { detectScenes } from '../utils/scenes.js';
24
+ import { planStructuralKeyframeTimes } from '../utils/structuralKeyframes.js';
25
+
26
+ const DEFAULT_SCENE_THRESHOLD = 0.4;
27
+ const DEFAULT_KEYFRAME_LIMIT = 8;
28
+
29
+ export const buildTimelineDocument = async (
30
+ sourcePath: string,
31
+ args: ReadVideoArgs,
32
+ version: string
33
+ ): Promise<TimelineDocument> => {
34
+ const includeStreams = args.include_streams ?? true;
35
+ const includeChapters = args.include_chapters ?? true;
36
+ const includeSubtitles = args.include_subtitles ?? true;
37
+ const includeScenes = args.include_scenes ?? true;
38
+ const includeTranscript = args.include_transcript ?? false;
39
+ const includeKeyframes = args.include_keyframes ?? false;
40
+ const includeKeyframeImages = args.include_keyframe_images ?? false;
41
+ const keyframeLimit = args.keyframe_limit ?? DEFAULT_KEYFRAME_LIMIT;
42
+ const keyframeMaxDimension = args.keyframe_max_dimension;
43
+ const sceneThreshold = args.scene_threshold ?? DEFAULT_SCENE_THRESHOLD;
44
+
45
+ const warnings: string[] = [];
46
+ const probe = await runFfprobe(sourcePath);
47
+
48
+ let format: TimelineDocument['format'];
49
+ let streams: TimelineDocument['streams'];
50
+ let chapters: TimelineDocument['chapters'];
51
+ let assemblyRoute = 'typescript-timeline-v1';
52
+ let sourceHash: string | undefined;
53
+ let cacheKey: string | undefined;
54
+
55
+ if (shouldUseRustTimelineEngine()) {
56
+ const assembled = assembleProbeTimelineViaRustEngine(probe, {
57
+ includeStreams,
58
+ includeChapters,
59
+ });
60
+ format = assembled.format;
61
+ streams = assembled.streams;
62
+ chapters = assembled.chapters;
63
+ warnings.push(...assembled.warnings);
64
+ assemblyRoute = assembled.route;
65
+ sourceHash = hashSourceViaRustEngine(sourcePath);
66
+ cacheKey = buildCacheKeyViaRustEngine(sourceHash, {
67
+ includeStreams,
68
+ includeChapters,
69
+ includeSubtitles,
70
+ includeScenes,
71
+ includeTranscript,
72
+ includeKeyframes,
73
+ includeKeyframeImages,
74
+ keyframeLimit,
75
+ keyframeMaxDimension,
76
+ sceneThreshold,
77
+ });
78
+ } else {
79
+ warnings.push(...collectProbeWarnings(probe, includeStreams));
80
+ format = {
81
+ ...(probe.format.format_name ? { format_name: probe.format.format_name } : {}),
82
+ duration_ms: secondsToMs(probe.format.duration),
83
+ ...(probe.format.bit_rate ? { bit_rate: Number.parseInt(probe.format.bit_rate, 10) } : {}),
84
+ ...(probe.format.size ? { size_bytes: Number.parseInt(probe.format.size, 10) } : {}),
85
+ ...(probe.format.tags ? { tags: probe.format.tags } : {}),
86
+ };
87
+ streams = includeStreams ? mapStreams(probe.streams) : [];
88
+ chapters = includeChapters ? mapChapters(probe.chapters) : [];
89
+ }
90
+
91
+ let subtitles: TimelineDocument['subtitles'] = [];
92
+ if (includeSubtitles) {
93
+ const subtitleStreams = findSubtitleStreams(probe.streams);
94
+ const extracted = await extractSubtitles(sourcePath, subtitleStreams);
95
+ subtitles = extracted.subtitles;
96
+ warnings.push(...extracted.warnings);
97
+ }
98
+
99
+ let scenes: TimelineDocument['scenes'] = [];
100
+ if (includeScenes) {
101
+ const detected = await detectScenes(sourcePath, sceneThreshold);
102
+ scenes = detected.scenes;
103
+ if (detected.warning) warnings.push(detected.warning);
104
+ }
105
+
106
+ let transcript: TimelineDocument['transcript'] = [];
107
+ if (includeTranscript) {
108
+ const asr = await tryAsrTranscript(sourcePath, true);
109
+ transcript = asr.transcript;
110
+ if (asr.warning) warnings.push(asr.warning);
111
+ }
112
+
113
+ let keyframes: TimelineDocument['keyframes'] = [];
114
+ const keyframePolicy = args.keyframe_policy ?? 'structural';
115
+ if (includeKeyframes) {
116
+ const extracted = await extractKeyframes(sourcePath, Math.max(keyframeLimit, 32), {
117
+ includeImages: false,
118
+ ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
119
+ });
120
+ if (extracted.warning) warnings.push(extracted.warning);
121
+
122
+ if (keyframePolicy === 'iframes') {
123
+ keyframes = extracted.keyframes.slice(0, keyframeLimit);
124
+ if (includeKeyframeImages && extracted.keyframes.length > 0) {
125
+ const withImages = await extractKeyframes(sourcePath, keyframeLimit, {
126
+ includeImages: true,
127
+ ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
128
+ });
129
+ keyframes = withImages.keyframes;
130
+ if (withImages.warning) warnings.push(withImages.warning);
131
+ }
132
+ } else {
133
+ const plan = planStructuralKeyframeTimes({
134
+ durationMs: format.duration_ms,
135
+ sceneTimesMs: scenes.map((s) => s.time_ms),
136
+ iframeTimesMs: extracted.keyframes.map((k) => k.time_ms),
137
+ limit: keyframeLimit,
138
+ });
139
+ warnings.push(...plan.notes);
140
+ keyframes = plan.times_ms.map((time_ms, index) => {
141
+ const nearest = extracted.keyframes.reduce<(typeof extracted.keyframes)[number] | null>(
142
+ (best, kf) => {
143
+ if (!best) return kf;
144
+ return Math.abs(kf.time_ms - time_ms) < Math.abs(best.time_ms - time_ms) ? kf : best;
145
+ },
146
+ null
147
+ );
148
+ return {
149
+ index,
150
+ time_ms,
151
+ provenance: {
152
+ method: 'ffmpeg_keyframe_select' as const,
153
+ pict_type: 'I' as const,
154
+ },
155
+ ...(nearest?.frame_hash ? { frame_hash: nearest.frame_hash } : {}),
156
+ ...(nearest?.route ? { route: nearest.route } : {}),
157
+ };
158
+ });
159
+ // Optional images still use iframe extractor for true pixel evidence when requested.
160
+ if (includeKeyframeImages) {
161
+ const withImages = await extractKeyframes(sourcePath, keyframeLimit, {
162
+ includeImages: true,
163
+ ...(keyframeMaxDimension !== undefined ? { maxDimension: keyframeMaxDimension } : {}),
164
+ });
165
+ if (withImages.warning) warnings.push(withImages.warning);
166
+ // attach images from nearest iframe evidence when hashes/times align
167
+ keyframes = keyframes.map((kf) => {
168
+ const hit = withImages.keyframes.find((x) => Math.abs(x.time_ms - kf.time_ms) < 50);
169
+ if (!hit) return kf;
170
+ return {
171
+ ...kf,
172
+ ...(hit.frame_hash ? { frame_hash: hit.frame_hash } : {}),
173
+ ...(hit.image_base64 ? { image_base64: hit.image_base64 } : {}),
174
+ ...(hit.mime ? { mime: hit.mime } : {}),
175
+ ...(hit.width ? { width: hit.width } : {}),
176
+ ...(hit.height ? { height: hit.height } : {}),
177
+ ...(hit.route ? { route: hit.route } : {}),
178
+ };
179
+ });
180
+ }
181
+ }
182
+ }
183
+
184
+ const includeAgentIndex = args.include_agent_index ?? true;
185
+ const documentBase = {
186
+ provenance: {
187
+ source: sourcePath,
188
+ tool: 'read_video' as const,
189
+ version,
190
+ extracted_at: new Date().toISOString(),
191
+ ...(sourceHash ? { source_hash: sourceHash } : {}),
192
+ ...(cacheKey ? { cache_key: cacheKey } : {}),
193
+ ...(shouldUseRustTimelineEngine() ? { assembly_route: assemblyRoute } : {}),
194
+ },
195
+ format,
196
+ streams,
197
+ chapters,
198
+ scenes,
199
+ subtitles,
200
+ transcript,
201
+ keyframes,
202
+ warnings,
203
+ };
204
+
205
+ return {
206
+ ...documentBase,
207
+ ...(includeAgentIndex
208
+ ? {
209
+ agent_index: buildAgentVideoIndex(
210
+ documentBase as Parameters<typeof buildAgentVideoIndex>[0]
211
+ ),
212
+ }
213
+ : {}),
214
+ };
215
+ };
216
+
217
+ export const processVideoSource = async (
218
+ userPath: string,
219
+ args: ReadVideoArgs,
220
+ version: string
221
+ ): Promise<VideoSourceResult> => {
222
+ try {
223
+ const sourcePath = resolvePath(userPath);
224
+ await access(sourcePath);
225
+
226
+ const data = await buildTimelineDocument(sourcePath, args, version);
227
+ return {
228
+ source: userPath,
229
+ success: true,
230
+ data,
231
+ };
232
+ } catch (error: unknown) {
233
+ const message = error instanceof Error ? error.message : String(error);
234
+ return {
235
+ source: userPath,
236
+ success: false,
237
+ error: message,
238
+ };
239
+ }
240
+ };
@@ -1,24 +0,0 @@
1
- import { text, tool, toolError } from '../mcp.js';
2
- import { readVideoArgsSchema } from '../schemas/readVideo.js';
3
- import { processVideoSource } from '../video/readCoordinator.js';
4
- const MAX_CONCURRENT_SOURCES = 2;
5
- export const createReadVideoHandler = (version) => tool()
6
- .description('Primary video reader. Returns a timeline document with ffprobe metadata, embedded subtitles, optional scene boundaries, and warnings — no per-frame vision LLM.')
7
- .input(readVideoArgsSchema)
8
- .handler(async ({ input }) => {
9
- const results = [];
10
- for (let i = 0; i < input.sources.length; i += MAX_CONCURRENT_SOURCES) {
11
- const batch = input.sources.slice(i, i + MAX_CONCURRENT_SOURCES);
12
- const batchResults = await Promise.all(batch.map((source) => processVideoSource(source.path, input, version)));
13
- results.push(...batchResults);
14
- }
15
- const allFailed = results.every((result) => !result.success);
16
- if (allFailed) {
17
- const errorMessages = results.map((result) => result.error).join('; ');
18
- return toolError(`All video sources failed to process: ${errorMessages}`);
19
- }
20
- return text(JSON.stringify({
21
- results,
22
- }, null, 2));
23
- });
24
- export const readVideo = createReadVideoHandler('0.1.0');
package/dist/index.js DELETED
@@ -1,59 +0,0 @@
1
- #!/usr/bin/env node
2
- import { createRequire } from 'node:module';
3
- import { createReadVideoHandler } from './handlers/readVideo.js';
4
- import { createServer, http, stdio } from './mcp.js';
5
- const require = createRequire(import.meta.url);
6
- const packageJson = require('../package.json');
7
- const transportType = process.env['MCP_TRANSPORT'] ?? 'stdio';
8
- const httpPort = Number.parseInt(process.env['MCP_HTTP_PORT'] ?? '8080', 10);
9
- const httpHost = process.env['MCP_HTTP_HOST'] ?? '127.0.0.1';
10
- const apiKey = process.env['MCP_API_KEY'];
11
- const corsOrigin = process.env['MCP_CORS_ORIGIN'];
12
- const isLoopbackHost = (host) => host === 'localhost' || host === '::1' || host === '127.0.0.1' || host.startsWith('127.');
13
- function createTransport() {
14
- if (transportType === 'http') {
15
- return http({
16
- port: httpPort,
17
- hostname: httpHost,
18
- ...(corsOrigin ? { cors: corsOrigin } : {}),
19
- ...(apiKey ? { apiKey } : {}),
20
- });
21
- }
22
- return stdio();
23
- }
24
- const server = createServer({
25
- name: 'video-reader-mcp',
26
- version: packageJson.version,
27
- instructions: 'Evidence-first video reader. Use read_video to extract ffprobe metadata, embedded subtitles, scene boundaries, and timeline warnings without frame-by-frame vision LLM calls.',
28
- tools: {
29
- read_video: createReadVideoHandler(packageJson.version),
30
- },
31
- transport: createTransport(),
32
- });
33
- async function main() {
34
- await server.start();
35
- if (transportType === 'http') {
36
- console.log(`[Video Reader MCP] Server running on http://${httpHost}:${httpPort}/mcp`);
37
- console.log(`[Video Reader MCP] Health check: http://${httpHost}:${httpPort}/mcp/health`);
38
- if (apiKey) {
39
- console.log('[Video Reader MCP] API key authentication enabled (X-API-Key header)');
40
- }
41
- else if (!isLoopbackHost(httpHost)) {
42
- console.warn(`[Video Reader MCP] WARNING: bound to non-loopback host ${httpHost} with no API key. ` +
43
- 'Any client that can reach this port can read every video this process can access. ' +
44
- 'Set MCP_API_KEY to require an X-API-Key header, or bind MCP_HTTP_HOST=127.0.0.1.');
45
- }
46
- if (corsOrigin) {
47
- console.log(`[Video Reader MCP] CORS allowed origin: ${corsOrigin}`);
48
- }
49
- console.log('[Video Reader MCP] Project root:', process.cwd());
50
- }
51
- else if (process.env['DEBUG_MCP']) {
52
- console.error('[Video Reader MCP] Server running on stdio');
53
- console.error('[Video Reader MCP] Project root:', process.cwd());
54
- }
55
- }
56
- main().catch((error) => {
57
- console.error('[Video Reader MCP] Server error:', error);
58
- process.exit(1);
59
- });