@sylphx/video-reader-mcp 0.1.0 → 0.1.1

package/src/handlers/videoEvidence.ts

@@ -0,0 +1,161 @@
+import {
+  cropFrameViaRustEngine,
+  renderFrameViaRustEngine,
+  shouldUseRustVideoEvidenceEngine,
+} from '../engine/rust-video-evidence.js';
+import { text, tool, toolError } from '../mcp.js';
+import { type VideoEvidenceArgs, videoEvidenceArgsSchema } from '../schemas/videoEvidence.js';
+import { ocrPngBase64 } from '../utils/frameOcr.js';
+import { resolvePath } from '../utils/pathUtils.js';
+
+type EvidenceResult = {
+  source: string;
+  success: boolean;
+  time_ms: number;
+  operation: VideoEvidenceArgs['operation'];
+  route?: string | undefined;
+  frame?: {
+    frame_hash: string;
+    mime: string;
+    width: number;
+    height: number;
+    image_base64?: string;
+    provenance: { method: string; time_ms: number };
+    crop?: VideoEvidenceArgs['sources'][number]['crop'];
+  };
+  ocr?: {
+    available: boolean;
+    route: string;
+    skipped_reason?: string;
+    languages: string[];
+    lines: { text: string; confidence?: number }[];
+    line_count: number;
+  };
+  error?: string | undefined;
+  code?: string | undefined;
+};
+
+const routeForOperation = (operation: VideoEvidenceArgs['operation']): string => {
+  switch (operation) {
+    case 'render_frame':
+      return 'rust-frame-render';
+    case 'crop_frame':
+      return 'rust-frame-crop';
+    case 'ocr_frame':
+      return 'rust-frame-render+tesseract_frame';
+  }
+};
+
+export const createVideoEvidenceHandler = () =>
+  tool()
+    .description(
+      'Runs focused video evidence follow-up operations: render_frame, crop_frame, or ocr_frame with timestamp locators after read_video.'
+    )
+    .input(videoEvidenceArgsSchema)
+    .handler(async ({ input }: { input: VideoEvidenceArgs }) => {
+      if (!shouldUseRustVideoEvidenceEngine()) {
+        return toolError(
+          'Rust video evidence engine is unavailable. Build video-reader-cli with cargo build --release or set VIDEO_READER_CLI.'
+        );
+      }
+
+      const results: EvidenceResult[] = [];
+      const ocrLanguages = (input as { ocr_languages?: string[] }).ocr_languages ??
+        (input as { languages?: string[] }).languages ?? ['eng'];
+
+      for (const source of input.sources) {
+        const resolvedPath = resolvePath(source.path);
+        const engineResult =
+          input.operation === 'crop_frame'
+            ? cropFrameViaRustEngine({
+                videoPath: resolvedPath,
+                timeMs: source.time_ms,
+                crop: source.crop as NonNullable<typeof source.crop>,
+                maxDimension: input.max_dimension,
+              })
+            : renderFrameViaRustEngine({
+                videoPath: resolvedPath,
+                timeMs: source.time_ms,
+                maxDimension: input.max_dimension,
+              });
+
+        if (!engineResult.ok) {
+          results.push({
+            source: source.path,
+            success: false,
+            time_ms: source.time_ms,
+            operation: input.operation,
+            error: engineResult.message,
+            code: engineResult.code,
+          });
+          continue;
+        }
+
+        if (input.operation === 'ocr_frame') {
+          const ocr = ocrPngBase64(engineResult.frame.image_base64, ocrLanguages);
+          results.push({
+            source: source.path,
+            success: true,
+            time_ms: source.time_ms,
+            operation: 'ocr_frame',
+            route: ocr.available
+              ? 'rust-frame-render+tesseract_frame'
+              : 'rust-frame-render+tesseract_unavailable',
+            frame: {
+              frame_hash: engineResult.frame.frame_hash,
+              mime: engineResult.frame.mime,
+              width: engineResult.frame.width,
+              height: engineResult.frame.height,
+              // omit large base64 by default for OCR answers (hash is the locator)
+              provenance: engineResult.frame.provenance,
+            },
+            ocr: {
+              available: ocr.available,
+              route: ocr.route,
+              languages: ocr.languages,
+              lines: ocr.lines,
+              line_count: ocr.line_count,
+              ...(ocr.skipped_reason !== undefined ? { skipped_reason: ocr.skipped_reason } : {}),
+            },
+          });
+          continue;
+        }
+
+        results.push({
+          source: source.path,
+          success: true,
+          time_ms: source.time_ms,
+          operation: input.operation,
+          route: engineResult.frame.route,
+          frame: {
+            frame_hash: engineResult.frame.frame_hash,
+            mime: engineResult.frame.mime,
+            width: engineResult.frame.width,
+            height: engineResult.frame.height,
+            image_base64: engineResult.frame.image_base64,
+            provenance: engineResult.frame.provenance,
+            ...(engineResult.frame.crop ? { crop: engineResult.frame.crop } : {}),
+          },
+        });
+      }
+
+      if (results.every((result) => !result.success)) {
+        const messages = results.map((result) => result.error).join('; ');
+        return toolError(`All video evidence sources failed: ${messages}`);
+      }
+
+      return text(
+        JSON.stringify(
+          {
+            profile: 'video_evidence_results',
+            operation: input.operation,
+            route: routeForOperation(input.operation),
+            results,
+          },
+          null,
+          2
+        )
+      );
+    });
+
+export const videoEvidence = createVideoEvidenceHandler();

package/src/mcp.ts

@@ -0,0 +1,302 @@
+import { createHash, timingSafeEqual } from 'node:crypto';
+import { createServer as createHttpServer, type Server as HttpServer } from 'node:http';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import type {
+  CallToolResult,
+  ContentBlock,
+  ImageContent,
+  TextContent,
+} from '@modelcontextprotocol/sdk/types.js';
+import { z } from 'zod';
+
+export const text = (value: string): TextContent => ({ type: 'text', text: value });
+
+export const image = (data: string, mimeType: string): ImageContent => ({
+  type: 'image',
+  data,
+  mimeType,
+});
+
+export const toolError = (message: string): CallToolResult => ({
+  content: [text(message)],
+  isError: true,
+});
+
+export type ToolHandlerResult = CallToolResult | ContentBlock | readonly ContentBlock[];
+
+export type ToolHandler<TInput> = (args: {
+  input: TInput;
+  ctx: unknown;
+}) => ToolHandlerResult | Promise<ToolHandlerResult>;
+
+export interface ToolDefinition<TInput = unknown> {
+  description: string;
+  inputSchema: z.ZodType<TInput>;
+  handler(args: { input: TInput; ctx: unknown }): ToolHandlerResult | Promise<ToolHandlerResult>;
+}
+
+class ToolBuilder<TInput = unknown> {
+  readonly #description: string | undefined;
+  readonly #inputSchema: z.ZodType<TInput> | undefined;
+
+  constructor(descriptionValue?: string, inputSchema?: z.ZodType<TInput>) {
+    this.#description = descriptionValue;
+    this.#inputSchema = inputSchema;
+  }
+
+  description(value: string): ToolBuilder<TInput> {
+    return new ToolBuilder(value, this.#inputSchema);
+  }
+
+  input<TSchema extends z.ZodType>(schema: TSchema): ToolBuilder<z.infer<TSchema>> {
+    return new ToolBuilder(this.#description, schema as z.ZodType<z.infer<TSchema>>);
+  }
+
+  handler(handler: ToolHandler<TInput>): ToolDefinition<TInput> {
+    return {
+      description: this.#description ?? '',
+      inputSchema: this.#inputSchema ?? (z.object({}) as z.ZodType<TInput>),
+      handler,
+    };
+  }
+}
+
+export const tool = (): ToolBuilder => new ToolBuilder();
+
+interface StdioTransportConfig {
+  kind: 'stdio';
+}
+
+interface HttpTransportConfig {
+  kind: 'http';
+  port: number;
+  hostname: string;
+  cors?: string;
+  apiKey?: string;
+}
+
+type TransportConfig = StdioTransportConfig | HttpTransportConfig;
+
+export const stdio = (): StdioTransportConfig => ({ kind: 'stdio' });
+
+export const http = (config: {
+  port: number;
+  hostname: string;
+  cors?: string;
+  apiKey?: string;
+}): HttpTransportConfig => ({ kind: 'http', ...config });
+
+interface CreateServerOptions {
+  name: string;
+  version: string;
+  instructions?: string;
+  tools: Record<string, ToolDefinition<unknown>>;
+  transport: TransportConfig;
+}
+
+interface ServerHandle {
+  start(): Promise<void>;
+  close(): Promise<void>;
+}
+
+const isCallToolResult = (result: ToolHandlerResult): result is CallToolResult =>
+  typeof result === 'object' && result !== null && 'content' in result;
+
+const isContentArray = (result: ToolHandlerResult): result is readonly ContentBlock[] =>
+  Array.isArray(result);
+
+const normalizeToolResult = (result: ToolHandlerResult): CallToolResult => {
+  if (isContentArray(result)) return { content: [...result] };
+  if (isCallToolResult(result)) return result;
+  return { content: [result] };
+};
+
+const withCors = (res: import('node:http').ServerResponse, origin: string | undefined) => {
+  if (!origin) return;
+  res.setHeader('Access-Control-Allow-Origin', origin);
+  res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
+  res.setHeader(
+    'Access-Control-Allow-Headers',
+    'Content-Type, Accept, MCP-Protocol-Version, Mcp-Session-Id, X-API-Key'
+  );
+};
+
+const isApiKeyValid = (
+  configuredKey: string,
+  presented: string | string[] | undefined
+): boolean => {
+  const provided = Array.isArray(presented) ? presented[0] : presented;
+  if (typeof provided !== 'string' || provided.length === 0) return false;
+  const expected = createHash('sha256').update(configuredKey).digest();
+  const actual = createHash('sha256').update(provided).digest();
+  return timingSafeEqual(expected, actual);
+};
+
+const unauthorized = (res: import('node:http').ServerResponse) => {
+  res.writeHead(401, { 'Content-Type': 'application/json', 'WWW-Authenticate': 'X-API-Key' });
+  res.end(
+    JSON.stringify({
+      jsonrpc: '2.0',
+      id: null,
+      error: { code: -32001, message: 'Unauthorized: missing or invalid X-API-Key header' },
+    })
+  );
+};
+
+const ensureStreamableAcceptHeader = (req: import('node:http').IncomingMessage) => {
+  const accept = req.headers.accept;
+  const acceptValues = Array.isArray(accept) ? accept.join(', ') : (accept ?? '');
+  if (acceptValues.includes('application/json') && acceptValues.includes('text/event-stream')) {
+    return;
+  }
+  req.headers.accept = 'application/json, text/event-stream';
+
+  const rawAcceptIndex = req.rawHeaders.findIndex(
+    (value, index) => index % 2 === 0 && value.toLowerCase() === 'accept'
+  );
+  if (rawAcceptIndex >= 0) {
+    req.rawHeaders[rawAcceptIndex + 1] = 'application/json, text/event-stream';
+    return;
+  }
+  req.rawHeaders.push('Accept', 'application/json, text/event-stream');
+};
+
+const handleNonMcpRoute = (
+  req: import('node:http').IncomingMessage,
+  res: import('node:http').ServerResponse,
+  pathname: string,
+  transportConfig: HttpTransportConfig
+): boolean => {
+  if (pathname === '/mcp/health' && req.method === 'GET') {
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ status: 'ok' }));
+    return true;
+  }
+
+  if (pathname !== '/mcp') {
+    res.writeHead(404, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ error: 'Not found' }));
+    return true;
+  }
+
+  if (req.method === 'OPTIONS') {
+    res.writeHead(204);
+    res.end();
+    return true;
+  }
+
+  if (
+    transportConfig.apiKey !== undefined &&
+    !isApiKeyValid(transportConfig.apiKey, req.headers['x-api-key'])
+  ) {
+    unauthorized(res);
+    return true;
+  }
+
+  return false;
+};
+
+const handleHttpRequest = async (
+  req: import('node:http').IncomingMessage,
+  res: import('node:http').ServerResponse,
+  mcpServer: McpServer,
+  transportConfig: HttpTransportConfig
+): Promise<void> => {
+  const url = new URL(req.url ?? '/', `http://${req.headers.host ?? transportConfig.hostname}`);
+
+  withCors(res, transportConfig.cors);
+
+  if (handleNonMcpRoute(req, res, url.pathname, transportConfig)) return;
+
+  ensureStreamableAcceptHeader(req);
+  const transport = new StreamableHTTPServerTransport({ enableJsonResponse: true });
+
+  try {
+    await mcpServer.connect(transport as unknown as Parameters<McpServer['connect']>[0]);
+    await transport.handleRequest(req, res);
+  } finally {
+    await mcpServer.close();
+  }
+};
+
+const startHttpServer = async (
+  serverInfo: Pick<CreateServerOptions, 'name' | 'version' | 'instructions' | 'tools'>,
+  transportConfig: HttpTransportConfig
+): Promise<{ mcpServer: McpServer; httpServer: HttpServer }> => {
+  const mcpServer = buildMcpServer(serverInfo);
+  const httpServer = createHttpServer((req, res) => {
+    void handleHttpRequest(req, res, mcpServer, transportConfig);
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    httpServer.once('error', reject);
+    httpServer.listen(transportConfig.port, transportConfig.hostname, () => {
+      httpServer.off('error', reject);
+      resolve();
+    });
+  });
+
+  return { mcpServer, httpServer };
+};
+
+const buildMcpServer = ({
+  name,
+  version,
+  instructions,
+  tools,
+}: Pick<CreateServerOptions, 'name' | 'version' | 'instructions' | 'tools'>): McpServer => {
+  const mcpServer = new McpServer(
+    { name, version },
+    {
+      ...(instructions ? { instructions } : {}),
+    }
+  );
+
+  for (const [toolName, definition] of Object.entries(tools)) {
+    mcpServer.registerTool(
+      toolName,
+      {
+        description: definition.description,
+        inputSchema: definition.inputSchema,
+      },
+      async (input, ctx) => normalizeToolResult(await definition.handler({ input, ctx }))
+    );
+  }
+
+  return mcpServer;
+};
+
+export const createServer = (options: CreateServerOptions): ServerHandle => {
+  let mcpServer: McpServer | undefined;
+  let httpServer: HttpServer | undefined;
+
+  return {
+    async start() {
+      if (options.transport.kind === 'stdio') {
+        mcpServer = buildMcpServer(options);
+        await mcpServer.connect(new StdioServerTransport());
+        return;
+      }
+
+      const started = await startHttpServer(options, options.transport);
+      mcpServer = started.mcpServer;
+      httpServer = started.httpServer;
+    },
+    async close() {
+      await mcpServer?.close();
+      const serverToClose = httpServer;
+      if (!serverToClose) return;
+      await new Promise<void>((resolve, reject) => {
+        serverToClose.close((error) => {
+          if (error) {
+            reject(error);
+            return;
+          }
+          resolve();
+        });
+      });
+    },
+  };
+};

package/src/schemas/readVideo.ts

@@ -0,0 +1,67 @@
+import { z } from 'zod';
+
+export const videoSourceSchema = z.object({
+  path: z.string().min(1).describe('Path to the local video file (absolute or relative to cwd).'),
+});
+
+export const readVideoArgsSchema = z.object({
+  sources: z.array(videoSourceSchema).min(1).describe('One or more local video sources to read.'),
+  include_streams: z
+    .boolean()
+    .optional()
+    .describe('Include stream metadata from ffprobe. Defaults to true.'),
+  include_chapters: z
+    .boolean()
+    .optional()
+    .describe('Include chapter markers when present. Defaults to true.'),
+  include_subtitles: z
+    .boolean()
+    .optional()
+    .describe('Extract embedded subtitles when available. Defaults to true.'),
+  include_scenes: z
+    .boolean()
+    .optional()
+    .describe('Detect scene boundaries with ffmpeg scene filter. Defaults to true.'),
+  scene_threshold: z
+    .number()
+    .min(0)
+    .max(1)
+    .optional()
+    .describe('Scene detection sensitivity for ffmpeg gt(scene,threshold). Defaults to 0.4.'),
+  include_transcript: z
+    .boolean()
+    .optional()
+    .describe(
+      'Attempt optional local ASR transcript when an adapter is installed. Defaults to false.'
+    ),
+  include_keyframes: z
+    .boolean()
+    .optional()
+    .describe(
+      'Index I-frame timestamps with ffmpeg for reproducible frame evidence follow-up. Defaults to false.'
+    ),
+  keyframe_limit: z
+    .number()
+    .int()
+    .min(1)
+    .max(64)
+    .optional()
+    .describe(
+      'Maximum number of keyframe locators to return when include_keyframes is true. Defaults to 8.'
+    ),
+  include_keyframe_images: z
+    .boolean()
+    .optional()
+    .describe(
+      'When include_keyframes is true, render citeable PNG thumbnails for each keyframe. Defaults to false.'
+    ),
+  keyframe_max_dimension: z
+    .number()
+    .int()
+    .positive()
+    .optional()
+    .describe('Maximum width or height when resizing keyframe PNG evidence.'),
+});
+
+export type ReadVideoArgs = z.infer<typeof readVideoArgsSchema>;
+export type VideoSource = z.infer<typeof videoSourceSchema>;

package/src/schemas/videoEvidence.ts

@@ -0,0 +1,53 @@
+import { z } from 'zod';
+import { videoSourceSchema } from './readVideo.js';
+
+const cropRegionSchema = z.object({
+  x: z.number().int().min(0).describe('Crop origin X in video pixel coordinates.'),
+  y: z.number().int().min(0).describe('Crop origin Y in video pixel coordinates.'),
+  width: z.number().int().positive().describe('Crop width in video pixel coordinates.'),
+  height: z.number().int().positive().describe('Crop height in video pixel coordinates.'),
+});
+
+const evidenceSourceSchema = videoSourceSchema.extend({
+  time_ms: z
+    .number()
+    .int()
+    .min(0)
+    .describe('Timestamp in milliseconds for frame evidence follow-up.'),
+  crop: cropRegionSchema
+    .optional()
+    .describe('Required for crop_frame; pixel crop bounds on the source video frame.'),
+});
+
+export const videoEvidenceArgsSchema = z
+  .object({
+    operation: z
+      .enum(['render_frame', 'crop_frame', 'ocr_frame'])
+      .describe('Focused frame evidence operation after read_video timeline discovery.'),
+    sources: z
+      .array(evidenceSourceSchema)
+      .min(1)
+      .describe('One or more local video sources with timestamp locators.'),
+    max_dimension: z
+      .number()
+      .int()
+      .positive()
+      .optional()
+      .describe('Maximum width or height when resizing rendered PNG evidence.'),
+  })
+  .superRefine((value, ctx) => {
+    if (value.operation === 'crop_frame') {
+      for (const [index, source] of value.sources.entries()) {
+        if (!source.crop) {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: 'crop_frame requires sources[].crop for each source.',
+            path: ['sources', index, 'crop'],
+          });
+        }
+      }
+    }
+  });
+
+export type VideoEvidenceArgs = z.infer<typeof videoEvidenceArgsSchema>;
+export type VideoEvidenceSource = z.infer<typeof evidenceSourceSchema>;

package/src/sdk.ts

@@ -0,0 +1,44 @@
+/**
+ * Cue SDK — programmatic video timeline API (Sylphx).
+ * Isomorphic with MCP tools `read_video` / `video_evidence`.
+ */
+import { readVideo } from './handlers/readVideo.js';
+import { videoEvidence } from './handlers/videoEvidence.js';
+import { type ReadVideoArgs, readVideoArgsSchema } from './schemas/readVideo.js';
+
+export type CueReadInput = ReadVideoArgs | { path: string; [key: string]: unknown };
+
+export { readVideoArgsSchema };
+
+function normalizeReadInput(input: CueReadInput): ReadVideoArgs {
+  if ('sources' in input && Array.isArray((input as ReadVideoArgs).sources)) {
+    return readVideoArgsSchema.parse(input);
+  }
+  const { path, ...rest } = input as { path: string; [key: string]: unknown };
+  if (!path || typeof path !== 'string') {
+    throw new Error('Cue.read requires sources[] or path');
+  }
+  return readVideoArgsSchema.parse({
+    ...rest,
+    sources: [{ path }],
+  });
+}
+
+export class Cue {
+  static create(): Cue {
+    return new Cue();
+  }
+
+  /** MCP: read_video — accepts `{ sources:[{path}] }` or ergonomic `{ path }` */
+  async read(input: CueReadInput) {
+    const parsed = normalizeReadInput(input);
+    return readVideo.handler({ input: parsed, ctx: {} });
+  }
+
+  /** MCP: video_evidence */
+  async evidence(input: Record<string, unknown>) {
+    return videoEvidence.handler({ input: input as never, ctx: {} });
+  }
+}
+
+export default Cue;