@vellumai/assistant 0.3.5 → 0.3.7

package/src/config/bundled-skills/media-processing/services/reduce.ts

@@ -0,0 +1,197 @@
+/**
+ * Reduce service — sends Map output to Claude as text-only for analysis.
+ *
+ * Two modes:
+ * - One-shot merge: assembles all Map results into a single document,
+ *   sends to Claude with the provided system prompt for analysis.
+ * - Interactive Q&A: loads existing map output + user query, sends to
+ *   Claude, returns the answer.
+ *
+ * Uses the existing provider infrastructure (getConfiguredProvider) so
+ * it works with whatever LLM provider is configured.
+ */
+
+import { readFile } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { getMediaAssetById } from '../../../../memory/media-store.js';
+import {
+  getConfiguredProvider,
+  createTimeout,
+  extractAllText,
+  userMessage,
+} from '../../../../providers/provider-send-message.js';
+import type { MapOutput } from './gemini-map.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ReduceOptions {
+  /** Natural language query about the video data. Optional for one-shot merge mode. */
+  query?: string;
+  /** Optional system prompt for Claude. */
+  systemPrompt?: string;
+  /** Model override. When omitted, the configured provider's default is used. */
+  model?: string;
+}
+
+export interface ReduceResult {
+  answer: string;
+  model: string;
+  inputTokens: number;
+  outputTokens: number;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+const REDUCE_TIMEOUT_MS = 120_000;
+
+/**
+ * Load map-output.json for an asset from its pipeline directory.
+ */
+async function loadMapOutput(assetId: string): Promise<MapOutput> {
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    throw new Error(`Media asset not found: ${assetId}`);
+  }
+
+  const pipelineDir = join(dirname(asset.filePath), 'pipeline', assetId);
+  const mapOutputPath = join(pipelineDir, 'map-output.json');
+
+  let raw: string;
+  try {
+    raw = await readFile(mapOutputPath, 'utf-8');
+  } catch {
+    throw new Error(
+      'No map output found. Run analyze_keyframes first to generate map-output.json.',
+    );
+  }
+
+  return JSON.parse(raw) as MapOutput;
+}
+
+/**
+ * Format map output segments into a text document for Claude.
+ * Strips image data — text only.
+ */
+function formatMapOutputAsText(mapOutput: MapOutput): string {
+  const lines: string[] = [];
+
+  lines.push(`Video Analysis Data (asset: ${mapOutput.assetId})`);
+  lines.push(`Model: ${mapOutput.model}`);
+  lines.push(`Segments analyzed: ${mapOutput.successCount}/${mapOutput.segmentCount}`);
+  if (mapOutput.failedCount > 0) {
+    lines.push(`Failed segments: ${mapOutput.failedCount}`);
+  }
+  if (mapOutput.skippedCount > 0) {
+    lines.push(`Skipped segments: ${mapOutput.skippedCount}`);
+  }
+  lines.push('');
+  lines.push('--- Segment Results ---');
+  lines.push('');
+
+  for (const segment of mapOutput.segments) {
+    const startMin = Math.floor(segment.startSeconds / 60);
+    const startSec = Math.floor(segment.startSeconds % 60);
+    const endMin = Math.floor(segment.endSeconds / 60);
+    const endSec = Math.floor(segment.endSeconds % 60);
+    const timeRange = `${startMin}:${String(startSec).padStart(2, '0')} - ${endMin}:${String(endSec).padStart(2, '0')}`;
+
+    lines.push(`[Segment ${segment.segmentId}] ${timeRange}`);
+    lines.push(JSON.stringify(segment.result, null, 2));
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Core: send to Claude via provider infrastructure
+// ---------------------------------------------------------------------------
+
+async function sendToClaude(
+  mapText: string,
+  query: string,
+  systemPrompt?: string,
+  model?: string,
+  onProgress?: (msg: string) => void,
+): Promise<ReduceResult> {
+  const provider = getConfiguredProvider();
+  if (!provider) {
+    throw new Error('No LLM provider available. Please configure an API key.');
+  }
+
+  const effectiveSystemPrompt = systemPrompt
+    ?? 'You are an expert video analyst. You have been given structured analysis data extracted from a video. Answer the user\'s question based on this data. Be specific, reference timestamps when relevant, and provide clear, actionable insights.';
+
+  const userContent = `Here is the video analysis data:\n\n${mapText}\n\n---\n\nUser query: ${query}`;
+
+  onProgress?.('Sending map output to Claude for analysis...\n');
+
+  const { signal, cleanup } = createTimeout(REDUCE_TIMEOUT_MS);
+
+  try {
+    const response = await provider.sendMessage(
+      [userMessage(userContent)],
+      [],
+      effectiveSystemPrompt,
+      {
+        config: model ? { model } : {},
+        signal,
+      },
+    );
+    cleanup();
+
+    const answer = extractAllText(response);
+
+    onProgress?.(`Reduce complete (${response.usage.inputTokens} input + ${response.usage.outputTokens} output tokens).\n`);
+
+    return {
+      answer,
+      model: response.model,
+      inputTokens: response.usage.inputTokens,
+      outputTokens: response.usage.outputTokens,
+    };
+  } finally {
+    cleanup();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * One-shot merge mode: load map output for an asset, send all data to
+ * Claude with a system prompt, and return the analysis.
+ */
+export async function reduceForAsset(
+  assetId: string,
+  options: ReduceOptions,
+  onProgress?: (msg: string) => void,
+): Promise<ReduceResult> {
+  const mapOutput = await loadMapOutput(assetId);
+
+  if (mapOutput.segments.length === 0) {
+    throw new Error('Map output contains no segments. Run analyze_keyframes first.');
+  }
+
+  const mapText = formatMapOutputAsText(mapOutput);
+  const effectiveQuery = options.query ?? 'Summarize the video content.';
+  return sendToClaude(mapText, effectiveQuery, options.systemPrompt, options.model, onProgress);
+}
+
+/**
+ * Interactive Q&A mode: load existing map output + user query, send to
+ * Claude, return the answer. Functionally identical to reduceForAsset
+ * but named distinctly for clarity in call sites.
+ */
+export async function queryMapOutput(
+  assetId: string,
+  options: ReduceOptions,
+  onProgress?: (msg: string) => void,
+): Promise<ReduceResult> {
+  return reduceForAsset(assetId, options, onProgress);
+}

package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts

@@ -1,42 +1,36 @@
+import { join, dirname } from 'node:path';
 import { readFile } from 'node:fs/promises';
-import Anthropic from '@anthropic-ai/sdk';
-import { getConfig } from '../../../../config/loader.js';
 import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import { getConfig } from '../../../../config/loader.js';
 import {
   getMediaAssetById,
-  getKeyframesForAsset,
-  getVisionOutputsForAsset,
-  insertVisionOutputsBatch,
-  createProcessingStage,
-  updateProcessingStage,
-  getProcessingStagesForAsset,
-  type MediaKeyframe,
-  type ProcessingStage,
 } from '../../../../memory/media-store.js';
-
-const VLM_PROMPT = `Analyze this image frame extracted from a video. Return a JSON object with the following fields:
-
-{
-  "sceneDescription": "A concise description of the overall scene",
-  "subjects": ["List of identifiable subjects/objects/people in the frame"],
-  "actions": ["List of actions or activities occurring"],
-  "context": "Environmental or situational context (setting, conditions, etc.)"
+import { mapSegments, type MapOutput } from '../services/gemini-map.js';
+import type { PreprocessManifest } from '../services/preprocess.js';
+
+// ---------------------------------------------------------------------------
+// Exported function for job handler use
+// ---------------------------------------------------------------------------
+
+export interface MapSegmentsOptions {
+  systemPrompt: string;
+  outputSchema: Record<string, unknown>;
+  context?: Record<string, unknown>;
+  model?: string;
+  concurrency?: number;
+  maxRetries?: number;
 }
 
-Return ONLY the JSON object, no additional text.`;
-
-export async function analyzeKeyframesForAsset(
+export async function mapSegmentsForAsset(
   assetId: string,
-  analysisType?: string,
-  batchSize?: number,
+  options: MapSegmentsOptions,
   onProgress?: (msg: string) => void,
-  signal?: AbortSignal,
-): Promise<void> {
-  const type = analysisType ?? 'scene_description';
-  const batch = batchSize ?? 10;
+): Promise<MapOutput> {
+  const config = getConfig();
+  const apiKey = config.apiKeys.gemini;
 
-  if (batch <= 0) {
-    throw new Error('batch_size must be greater than 0.');
+  if (!apiKey) {
+    throw new Error('No Gemini API key configured. Please set your Gemini API key to use keyframe analysis.');
   }
 
   const asset = getMediaAssetById(assetId);
@@ -44,123 +38,37 @@ export async function analyzeKeyframesForAsset(
     throw new Error(`Media asset not found: ${assetId}`);
   }
 
-  // Get all keyframes for this asset
-  const keyframes = getKeyframesForAsset(assetId);
-  if (keyframes.length === 0) {
-    throw new Error('No keyframes found for this asset. Run extract_keyframes first.');
-  }
-
-  // Resumability: find already-analyzed keyframe IDs for this analysis type
-  const existingOutputs = getVisionOutputsForAsset(assetId, type);
-  const analyzedKeyframeIds = new Set(existingOutputs.map((o) => o.keyframeId));
-  const pendingKeyframes = keyframes.filter((kf) => !analyzedKeyframeIds.has(kf.id));
+  // Load preprocess manifest
+  const pipelineDir = join(dirname(asset.filePath), 'pipeline', assetId);
+  const manifestPath = join(pipelineDir, 'manifest.json');
 
-  if (pendingKeyframes.length === 0) {
-    // Nothing to do — all keyframes already analyzed
-    return;
+  let manifest: PreprocessManifest;
+  try {
+    const raw = await readFile(manifestPath, 'utf-8');
+    manifest = JSON.parse(raw) as PreprocessManifest;
+  } catch {
+    throw new Error('No preprocess manifest found. Run extract_keyframes first.');
   }
 
-  // Find or create the vision_analysis processing stage
-  let stage: ProcessingStage | undefined;
-  const existingStages = getProcessingStagesForAsset(assetId);
-  stage = existingStages.find((s) => s.stage === 'vision_analysis');
-  if (!stage) {
-    stage = createProcessingStage({ assetId, stage: 'vision_analysis' });
+  if (manifest.segments.length === 0) {
+    throw new Error('No segments found in preprocess manifest. Run extract_keyframes first.');
   }
 
-  updateProcessingStage(stage.id, { status: 'running', startedAt: Date.now() });
-
-  const config = getConfig();
-  const apiKey = config.apiKeys.anthropic ?? process.env.ANTHROPIC_API_KEY;
-  if (!apiKey) {
-    updateProcessingStage(stage.id, {
-      status: 'failed',
-      lastError: 'Anthropic API key not configured',
-    });
-    throw new Error('No Anthropic API key available. Configure it in settings or set ANTHROPIC_API_KEY.');
-  }
-
-  const client = new Anthropic({ apiKey });
-  let analyzedCount = analyzedKeyframeIds.size;
-  const totalKeyframes = keyframes.length;
-
-  onProgress?.(`Analyzing ${pendingKeyframes.length} keyframes (${analyzedKeyframeIds.size} already done)...\n`);
-
-  let aborted = false;
-
-  try {
-    // Process in batches
-    for (let i = 0; i < pendingKeyframes.length; i += batch) {
-      if (signal?.aborted) {
-        onProgress?.('Aborted.\n');
-        aborted = true;
-        break;
-      }
-
-      const currentBatch = pendingKeyframes.slice(i, i + batch);
-      const batchResults: Array<{
-        assetId: string;
-        keyframeId: string;
-        analysisType: string;
-        output: Record<string, unknown>;
-        confidence?: number;
-      }> = [];
-
-      for (const keyframe of currentBatch) {
-        if (signal?.aborted) {
-          onProgress?.('Aborted.\n');
-          aborted = true;
-          break;
-        }
-
-        try {
-          const result = await analyzeKeyframe(client, keyframe);
-          batchResults.push({
-            assetId,
-            keyframeId: keyframe.id,
-            analysisType: type,
-            output: result.output,
-            confidence: result.confidence,
-          });
-          analyzedCount++;
-        } catch (err) {
-          onProgress?.(`  Warning: failed to analyze frame at ${keyframe.timestamp}s: ${(err as Error).message}\n`);
-        }
-      }
-
-      // Batch insert results
-      if (batchResults.length > 0) {
-        insertVisionOutputsBatch(batchResults);
-      }
-
-      // Update progress
-      const progress = Math.round((analyzedCount / totalKeyframes) * 100);
-      updateProcessingStage(stage.id, { progress });
-
-      onProgress?.(`  Batch ${Math.floor(i / batch) + 1}: analyzed ${batchResults.length}/${currentBatch.length} frames (${progress}% total)\n`);
-    }
-
-    if (aborted) {
-      throw new Error('Analysis aborted');
-    }
-
-    const finalProgress = Math.round((analyzedCount / totalKeyframes) * 100);
-    const isComplete = analyzedCount >= totalKeyframes;
-
-    updateProcessingStage(stage.id, {
-      status: isComplete ? 'completed' : 'running',
-      progress: finalProgress,
-      ...(isComplete ? { completedAt: Date.now() } : {}),
-    });
-  } catch (err) {
-    updateProcessingStage(stage.id, {
-      status: 'failed',
-      lastError: (err as Error).message.slice(0, 500),
-    });
-    throw err;
-  }
+  return mapSegments(assetId, pipelineDir, manifest.segments, {
+    apiKey,
+    systemPrompt: options.systemPrompt,
+    outputSchema: options.outputSchema,
+    context: options.context,
+    model: options.model,
+    concurrency: options.concurrency,
+    maxRetries: options.maxRetries,
+  }, onProgress);
 }
 
+// ---------------------------------------------------------------------------
+// Tool entry point
+// ---------------------------------------------------------------------------
+
 export async function run(
   input: Record<string, unknown>,
   context: ToolContext,
@@ -170,132 +78,59 @@ export async function run(
     return { content: 'asset_id is required.', isError: true };
   }
 
-  const analysisType = (input.analysis_type as string) || 'scene_description';
-  const batchSize = (input.batch_size as number) || 10;
+  const systemPrompt = input.system_prompt as string | undefined;
+  if (!systemPrompt) {
+    return { content: 'system_prompt is required.', isError: true };
+  }
 
-  try {
-    // Check if all keyframes are already analyzed before calling the core function
-    const keyframes = getKeyframesForAsset(assetId);
-    const existingOutputs = getVisionOutputsForAsset(assetId, analysisType);
-    const analyzedKeyframeIds = new Set(existingOutputs.map((o) => o.keyframeId));
-    const pendingKeyframes = keyframes.filter((kf) => !analyzedKeyframeIds.has(kf.id));
+  const outputSchema = input.output_schema as Record<string, unknown> | undefined;
+  if (!outputSchema) {
+    return { content: 'output_schema is required.', isError: true };
+  }
 
-    if (keyframes.length > 0 && pendingKeyframes.length === 0) {
-      return {
-        content: JSON.stringify({
-          message: 'All keyframes already analyzed',
-          assetId,
-          analysisType,
-          totalKeyframes: keyframes.length,
-          alreadyAnalyzed: existingOutputs.length,
-        }, null, 2),
-        isError: false,
-      };
-    }
+  const contextObj = input.context as Record<string, unknown> | undefined;
+  const model = input.model as string | undefined;
+  const concurrency = input.concurrency as number | undefined;
+  const maxRetries = input.max_retries as number | undefined;
 
-    await analyzeKeyframesForAsset(assetId, analysisType, batchSize, context.onOutput, context.signal);
+  if (concurrency !== undefined && concurrency < 1) {
+    return { content: 'concurrency must be at least 1.', isError: true };
+  }
+  if (maxRetries !== undefined && maxRetries < 0) {
+    return { content: 'max_retries must be non-negative.', isError: true };
+  }
 
-    // Gather final stats
-    const allKeyframes = getKeyframesForAsset(assetId);
-    const allOutputs = getVisionOutputsForAsset(assetId, analysisType);
-    const totalKeyframes = allKeyframes.length;
-    const analyzedCount = allOutputs.length;
-    const finalProgress = Math.round((analyzedCount / totalKeyframes) * 100);
-    const isComplete = analyzedCount >= totalKeyframes;
+  try {
+    const output = await mapSegmentsForAsset(
+      assetId,
+      {
+        systemPrompt,
+        outputSchema,
+        context: contextObj,
+        model,
+        concurrency,
+        maxRetries,
+      },
+      context.onOutput,
+    );
 
     return {
       content: JSON.stringify({
-        message: `Vision analysis ${isComplete ? 'completed' : 'in progress'}`,
+        message: `Map ${output.failedCount === 0 ? 'completed' : 'completed with errors'}`,
         assetId,
-        analysisType,
-        totalKeyframes,
-        analyzedCount,
-        newlyAnalyzed: analyzedCount - analyzedKeyframeIds.size,
-        errorCount: pendingKeyframes.length - (analyzedCount - analyzedKeyframeIds.size),
-        progress: finalProgress,
+        model: output.model,
+        segmentCount: output.segmentCount,
+        successCount: output.successCount,
+        failedCount: output.failedCount,
+        skippedCount: output.skippedCount,
+        totalInputTokens: output.costSummary.totalInputTokens,
+        totalOutputTokens: output.costSummary.totalOutputTokens,
+        estimatedCostUSD: output.costSummary.totalEstimatedUSD,
       }, null, 2),
       isError: false,
     };
   } catch (err) {
     const msg = (err as Error).message;
-    // Preserve original error message format
-    if (
-      msg === 'batch_size must be greater than 0.' ||
-      msg.startsWith('Media asset not found:') ||
-      msg === 'No keyframes found for this asset. Run extract_keyframes first.' ||
-      msg === 'No Anthropic API key available. Configure it in settings or set ANTHROPIC_API_KEY.'
-    ) {
-      return { content: msg, isError: true };
-    }
-    return { content: `Vision analysis failed: ${(err as Error).message}`, isError: true };
+    return { content: msg, isError: true };
   }
 }
-
-async function analyzeKeyframe(
-  client: Anthropic,
-  keyframe: MediaKeyframe,
-): Promise<{ output: Record<string, unknown>; confidence: number }> {
-  // Read the image file and encode as base64
-  const imageData = await readFile(keyframe.filePath);
-  const base64 = imageData.toString('base64');
-
-  // Determine media type from file extension
-  const ext = keyframe.filePath.split('.').pop()?.toLowerCase() ?? 'jpg';
-  const mediaTypeMap: Record<string, string> = {
-    jpg: 'image/jpeg',
-    jpeg: 'image/jpeg',
-    png: 'image/png',
-    gif: 'image/gif',
-    webp: 'image/webp',
-  };
-  const mediaType = mediaTypeMap[ext] ?? 'image/jpeg';
-
-  const response = await client.messages.create({
-    model: 'claude-sonnet-4-6-20250514',
-    max_tokens: 1024,
-    messages: [
-      {
-        role: 'user',
-        content: [
-          {
-            type: 'image',
-            source: {
-              type: 'base64',
-              media_type: mediaType as 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp',
-              data: base64,
-            },
-          },
-          {
-            type: 'text',
-            text: VLM_PROMPT,
-          },
-        ],
-      },
-    ],
-  });
-
-  // Extract text from response
-  const textBlock = response.content.find((block) => block.type === 'text');
-  const responseText = textBlock && 'text' in textBlock ? textBlock.text : '';
-
-  // Parse JSON from response
-  let output: Record<string, unknown>;
-  try {
-    // Try to extract JSON from the response (handle markdown code fences)
-    const jsonMatch = responseText.match(/```(?:json)?\s*([\s\S]*?)```/) ?? [null, responseText];
-    output = JSON.parse(jsonMatch[1]!.trim()) as Record<string, unknown>;
-  } catch {
-    // If JSON parsing fails, wrap raw text as output
-    output = {
-      sceneDescription: responseText,
-      subjects: [],
-      actions: [],
-      context: '',
-    };
-  }
-
-  // Add timestamp context to the output
-  output.timestamp = keyframe.timestamp;
-
-  return { output, confidence: 0.8 };
-}