@vellumai/assistant 0.3.3 → 0.3.5

package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts

@@ -0,0 +1,301 @@
+import { readFile } from 'node:fs/promises';
+import Anthropic from '@anthropic-ai/sdk';
+import { getConfig } from '../../../../config/loader.js';
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import {
+  getMediaAssetById,
+  getKeyframesForAsset,
+  getVisionOutputsForAsset,
+  insertVisionOutputsBatch,
+  createProcessingStage,
+  updateProcessingStage,
+  getProcessingStagesForAsset,
+  type MediaKeyframe,
+  type ProcessingStage,
+} from '../../../../memory/media-store.js';
+
+const VLM_PROMPT = `Analyze this image frame extracted from a video. Return a JSON object with the following fields:
+
+{
+  "sceneDescription": "A concise description of the overall scene",
+  "subjects": ["List of identifiable subjects/objects/people in the frame"],
+  "actions": ["List of actions or activities occurring"],
+  "context": "Environmental or situational context (setting, conditions, etc.)"
+}
+
+Return ONLY the JSON object, no additional text.`;
+
+export async function analyzeKeyframesForAsset(
+  assetId: string,
+  analysisType?: string,
+  batchSize?: number,
+  onProgress?: (msg: string) => void,
+  signal?: AbortSignal,
+): Promise<void> {
+  const type = analysisType ?? 'scene_description';
+  const batch = batchSize ?? 10;
+
+  if (batch <= 0) {
+    throw new Error('batch_size must be greater than 0.');
+  }
+
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    throw new Error(`Media asset not found: ${assetId}`);
+  }
+
+  // Get all keyframes for this asset
+  const keyframes = getKeyframesForAsset(assetId);
+  if (keyframes.length === 0) {
+    throw new Error('No keyframes found for this asset. Run extract_keyframes first.');
+  }
+
+  // Resumability: find already-analyzed keyframe IDs for this analysis type
+  const existingOutputs = getVisionOutputsForAsset(assetId, type);
+  const analyzedKeyframeIds = new Set(existingOutputs.map((o) => o.keyframeId));
+  const pendingKeyframes = keyframes.filter((kf) => !analyzedKeyframeIds.has(kf.id));
+
+  if (pendingKeyframes.length === 0) {
+    // Nothing to do — all keyframes already analyzed
+    return;
+  }
+
+  // Find or create the vision_analysis processing stage
+  let stage: ProcessingStage | undefined;
+  const existingStages = getProcessingStagesForAsset(assetId);
+  stage = existingStages.find((s) => s.stage === 'vision_analysis');
+  if (!stage) {
+    stage = createProcessingStage({ assetId, stage: 'vision_analysis' });
+  }
+
+  updateProcessingStage(stage.id, { status: 'running', startedAt: Date.now() });
+
+  const config = getConfig();
+  const apiKey = config.apiKeys.anthropic ?? process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) {
+    updateProcessingStage(stage.id, {
+      status: 'failed',
+      lastError: 'Anthropic API key not configured',
+    });
+    throw new Error('No Anthropic API key available. Configure it in settings or set ANTHROPIC_API_KEY.');
+  }
+
+  const client = new Anthropic({ apiKey });
+  let analyzedCount = analyzedKeyframeIds.size;
+  const totalKeyframes = keyframes.length;
+
+  onProgress?.(`Analyzing ${pendingKeyframes.length} keyframes (${analyzedKeyframeIds.size} already done)...\n`);
+
+  let aborted = false;
+
+  try {
+    // Process in batches
+    for (let i = 0; i < pendingKeyframes.length; i += batch) {
+      if (signal?.aborted) {
+        onProgress?.('Aborted.\n');
+        aborted = true;
+        break;
+      }
+
+      const currentBatch = pendingKeyframes.slice(i, i + batch);
+      const batchResults: Array<{
+        assetId: string;
+        keyframeId: string;
+        analysisType: string;
+        output: Record<string, unknown>;
+        confidence?: number;
+      }> = [];
+
+      for (const keyframe of currentBatch) {
+        if (signal?.aborted) {
+          onProgress?.('Aborted.\n');
+          aborted = true;
+          break;
+        }
+
+        try {
+          const result = await analyzeKeyframe(client, keyframe);
+          batchResults.push({
+            assetId,
+            keyframeId: keyframe.id,
+            analysisType: type,
+            output: result.output,
+            confidence: result.confidence,
+          });
+          analyzedCount++;
+        } catch (err) {
+          onProgress?.(`  Warning: failed to analyze frame at ${keyframe.timestamp}s: ${(err as Error).message}\n`);
+        }
+      }
+
+      // Batch insert results
+      if (batchResults.length > 0) {
+        insertVisionOutputsBatch(batchResults);
+      }
+
+      // Update progress
+      const progress = Math.round((analyzedCount / totalKeyframes) * 100);
+      updateProcessingStage(stage.id, { progress });
+
+      onProgress?.(`  Batch ${Math.floor(i / batch) + 1}: analyzed ${batchResults.length}/${currentBatch.length} frames (${progress}% total)\n`);
+    }
+
+    if (aborted) {
+      throw new Error('Analysis aborted');
+    }
+
+    const finalProgress = Math.round((analyzedCount / totalKeyframes) * 100);
+    const isComplete = analyzedCount >= totalKeyframes;
+
+    updateProcessingStage(stage.id, {
+      status: isComplete ? 'completed' : 'running',
+      progress: finalProgress,
+      ...(isComplete ? { completedAt: Date.now() } : {}),
+    });
+  } catch (err) {
+    updateProcessingStage(stage.id, {
+      status: 'failed',
+      lastError: (err as Error).message.slice(0, 500),
+    });
+    throw err;
+  }
+}
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  if (!assetId) {
+    return { content: 'asset_id is required.', isError: true };
+  }
+
+  const analysisType = (input.analysis_type as string) || 'scene_description';
+  const batchSize = (input.batch_size as number) || 10;
+
+  try {
+    // Check if all keyframes are already analyzed before calling the core function
+    const keyframes = getKeyframesForAsset(assetId);
+    const existingOutputs = getVisionOutputsForAsset(assetId, analysisType);
+    const analyzedKeyframeIds = new Set(existingOutputs.map((o) => o.keyframeId));
+    const pendingKeyframes = keyframes.filter((kf) => !analyzedKeyframeIds.has(kf.id));
+
+    if (keyframes.length > 0 && pendingKeyframes.length === 0) {
+      return {
+        content: JSON.stringify({
+          message: 'All keyframes already analyzed',
+          assetId,
+          analysisType,
+          totalKeyframes: keyframes.length,
+          alreadyAnalyzed: existingOutputs.length,
+        }, null, 2),
+        isError: false,
+      };
+    }
+
+    await analyzeKeyframesForAsset(assetId, analysisType, batchSize, context.onOutput, context.signal);
+
+    // Gather final stats
+    const allKeyframes = getKeyframesForAsset(assetId);
+    const allOutputs = getVisionOutputsForAsset(assetId, analysisType);
+    const totalKeyframes = allKeyframes.length;
+    const analyzedCount = allOutputs.length;
+    const finalProgress = Math.round((analyzedCount / totalKeyframes) * 100);
+    const isComplete = analyzedCount >= totalKeyframes;
+
+    return {
+      content: JSON.stringify({
+        message: `Vision analysis ${isComplete ? 'completed' : 'in progress'}`,
+        assetId,
+        analysisType,
+        totalKeyframes,
+        analyzedCount,
+        newlyAnalyzed: analyzedCount - analyzedKeyframeIds.size,
+        errorCount: pendingKeyframes.length - (analyzedCount - analyzedKeyframeIds.size),
+        progress: finalProgress,
+      }, null, 2),
+      isError: false,
+    };
+  } catch (err) {
+    const msg = (err as Error).message;
+    // Preserve original error message format
+    if (
+      msg === 'batch_size must be greater than 0.' ||
+      msg.startsWith('Media asset not found:') ||
+      msg === 'No keyframes found for this asset. Run extract_keyframes first.' ||
+      msg === 'No Anthropic API key available. Configure it in settings or set ANTHROPIC_API_KEY.'
+    ) {
+      return { content: msg, isError: true };
+    }
+    return { content: `Vision analysis failed: ${(err as Error).message}`, isError: true };
+  }
+}
+
+async function analyzeKeyframe(
+  client: Anthropic,
+  keyframe: MediaKeyframe,
+): Promise<{ output: Record<string, unknown>; confidence: number }> {
+  // Read the image file and encode as base64
+  const imageData = await readFile(keyframe.filePath);
+  const base64 = imageData.toString('base64');
+
+  // Determine media type from file extension
+  const ext = keyframe.filePath.split('.').pop()?.toLowerCase() ?? 'jpg';
+  const mediaTypeMap: Record<string, string> = {
+    jpg: 'image/jpeg',
+    jpeg: 'image/jpeg',
+    png: 'image/png',
+    gif: 'image/gif',
+    webp: 'image/webp',
+  };
+  const mediaType = mediaTypeMap[ext] ?? 'image/jpeg';
+
+  const response = await client.messages.create({
+    model: 'claude-sonnet-4-6-20250514',
+    max_tokens: 1024,
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'image',
+            source: {
+              type: 'base64',
+              media_type: mediaType as 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp',
+              data: base64,
+            },
+          },
+          {
+            type: 'text',
+            text: VLM_PROMPT,
+          },
+        ],
+      },
+    ],
+  });
+
+  // Extract text from response
+  const textBlock = response.content.find((block) => block.type === 'text');
+  const responseText = textBlock && 'text' in textBlock ? textBlock.text : '';
+
+  // Parse JSON from response
+  let output: Record<string, unknown>;
+  try {
+    // Try to extract JSON from the response (handle markdown code fences)
+    const jsonMatch = responseText.match(/```(?:json)?\s*([\s\S]*?)```/) ?? [null, responseText];
+    output = JSON.parse(jsonMatch[1]!.trim()) as Record<string, unknown>;
+  } catch {
+    // If JSON parsing fails, wrap raw text as output
+    output = {
+      sceneDescription: responseText,
+      subjects: [],
+      actions: [],
+      context: '',
+    };
+  }
+
+  // Add timestamp context to the output
+  output.timestamp = keyframe.timestamp;
+
+  return { output, confidence: 0.8 };
+}

package/src/config/bundled-skills/media-processing/tools/detect-events.ts

@@ -0,0 +1,110 @@
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import { detectEvents, type DetectionConfig, type DetectionRule } from '../services/event-detection-service.js';
+
+/**
+ * Sensible default detection rules for common event types.
+ * These are fallbacks when the caller doesn't provide explicit rules —
+ * the system is not limited to these event types.
+ */
+const DEFAULT_RULES_BY_EVENT_TYPE: Record<string, DetectionRule[]> = {
+  turnover: [
+    { ruleType: 'segment_transition', params: { field: 'subjects' }, weight: 0.5 },
+    { ruleType: 'short_segment', params: { maxDurationSeconds: 5 }, weight: 0.3 },
+    { ruleType: 'attribute_match', params: { field: 'actions', pattern: 'steal|turnover|loss|intercept' }, weight: 0.2 },
+  ],
+  scene_change: [
+    { ruleType: 'segment_transition', params: { field: 'segmentType' }, weight: 1.0 },
+  ],
+  short_play: [
+    { ruleType: 'short_segment', params: { maxDurationSeconds: 3 }, weight: 0.6 },
+    { ruleType: 'segment_transition', params: { field: 'subjects' }, weight: 0.4 },
+  ],
+};
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  if (!assetId) {
+    return { content: 'asset_id is required.', isError: true };
+  }
+
+  const eventType = input.event_type as string | undefined;
+  if (!eventType) {
+    return { content: 'event_type is required.', isError: true };
+  }
+
+  // Parse detection rules: use provided rules or fall back to defaults
+  let rules: DetectionRule[];
+  const rawRules = input.detection_rules;
+
+  if (rawRules) {
+    // Accept rules as a JSON string or as an already-parsed array
+    if (typeof rawRules === 'string') {
+      try {
+        const parsed = JSON.parse(rawRules);
+        if (!Array.isArray(parsed)) {
+          return { content: 'detection_rules must be a valid JSON array of rule objects.', isError: true };
+        }
+        rules = parsed as DetectionRule[];
+      } catch {
+        return { content: 'detection_rules must be a valid JSON array of rule objects.', isError: true };
+      }
+    } else if (Array.isArray(rawRules)) {
+      rules = rawRules as DetectionRule[];
+    } else {
+      return { content: 'detection_rules must be an array of rule objects.', isError: true };
+    }
+
+    // Validate each rule has the required shape
+    for (const rule of rules) {
+      if (!rule.ruleType || typeof rule.ruleType !== 'string') {
+        return { content: 'Each detection rule must have a "ruleType" string.', isError: true };
+      }
+      if (rule.weight === undefined || typeof rule.weight !== 'number') {
+        return { content: 'Each detection rule must have a "weight" number.', isError: true };
+      }
+      if (!rule.params || typeof rule.params !== 'object') {
+        return { content: 'Each detection rule must have a "params" object.', isError: true };
+      }
+    }
+  } else {
+    // Use defaults for known event types, or a generic transition-based fallback
+    rules = DEFAULT_RULES_BY_EVENT_TYPE[eventType] ?? [
+      { ruleType: 'segment_transition', params: { field: 'segmentType' }, weight: 0.6 },
+      { ruleType: 'short_segment', params: { maxDurationSeconds: 5 }, weight: 0.4 },
+    ];
+  }
+
+  const config: DetectionConfig = { eventType, rules };
+
+  try {
+    const result = detectEvents(assetId, config, {
+      onProgress: (msg) => context.onOutput?.(`${msg}\n`),
+    });
+
+    return {
+      content: JSON.stringify({
+        message: `Detected ${result.candidateCount} ${result.eventType} events`,
+        assetId: result.assetId,
+        eventType: result.eventType,
+        totalEvents: result.candidateCount,
+        rulesUsed: rules.map((r) => r.ruleType),
+        events: result.events.map((e) => ({
+          id: e.id,
+          startTime: e.startTime,
+          endTime: e.endTime,
+          confidence: e.confidence,
+          reasons: e.reasons,
+        })),
+      }, null, 2),
+      isError: false,
+    };
+  } catch (err) {
+    return {
+      content: `Event detection failed: ${(err as Error).message}`,
+      isError: true,
+    };
+  }
+}

package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts

@@ -0,0 +1,190 @@
+import { join, dirname } from 'node:path';
+import { mkdir, readdir, rename, rm } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import {
+  getMediaAssetById,
+  getKeyframesForAsset,
+  insertKeyframesBatch,
+  deleteKeyframesForAsset,
+  createProcessingStage,
+  updateProcessingStage,
+  getProcessingStagesForAsset,
+  type ProcessingStage,
+} from '../../../../memory/media-store.js';
+
+const FFMPEG_TIMEOUT_MS = 300_000;
+
+function spawnWithTimeout(
+  cmd: string[],
+  timeoutMs: number,
+): Promise<{ exitCode: number; stdout: string; stderr: string }> {
+  return new Promise((resolve, reject) => {
+    const proc = Bun.spawn(cmd, { stdout: 'pipe', stderr: 'pipe' });
+    const timer = setTimeout(() => {
+      proc.kill();
+      reject(new Error(`Process timed out after ${timeoutMs}ms: ${cmd[0]}`));
+    }, timeoutMs);
+    proc.exited.then(async (exitCode) => {
+      clearTimeout(timer);
+      const stdout = await new Response(proc.stdout).text();
+      const stderr = await new Response(proc.stderr).text();
+      resolve({ exitCode, stdout, stderr });
+    });
+  });
+}
+
+export async function extractKeyframesForAsset(
+  assetId: string,
+  intervalSeconds?: number,
+  onProgress?: (msg: string) => void,
+): Promise<void> {
+  const interval = intervalSeconds ?? 3;
+
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    throw new Error(`Media asset not found: ${assetId}`);
+  }
+
+  if (asset.mediaType !== 'video') {
+    throw new Error(`Keyframe extraction requires a video asset. Got: ${asset.mediaType}`);
+  }
+
+  // Find or create the keyframe_extraction processing stage
+  let stage: ProcessingStage | undefined;
+  const existingStages = getProcessingStagesForAsset(assetId);
+  stage = existingStages.find((s) => s.stage === 'keyframe_extraction');
+  if (!stage) {
+    stage = createProcessingStage({ assetId, stage: 'keyframe_extraction' });
+  }
+
+  updateProcessingStage(stage.id, { status: 'running', startedAt: Date.now() });
+
+  // Store keyframes in a durable directory alongside the source file.
+  // Extract to a temp dir first so that if ffmpeg fails the old frames remain intact.
+  const outputDir = join(dirname(asset.filePath), 'keyframes', assetId);
+  const tempDir = outputDir + '-tmp-' + randomUUID();
+  await mkdir(tempDir, { recursive: true });
+
+  try {
+    onProgress?.(`Extracting keyframes every ${interval}s from ${asset.title}...\n`);
+
+    // Use ffmpeg to extract frames at the specified interval
+    const result = await spawnWithTimeout([
+      'ffmpeg', '-y',
+      '-i', asset.filePath,
+      '-vf', `fps=1/${interval}`,
+      '-q:v', '2',
+      join(tempDir, 'frame-%06d.jpg'),
+    ], FFMPEG_TIMEOUT_MS);
+
+    if (result.exitCode !== 0) {
+      await rm(tempDir, { recursive: true, force: true });
+      updateProcessingStage(stage.id, {
+        status: 'failed',
+        lastError: result.stderr.slice(0, 500),
+      });
+      throw new Error(`ffmpeg failed: ${result.stderr.slice(0, 500)}`);
+    }
+
+    // List extracted frames
+    const files = await readdir(tempDir);
+    const frameFiles = files
+      .filter((f) => f.startsWith('frame-') && f.endsWith('.jpg'))
+      .sort();
+
+    if (frameFiles.length === 0) {
+      await rm(tempDir, { recursive: true, force: true });
+      updateProcessingStage(stage.id, {
+        status: 'failed',
+        lastError: 'No frames extracted',
+      });
+      throw new Error('No frames were extracted from the video.');
+    }
+
+    // Extraction succeeded — atomically swap temp dir into the durable path
+    await rm(outputDir, { recursive: true, force: true });
+    await rename(tempDir, outputDir);
+
+    onProgress?.(`Extracted ${frameFiles.length} frames. Registering in database...\n`);
+
+    // Build keyframe rows
+    const keyframeRows = frameFiles.map((file, index) => ({
+      assetId,
+      timestamp: index * interval,
+      filePath: join(outputDir, file),
+      metadata: { frameIndex: index, intervalSeconds: interval },
+    }));
+
+    // Clear existing keyframes to prevent duplicates on re-extraction
+    deleteKeyframesForAsset(assetId);
+
+    // Batch insert
+    const keyframes = insertKeyframesBatch(keyframeRows);
+
+    // Update progress
+    updateProcessingStage(stage.id, {
+      status: 'completed',
+      progress: 100,
+      completedAt: Date.now(),
+    });
+
+    onProgress?.(`Registered ${keyframes.length} keyframes.\n`);
+  } catch (err) {
+    // Clean up temp dir if it still exists (no-op when already removed above)
+    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+    // Update stage for unexpected errors (ffmpeg/no-frames cases already updated above)
+    const msg = (err as Error).message;
+    if (!msg.startsWith('ffmpeg failed:') && msg !== 'No frames were extracted from the video.') {
+      updateProcessingStage(stage.id, {
+        status: 'failed',
+        lastError: msg.slice(0, 500),
+      });
+    }
+    throw err;
+  }
+}
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  if (!assetId) {
+    return { content: 'asset_id is required.', isError: true };
+  }
+
+  const intervalSeconds = (input.interval_seconds as number) || 3;
+
+  try {
+    await extractKeyframesForAsset(assetId, intervalSeconds, context.onOutput);
+
+    const asset = getMediaAssetById(assetId);
+    const outputDir = join(dirname(asset!.filePath), 'keyframes', assetId);
+    const keyframes = getKeyframesForAsset(assetId);
+
+    return {
+      content: JSON.stringify({
+        message: `Extracted and registered ${keyframes.length} keyframes`,
+        assetId,
+        keyframeCount: keyframes.length,
+        intervalSeconds,
+        outputDir,
+      }, null, 2),
+      isError: false,
+    };
+  } catch (err) {
+    const msg = (err as Error).message;
+    // Preserve original error message format: validation and known failures
+    // are returned directly; unexpected errors get the prefix
+    if (
+      msg.startsWith('Media asset not found:') ||
+      msg.startsWith('Keyframe extraction requires a video asset.') ||
+      msg.startsWith('ffmpeg failed:') ||
+      msg === 'No frames were extracted from the video.'
+    ) {
+      return { content: msg, isError: true };
+    }
+    return { content: `Keyframe extraction failed: ${msg}`, isError: true };
+  }
+}