@vellumai/assistant 0.3.4 → 0.3.6

package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts

@@ -0,0 +1,259 @@
+/**
+ * Processing pipeline service.
+ *
+ * Orchestrates the full media processing pipeline with reliability features:
+ * - Sequential stage execution: preprocess -> map -> reduce
+ * - Stage-level retries with exponential backoff
+ * - Resumability: checks processing_stages to find last completed stage
+ * - Cancellation support: cooperative cancellation via asset status = 'cancelled'
+ * - Idempotency: respects content-hash dedup from media-store
+ * - Graceful degradation: saves partial results on failure
+ *
+ * All reliability infrastructure is generic media-processing, not domain-specific.
+ */
+
+import {
+  getMediaAssetById,
+  getProcessingStagesForAsset,
+  createProcessingStage,
+  updateProcessingStage,
+  updateMediaAssetStatus,
+  type ProcessingStage,
+} from '../../../../memory/media-store.js';
+import { computeRetryDelay, sleep } from '../../../../util/retry.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type PipelineStageName =
+  | 'preprocess'
+  | 'map'
+  | 'reduce';
+
+export interface StageHandler {
+  /** Execute the stage. Throw on failure. */
+  execute: (assetId: string, onProgress?: (msg: string) => void) => Promise<void>;
+}
+
+export interface PipelineOptions {
+  /** Maximum retry attempts per stage (default: 3). */
+  maxRetries?: number;
+  /** Base delay in ms for exponential backoff between retries (default: 1000). */
+  baseDelayMs?: number;
+  /** Progress callback for streaming status updates. */
+  onProgress?: (message: string) => void;
+}
+
+export interface PipelineResult {
+  assetId: string;
+  completedStages: PipelineStageName[];
+  failedStage: PipelineStageName | null;
+  failureReason: string | null;
+  cancelled: boolean;
+  resumedFrom: PipelineStageName | null;
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline stage ordering
+// ---------------------------------------------------------------------------
+
+const STAGE_ORDER: PipelineStageName[] = [
+  'preprocess',
+  'map',
+  'reduce',
+];
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function findOrCreateStage(assetId: string, stageName: string): ProcessingStage {
+  const stages = getProcessingStagesForAsset(assetId);
+  const existing = stages.find((s) => s.stage === stageName);
+  if (existing) return existing;
+  return createProcessingStage({ assetId, stage: stageName });
+}
+
+function isStageCompleted(stage: ProcessingStage): boolean {
+  return stage.status === 'completed';
+}
+
+/**
+ * Check if the asset has been cancelled. Cooperative cancellation:
+ * the pipeline checks this between stages to allow graceful stopping.
+ */
+function isAssetCancelled(assetId: string): boolean {
+  const asset = getMediaAssetById(assetId);
+  if (!asset) return true;
+  return (asset.status as string) === 'cancelled';
+}
+
+// ---------------------------------------------------------------------------
+// Main pipeline
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the full processing pipeline for a media asset.
+ *
+ * The pipeline is resumable: if previous stages are already completed,
+ * execution resumes from the first incomplete stage. Each stage is
+ * retried with exponential backoff on failure. If a stage exhausts
+ * its retries, partial results are preserved and the pipeline stops.
+ */
+export async function runPipeline(
+  assetId: string,
+  handlers: Record<PipelineStageName, StageHandler>,
+  options?: PipelineOptions,
+): Promise<PipelineResult> {
+  const maxRetries = options?.maxRetries ?? 3;
+  const baseDelayMs = options?.baseDelayMs ?? 1000;
+  const onProgress = options?.onProgress;
+
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    throw new Error(`Media asset not found: ${assetId}`);
+  }
+
+  // Check if asset is already cancelled before forcing processing
+  if ((asset.status as string) === 'cancelled') {
+    return {
+      assetId,
+      completedStages: [],
+      failedStage: null,
+      failureReason: null,
+      cancelled: true,
+      resumedFrom: null,
+    };
+  }
+
+  // Mark asset as processing
+  updateMediaAssetStatus(assetId, 'processing');
+
+  const completedStages: PipelineStageName[] = [];
+  let failedStage: PipelineStageName | null = null;
+  let failureReason: string | null = null;
+  let cancelled = false;
+  let resumedFrom: PipelineStageName | null = null;
+
+  // Find where to resume from by checking existing stage records
+  let startIndex = 0;
+  for (let i = 0; i < STAGE_ORDER.length; i++) {
+    const stage = findOrCreateStage(assetId, STAGE_ORDER[i]);
+    if (isStageCompleted(stage)) {
+      completedStages.push(STAGE_ORDER[i]);
+      startIndex = i + 1;
+    } else {
+      break;
+    }
+  }
+
+  if (startIndex > 0 && startIndex < STAGE_ORDER.length) {
+    resumedFrom = STAGE_ORDER[startIndex];
+    onProgress?.(`Resuming pipeline from stage: ${resumedFrom}`);
+  } else if (startIndex >= STAGE_ORDER.length) {
+    // All stages already completed — idempotent no-op
+    onProgress?.('All pipeline stages already completed.');
+    updateMediaAssetStatus(assetId, 'indexed');
+    return {
+      assetId,
+      completedStages,
+      failedStage: null,
+      failureReason: null,
+      cancelled: false,
+      resumedFrom: null,
+    };
+  }
+
+  // Execute stages sequentially from the resume point
+  for (let i = startIndex; i < STAGE_ORDER.length; i++) {
+    const stageName = STAGE_ORDER[i];
+
+    // Cooperative cancellation check between stages
+    if (isAssetCancelled(assetId)) {
+      onProgress?.(`Pipeline cancelled before stage: ${stageName}`);
+      cancelled = true;
+      break;
+    }
+
+    const stageRecord = findOrCreateStage(assetId, stageName);
+    const handler = handlers[stageName];
+
+    onProgress?.(`Starting stage: ${stageName}`);
+    updateProcessingStage(stageRecord.id, {
+      status: 'running',
+      startedAt: Date.now(),
+      lastError: null,
+    });
+
+    let succeeded = false;
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        if (attempt > 0) {
+          const delay = computeRetryDelay(attempt - 1, baseDelayMs);
+          onProgress?.(`Retrying stage ${stageName} (attempt ${attempt + 1}/${maxRetries + 1}) after ${Math.round(delay)}ms...`);
+          await sleep(delay);
+
+          // Re-check cancellation before retry
+          if (isAssetCancelled(assetId)) {
+            onProgress?.(`Pipeline cancelled during retry of stage: ${stageName}`);
+            cancelled = true;
+            break;
+          }
+        }
+
+        await handler.execute(assetId, onProgress);
+
+        // Mark stage as completed
+        updateProcessingStage(stageRecord.id, {
+          status: 'completed',
+          progress: 100,
+          completedAt: Date.now(),
+        });
+
+        completedStages.push(stageName);
+        succeeded = true;
+        onProgress?.(`Completed stage: ${stageName}`);
+        break;
+      } catch (err) {
+        const errorMsg = (err as Error).message.slice(0, 500);
+        onProgress?.(`Stage ${stageName} failed (attempt ${attempt + 1}/${maxRetries + 1}): ${errorMsg}`);
+
+        // Save partial progress — the stage handler should have already
+        // persisted any partial results before throwing
+        updateProcessingStage(stageRecord.id, {
+          status: attempt >= maxRetries ? 'failed' : 'running',
+          lastError: errorMsg,
+        });
+      }
+    }
+
+    if (cancelled) break;
+
+    if (!succeeded) {
+      failedStage = stageName;
+      failureReason = `Stage ${stageName} failed after ${maxRetries + 1} attempts`;
+      onProgress?.(`Pipeline stopped: ${failureReason}`);
+      break;
+    }
+  }
+
+  // Update final asset status
+  if (cancelled) {
+    // Leave status as-is (already 'cancelled')
+  } else if (failedStage) {
+    updateMediaAssetStatus(assetId, 'failed');
+  } else {
+    updateMediaAssetStatus(assetId, 'indexed');
+  }
+
+  return {
+    assetId,
+    completedStages,
+    failedStage,
+    failureReason,
+    cancelled,
+    resumedFrom,
+  };
+}

package/src/config/bundled-skills/media-processing/services/reduce.ts

@@ -0,0 +1,197 @@
+/**
+ * Reduce service — sends Map output to Claude as text-only for analysis.
+ *
+ * Two modes:
+ * - One-shot merge: assembles all Map results into a single document,
+ *   sends to Claude with the provided system prompt for analysis.
+ * - Interactive Q&A: loads existing map output + user query, sends to
+ *   Claude, returns the answer.
+ *
+ * Uses the existing provider infrastructure (getConfiguredProvider) so
+ * it works with whatever LLM provider is configured.
+ */
+
+import { readFile } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { getMediaAssetById } from '../../../../memory/media-store.js';
+import {
+  getConfiguredProvider,
+  createTimeout,
+  extractAllText,
+  userMessage,
+} from '../../../../providers/provider-send-message.js';
+import type { MapOutput } from './gemini-map.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ReduceOptions {
+  /** Natural language query about the video data. Optional for one-shot merge mode. */
+  query?: string;
+  /** Optional system prompt for Claude. */
+  systemPrompt?: string;
+  /** Model override. When omitted, the configured provider's default is used. */
+  model?: string;
+}
+
+export interface ReduceResult {
+  answer: string;
+  model: string;
+  inputTokens: number;
+  outputTokens: number;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+const REDUCE_TIMEOUT_MS = 120_000;
+
+/**
+ * Load map-output.json for an asset from its pipeline directory.
+ */
+async function loadMapOutput(assetId: string): Promise<MapOutput> {
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    throw new Error(`Media asset not found: ${assetId}`);
+  }
+
+  const pipelineDir = join(dirname(asset.filePath), 'pipeline', assetId);
+  const mapOutputPath = join(pipelineDir, 'map-output.json');
+
+  let raw: string;
+  try {
+    raw = await readFile(mapOutputPath, 'utf-8');
+  } catch {
+    throw new Error(
+      'No map output found. Run analyze_keyframes first to generate map-output.json.',
+    );
+  }
+
+  return JSON.parse(raw) as MapOutput;
+}
+
+/**
+ * Format map output segments into a text document for Claude.
+ * Strips image data — text only.
+ */
+function formatMapOutputAsText(mapOutput: MapOutput): string {
+  const lines: string[] = [];
+
+  lines.push(`Video Analysis Data (asset: ${mapOutput.assetId})`);
+  lines.push(`Model: ${mapOutput.model}`);
+  lines.push(`Segments analyzed: ${mapOutput.successCount}/${mapOutput.segmentCount}`);
+  if (mapOutput.failedCount > 0) {
+    lines.push(`Failed segments: ${mapOutput.failedCount}`);
+  }
+  if (mapOutput.skippedCount > 0) {
+    lines.push(`Skipped segments: ${mapOutput.skippedCount}`);
+  }
+  lines.push('');
+  lines.push('--- Segment Results ---');
+  lines.push('');
+
+  for (const segment of mapOutput.segments) {
+    const startMin = Math.floor(segment.startSeconds / 60);
+    const startSec = Math.floor(segment.startSeconds % 60);
+    const endMin = Math.floor(segment.endSeconds / 60);
+    const endSec = Math.floor(segment.endSeconds % 60);
+    const timeRange = `${startMin}:${String(startSec).padStart(2, '0')} - ${endMin}:${String(endSec).padStart(2, '0')}`;
+
+    lines.push(`[Segment ${segment.segmentId}] ${timeRange}`);
+    lines.push(JSON.stringify(segment.result, null, 2));
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Core: send to Claude via provider infrastructure
+// ---------------------------------------------------------------------------
+
+async function sendToClaude(
+  mapText: string,
+  query: string,
+  systemPrompt?: string,
+  model?: string,
+  onProgress?: (msg: string) => void,
+): Promise<ReduceResult> {
+  const provider = getConfiguredProvider();
+  if (!provider) {
+    throw new Error('No LLM provider available. Please configure an API key.');
+  }
+
+  const effectiveSystemPrompt = systemPrompt
+    ?? 'You are an expert video analyst. You have been given structured analysis data extracted from a video. Answer the user\'s question based on this data. Be specific, reference timestamps when relevant, and provide clear, actionable insights.';
+
+  const userContent = `Here is the video analysis data:\n\n${mapText}\n\n---\n\nUser query: ${query}`;
+
+  onProgress?.('Sending map output to Claude for analysis...\n');
+
+  const { signal, cleanup } = createTimeout(REDUCE_TIMEOUT_MS);
+
+  try {
+    const response = await provider.sendMessage(
+      [userMessage(userContent)],
+      [],
+      effectiveSystemPrompt,
+      {
+        config: model ? { model } : {},
+        signal,
+      },
+    );
+    cleanup();
+
+    const answer = extractAllText(response);
+
+    onProgress?.(`Reduce complete (${response.usage.inputTokens} input + ${response.usage.outputTokens} output tokens).\n`);
+
+    return {
+      answer,
+      model: response.model,
+      inputTokens: response.usage.inputTokens,
+      outputTokens: response.usage.outputTokens,
+    };
+  } finally {
+    cleanup();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * One-shot merge mode: load map output for an asset, send all data to
+ * Claude with a system prompt, and return the analysis.
+ */
+export async function reduceForAsset(
+  assetId: string,
+  options: ReduceOptions,
+  onProgress?: (msg: string) => void,
+): Promise<ReduceResult> {
+  const mapOutput = await loadMapOutput(assetId);
+
+  if (mapOutput.segments.length === 0) {
+    throw new Error('Map output contains no segments. Run analyze_keyframes first.');
+  }
+
+  const mapText = formatMapOutputAsText(mapOutput);
+  const effectiveQuery = options.query ?? 'Summarize the video content.';
+  return sendToClaude(mapText, effectiveQuery, options.systemPrompt, options.model, onProgress);
+}
+
+/**
+ * Interactive Q&A mode: load existing map output + user query, send to
+ * Claude, return the answer. Functionally identical to reduceForAsset
+ * but named distinctly for clarity in call sites.
+ */
+export async function queryMapOutput(
+  assetId: string,
+  options: ReduceOptions,
+  onProgress?: (msg: string) => void,
+): Promise<ReduceResult> {
+  return reduceForAsset(assetId, options, onProgress);
+}

package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts

@@ -0,0 +1,136 @@
+import { join, dirname } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import { getConfig } from '../../../../config/loader.js';
+import {
+  getMediaAssetById,
+} from '../../../../memory/media-store.js';
+import { mapSegments, type MapOutput } from '../services/gemini-map.js';
+import type { PreprocessManifest } from '../services/preprocess.js';
+
+// ---------------------------------------------------------------------------
+// Exported function for job handler use
+// ---------------------------------------------------------------------------
+
+export interface MapSegmentsOptions {
+  systemPrompt: string;
+  outputSchema: Record<string, unknown>;
+  context?: Record<string, unknown>;
+  model?: string;
+  concurrency?: number;
+  maxRetries?: number;
+}
+
+export async function mapSegmentsForAsset(
+  assetId: string,
+  options: MapSegmentsOptions,
+  onProgress?: (msg: string) => void,
+): Promise<MapOutput> {
+  const config = getConfig();
+  const apiKey = config.apiKeys.gemini;
+
+  if (!apiKey) {
+    throw new Error('No Gemini API key configured. Please set your Gemini API key to use keyframe analysis.');
+  }
+
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    throw new Error(`Media asset not found: ${assetId}`);
+  }
+
+  // Load preprocess manifest
+  const pipelineDir = join(dirname(asset.filePath), 'pipeline', assetId);
+  const manifestPath = join(pipelineDir, 'manifest.json');
+
+  let manifest: PreprocessManifest;
+  try {
+    const raw = await readFile(manifestPath, 'utf-8');
+    manifest = JSON.parse(raw) as PreprocessManifest;
+  } catch {
+    throw new Error('No preprocess manifest found. Run extract_keyframes first.');
+  }
+
+  if (manifest.segments.length === 0) {
+    throw new Error('No segments found in preprocess manifest. Run extract_keyframes first.');
+  }
+
+  return mapSegments(assetId, pipelineDir, manifest.segments, {
+    apiKey,
+    systemPrompt: options.systemPrompt,
+    outputSchema: options.outputSchema,
+    context: options.context,
+    model: options.model,
+    concurrency: options.concurrency,
+    maxRetries: options.maxRetries,
+  }, onProgress);
+}
+
+// ---------------------------------------------------------------------------
+// Tool entry point
+// ---------------------------------------------------------------------------
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  if (!assetId) {
+    return { content: 'asset_id is required.', isError: true };
+  }
+
+  const systemPrompt = input.system_prompt as string | undefined;
+  if (!systemPrompt) {
+    return { content: 'system_prompt is required.', isError: true };
+  }
+
+  const outputSchema = input.output_schema as Record<string, unknown> | undefined;
+  if (!outputSchema) {
+    return { content: 'output_schema is required.', isError: true };
+  }
+
+  const contextObj = input.context as Record<string, unknown> | undefined;
+  const model = input.model as string | undefined;
+  const concurrency = input.concurrency as number | undefined;
+  const maxRetries = input.max_retries as number | undefined;
+
+  if (concurrency !== undefined && concurrency < 1) {
+    return { content: 'concurrency must be at least 1.', isError: true };
+  }
+  if (maxRetries !== undefined && maxRetries < 0) {
+    return { content: 'max_retries must be non-negative.', isError: true };
+  }
+
+  try {
+    const output = await mapSegmentsForAsset(
+      assetId,
+      {
+        systemPrompt,
+        outputSchema,
+        context: contextObj,
+        model,
+        concurrency,
+        maxRetries,
+      },
+      context.onOutput,
+    );
+
+    return {
+      content: JSON.stringify({
+        message: `Map ${output.failedCount === 0 ? 'completed' : 'completed with errors'}`,
+        assetId,
+        model: output.model,
+        segmentCount: output.segmentCount,
+        successCount: output.successCount,
+        failedCount: output.failedCount,
+        skippedCount: output.skippedCount,
+        totalInputTokens: output.costSummary.totalInputTokens,
+        totalOutputTokens: output.costSummary.totalOutputTokens,
+        estimatedCostUSD: output.costSummary.totalEstimatedUSD,
+      }, null, 2),
+      isError: false,
+    };
+  } catch (err) {
+    const msg = (err as Error).message;
+    return { content: msg, isError: true };
+  }
+}

package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts

@@ -0,0 +1,59 @@
+import { join, dirname } from 'node:path';
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import { getMediaAssetById, getKeyframesForAsset } from '../../../../memory/media-store.js';
+import { preprocessForAsset, type PreprocessOptions } from '../services/preprocess.js';
+
+export { preprocessForAsset } from '../services/preprocess.js';
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  if (!assetId) {
+    return { content: 'asset_id is required.', isError: true };
+  }
+
+  const options: PreprocessOptions = {
+    intervalSeconds: (input.interval_seconds as number) || undefined,
+    segmentDuration: (input.segment_duration as number) || undefined,
+    deadTimeThreshold: (input.dead_time_threshold as number) || undefined,
+    sectionConfigPath: (input.section_config as string) || undefined,
+    skipDeadTime: input.skip_dead_time !== undefined ? Boolean(input.skip_dead_time) : undefined,
+    shortEdge: (input.short_edge as number) || undefined,
+  };
+
+  try {
+    const manifest = await preprocessForAsset(assetId, options, context.onOutput);
+
+    const asset = getMediaAssetById(assetId);
+    const pipelineDir = join(dirname(asset!.filePath), 'pipeline', assetId);
+    const keyframes = getKeyframesForAsset(assetId);
+
+    return {
+      content: JSON.stringify({
+        message: `Preprocessed video: ${manifest.segments.length} segments, ${keyframes.length} keyframes`,
+        assetId,
+        segmentCount: manifest.segments.length,
+        keyframeCount: keyframes.length,
+        deadTimeRanges: manifest.deadTimeRanges.length,
+        subjectGroups: manifest.subjectRegistry.groups.length,
+        manifestPath: join(pipelineDir, 'manifest.json'),
+        config: manifest.config,
+      }, null, 2),
+      isError: false,
+    };
+  } catch (err) {
+    const msg = (err as Error).message;
+    if (
+      msg.startsWith('Media asset not found:') ||
+      msg.startsWith('Preprocess requires a video asset.') ||
+      msg.startsWith('Video asset has no duration') ||
+      msg.startsWith('ffmpeg failed:') ||
+      msg === 'No frames were extracted from the video.'
+    ) {
+      return { content: msg, isError: true };
+    }
+    return { content: `Preprocess failed: ${msg}`, isError: true };
+  }
+}