@vellumai/assistant 0.3.3 → 0.3.5

package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts

@@ -0,0 +1,261 @@
+/**
+ * Processing pipeline service.
+ *
+ * Orchestrates the full media processing pipeline with reliability features:
+ * - Sequential stage execution: keyframe_extraction -> vision_analysis -> timeline_generation -> event_detection
+ * - Stage-level retries with exponential backoff
+ * - Resumability: checks processing_stages to find last completed stage
+ * - Cancellation support: cooperative cancellation via asset status = 'cancelled'
+ * - Idempotency: respects content-hash dedup from media-store
+ * - Graceful degradation: saves partial results on failure
+ *
+ * All reliability infrastructure is generic media-processing, not domain-specific.
+ */
+
+import {
+  getMediaAssetById,
+  getProcessingStagesForAsset,
+  createProcessingStage,
+  updateProcessingStage,
+  updateMediaAssetStatus,
+  type ProcessingStage,
+} from '../../../../memory/media-store.js';
+import { computeRetryDelay, sleep } from '../../../../util/retry.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type PipelineStageName =
+  | 'keyframe_extraction'
+  | 'vision_analysis'
+  | 'timeline_generation'
+  | 'event_detection';
+
+export interface StageHandler {
+  /** Execute the stage. Throw on failure. */
+  execute: (assetId: string, onProgress?: (msg: string) => void) => Promise<void>;
+}
+
+export interface PipelineOptions {
+  /** Maximum retry attempts per stage (default: 3). */
+  maxRetries?: number;
+  /** Base delay in ms for exponential backoff between retries (default: 1000). */
+  baseDelayMs?: number;
+  /** Progress callback for streaming status updates. */
+  onProgress?: (message: string) => void;
+}
+
+export interface PipelineResult {
+  assetId: string;
+  completedStages: PipelineStageName[];
+  failedStage: PipelineStageName | null;
+  failureReason: string | null;
+  cancelled: boolean;
+  resumedFrom: PipelineStageName | null;
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline stage ordering
+// ---------------------------------------------------------------------------
+
+const STAGE_ORDER: PipelineStageName[] = [
+  'keyframe_extraction',
+  'vision_analysis',
+  'timeline_generation',
+  'event_detection',
+];
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function findOrCreateStage(assetId: string, stageName: string): ProcessingStage {
+  const stages = getProcessingStagesForAsset(assetId);
+  const existing = stages.find((s) => s.stage === stageName);
+  if (existing) return existing;
+  return createProcessingStage({ assetId, stage: stageName });
+}
+
+function isStageCompleted(stage: ProcessingStage): boolean {
+  return stage.status === 'completed';
+}
+
+/**
+ * Check if the asset has been cancelled. Cooperative cancellation:
+ * the pipeline checks this between stages to allow graceful stopping.
+ */
+function isAssetCancelled(assetId: string): boolean {
+  const asset = getMediaAssetById(assetId);
+  if (!asset) return true;
+  return (asset.status as string) === 'cancelled';
+}
+
+// ---------------------------------------------------------------------------
+// Main pipeline
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the full processing pipeline for a media asset.
+ *
+ * The pipeline is resumable: if previous stages are already completed,
+ * execution resumes from the first incomplete stage. Each stage is
+ * retried with exponential backoff on failure. If a stage exhausts
+ * its retries, partial results are preserved and the pipeline stops.
+ */
+export async function runPipeline(
+  assetId: string,
+  handlers: Record<PipelineStageName, StageHandler>,
+  options?: PipelineOptions,
+): Promise<PipelineResult> {
+  const maxRetries = options?.maxRetries ?? 3;
+  const baseDelayMs = options?.baseDelayMs ?? 1000;
+  const onProgress = options?.onProgress;
+
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    throw new Error(`Media asset not found: ${assetId}`);
+  }
+
+  // Check if asset is already cancelled before forcing processing
+  if ((asset.status as string) === 'cancelled') {
+    return {
+      assetId,
+      completedStages: [],
+      failedStage: null,
+      failureReason: null,
+      cancelled: true,
+      resumedFrom: null,
+    };
+  }
+
+  // Mark asset as processing
+  updateMediaAssetStatus(assetId, 'processing');
+
+  const completedStages: PipelineStageName[] = [];
+  let failedStage: PipelineStageName | null = null;
+  let failureReason: string | null = null;
+  let cancelled = false;
+  let resumedFrom: PipelineStageName | null = null;
+
+  // Find where to resume from by checking existing stage records
+  let startIndex = 0;
+  for (let i = 0; i < STAGE_ORDER.length; i++) {
+    const stage = findOrCreateStage(assetId, STAGE_ORDER[i]);
+    if (isStageCompleted(stage)) {
+      completedStages.push(STAGE_ORDER[i]);
+      startIndex = i + 1;
+    } else {
+      break;
+    }
+  }
+
+  if (startIndex > 0 && startIndex < STAGE_ORDER.length) {
+    resumedFrom = STAGE_ORDER[startIndex];
+    onProgress?.(`Resuming pipeline from stage: ${resumedFrom}`);
+  } else if (startIndex >= STAGE_ORDER.length) {
+    // All stages already completed — idempotent no-op
+    onProgress?.('All pipeline stages already completed.');
+    updateMediaAssetStatus(assetId, 'indexed');
+    return {
+      assetId,
+      completedStages,
+      failedStage: null,
+      failureReason: null,
+      cancelled: false,
+      resumedFrom: null,
+    };
+  }
+
+  // Execute stages sequentially from the resume point
+  for (let i = startIndex; i < STAGE_ORDER.length; i++) {
+    const stageName = STAGE_ORDER[i];
+
+    // Cooperative cancellation check between stages
+    if (isAssetCancelled(assetId)) {
+      onProgress?.(`Pipeline cancelled before stage: ${stageName}`);
+      cancelled = true;
+      break;
+    }
+
+    const stageRecord = findOrCreateStage(assetId, stageName);
+    const handler = handlers[stageName];
+
+    onProgress?.(`Starting stage: ${stageName}`);
+    updateProcessingStage(stageRecord.id, {
+      status: 'running',
+      startedAt: Date.now(),
+      lastError: null,
+    });
+
+    let succeeded = false;
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        if (attempt > 0) {
+          const delay = computeRetryDelay(attempt - 1, baseDelayMs);
+          onProgress?.(`Retrying stage ${stageName} (attempt ${attempt + 1}/${maxRetries + 1}) after ${Math.round(delay)}ms...`);
+          await sleep(delay);
+
+          // Re-check cancellation before retry
+          if (isAssetCancelled(assetId)) {
+            onProgress?.(`Pipeline cancelled during retry of stage: ${stageName}`);
+            cancelled = true;
+            break;
+          }
+        }
+
+        await handler.execute(assetId, onProgress);
+
+        // Mark stage as completed
+        updateProcessingStage(stageRecord.id, {
+          status: 'completed',
+          progress: 100,
+          completedAt: Date.now(),
+        });
+
+        completedStages.push(stageName);
+        succeeded = true;
+        onProgress?.(`Completed stage: ${stageName}`);
+        break;
+      } catch (err) {
+        const errorMsg = (err as Error).message.slice(0, 500);
+        onProgress?.(`Stage ${stageName} failed (attempt ${attempt + 1}/${maxRetries + 1}): ${errorMsg}`);
+
+        // Save partial progress — the stage handler should have already
+        // persisted any partial results before throwing
+        updateProcessingStage(stageRecord.id, {
+          status: attempt >= maxRetries ? 'failed' : 'running',
+          lastError: errorMsg,
+        });
+      }
+    }
+
+    if (cancelled) break;
+
+    if (!succeeded) {
+      failedStage = stageName;
+      failureReason = `Stage ${stageName} failed after ${maxRetries + 1} attempts`;
+      onProgress?.(`Pipeline stopped: ${failureReason}`);
+      break;
+    }
+  }
+
+  // Update final asset status
+  if (cancelled) {
+    // Leave status as-is (already 'cancelled')
+  } else if (failedStage) {
+    updateMediaAssetStatus(assetId, 'failed');
+  } else {
+    updateMediaAssetStatus(assetId, 'indexed');
+  }
+
+  return {
+    assetId,
+    completedStages,
+    failedStage,
+    failureReason,
+    cancelled,
+    resumedFrom,
+  };
+}

package/src/config/bundled-skills/media-processing/services/retrieval-service.ts

@@ -0,0 +1,95 @@
+/**
+ * Generic media event retrieval service.
+ *
+ * Pure data retrieval with configurable filters and ranking — no
+ * domain-specific logic. Callers (e.g. a query tool) are responsible
+ * for translating domain concepts into filter parameters.
+ */
+
+import {
+  getEventsForAsset,
+  type MediaEvent,
+} from '../../../../memory/media-store.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface RetrievalFilters {
+  /** Scope results to a specific media asset. */
+  assetId?: string;
+  /** Filter by event type label. */
+  eventType?: string;
+  /** Minimum confidence threshold (0–1). */
+  minConfidence?: number;
+  /** Maximum number of results to return. */
+  limit?: number;
+  /** Sort order for results. */
+  sortBy?: 'confidence' | 'startTime';
+  /** Only return events that start at or after this time (seconds). */
+  startTimeMin?: number;
+  /** Only return events that start at or before this time (seconds). */
+  startTimeMax?: number;
+}
+
+export interface RetrievalResult {
+  events: MediaEvent[];
+  totalReturned: number;
+  filters: RetrievalFilters;
+}
+
+// ---------------------------------------------------------------------------
+// Retrieval
+// ---------------------------------------------------------------------------
+
+/**
+ * Query the media_events table with the given filters and return ranked
+ * results with full event metadata.
+ *
+ * When `assetId` is not provided the function returns an empty result set
+ * because the underlying store requires an asset scope. Callers that want
+ * cross-asset queries should iterate over asset IDs externally.
+ */
+export function retrieveEvents(filters: RetrievalFilters): RetrievalResult {
+  const {
+    assetId,
+    eventType,
+    minConfidence,
+    limit = 10,
+    sortBy = 'confidence',
+    startTimeMin,
+    startTimeMax,
+  } = filters;
+
+  if (!assetId) {
+    return { events: [], totalReturned: 0, filters };
+  }
+
+  // Fetch from the store with the subset of filters it supports natively
+  let events = getEventsForAsset(assetId, {
+    eventType,
+    minConfidence,
+    sortBy,
+    // Fetch more than needed so we can apply time-range filtering locally
+    limit: startTimeMin !== undefined || startTimeMax !== undefined ? undefined : limit,
+  });
+
+  // Apply time-range filters that the store doesn't support directly
+  if (startTimeMin !== undefined) {
+    events = events.filter((e) => e.startTime >= startTimeMin);
+  }
+  if (startTimeMax !== undefined) {
+    events = events.filter((e) => e.startTime <= startTimeMax);
+  }
+
+  // Re-apply limit after local filtering
+  if (limit && events.length > limit) {
+    events = events.slice(0, limit);
+  }
+
+  return {
+    events,
+    totalReturned: events.length,
+    filters,
+  };
+}

package/src/config/bundled-skills/media-processing/services/timeline-service.ts

@@ -0,0 +1,267 @@
+/**
+ * Timeline generation service.
+ *
+ * Aggregates sequential vision outputs into coherent timeline segments.
+ * Each segment groups adjacent keyframes that share similar scene characteristics
+ * into a single time range with merged attributes.
+ */
+
+import {
+  getMediaAssetById,
+  getKeyframesForAsset,
+  getVisionOutputsForAsset,
+  deleteTimelineForAsset,
+  insertTimelineSegmentsBatch,
+  createProcessingStage,
+  updateProcessingStage,
+  getProcessingStagesForAsset,
+  type MediaVisionOutput,
+  type MediaKeyframe,
+  type MediaTimeline,
+} from '../../../../memory/media-store.js';
+
+export interface TimelineGenerationResult {
+  assetId: string;
+  segmentCount: number;
+  segments: MediaTimeline[];
+}
+
+/**
+ * Generate a timeline for a media asset from its vision analysis outputs.
+ *
+ * Groups consecutive keyframes with similar scene descriptions into segments.
+ * If a timeline already exists for this asset, it is replaced.
+ */
+export function generateTimeline(
+  assetId: string,
+  options?: {
+    analysisType?: string;
+    onProgress?: (message: string) => void;
+  },
+): TimelineGenerationResult {
+  const analysisType = options?.analysisType ?? 'scene_description';
+  const onProgress = options?.onProgress;
+
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    throw new Error(`Media asset not found: ${assetId}`);
+  }
+
+  const keyframes = getKeyframesForAsset(assetId);
+  if (keyframes.length === 0) {
+    throw new Error('No keyframes found for this asset. Run extract_keyframes first.');
+  }
+
+  const visionOutputs = getVisionOutputsForAsset(assetId, analysisType);
+  if (visionOutputs.length === 0) {
+    throw new Error(`No vision outputs found for analysis type "${analysisType}". Run analyze_keyframes first.`);
+  }
+
+  // Find or create the timeline_generation processing stage
+  const existingStages = getProcessingStagesForAsset(assetId);
+  let stage = existingStages.find((s) => s.stage === 'timeline_generation');
+  if (!stage) {
+    stage = createProcessingStage({ assetId, stage: 'timeline_generation' });
+  }
+  updateProcessingStage(stage.id, { status: 'running', startedAt: Date.now() });
+
+  try {
+    // Build a map of keyframeId -> keyframe for timestamp lookup
+    const keyframeMap = new Map<string, MediaKeyframe>();
+    for (const kf of keyframes) {
+      keyframeMap.set(kf.id, kf);
+    }
+
+    // Build a map of keyframeId -> vision output
+    const outputByKeyframe = new Map<string, MediaVisionOutput>();
+    for (const vo of visionOutputs) {
+      outputByKeyframe.set(vo.keyframeId, vo);
+    }
+
+    // Sort keyframes by timestamp to ensure sequential processing
+    const sortedKeyframes = [...keyframes]
+      .filter((kf) => outputByKeyframe.has(kf.id))
+      .sort((a, b) => a.timestamp - b.timestamp);
+
+    if (sortedKeyframes.length === 0) {
+      updateProcessingStage(stage.id, {
+        status: 'completed',
+        progress: 100,
+        completedAt: Date.now(),
+      });
+      return { assetId, segmentCount: 0, segments: [] };
+    }
+
+    onProgress?.('Aggregating vision outputs into timeline segments...');
+
+    // Aggregate consecutive frames into segments based on scene similarity
+    const segmentRows: Array<{
+      assetId: string;
+      startTime: number;
+      endTime: number;
+      segmentType: string;
+      attributes: Record<string, unknown>;
+      confidence: number;
+    }> = [];
+
+    let currentSegment = createSegmentFromOutput(
+      assetId,
+      sortedKeyframes[0],
+      outputByKeyframe.get(sortedKeyframes[0].id)!,
+    );
+
+    for (let i = 1; i < sortedKeyframes.length; i++) {
+      const kf = sortedKeyframes[i];
+      const vo = outputByKeyframe.get(kf.id)!;
+
+      if (shouldMergeIntoSegment(currentSegment, vo)) {
+        // Extend the current segment
+        currentSegment.endTime = kf.timestamp;
+        const newConfidence = vo.confidence ?? 0.5;
+        currentSegment.confidence =
+          (currentSegment.confidence * currentSegment.frameCount + newConfidence) / (currentSegment.frameCount + 1);
+        currentSegment.frameCount++;
+        mergeSubjects(currentSegment.attributes, vo.output);
+        mergeActions(currentSegment.attributes, vo.output);
+      } else {
+        // Finalize current segment and start a new one
+        segmentRows.push(currentSegment);
+        currentSegment = createSegmentFromOutput(assetId, kf, vo);
+      }
+
+      // Update progress
+      const progress = Math.round((i / sortedKeyframes.length) * 100);
+      updateProcessingStage(stage.id, { progress });
+    }
+
+    // Don't forget the last segment
+    segmentRows.push(currentSegment);
+
+    // Clear existing timeline and insert new segments
+    deleteTimelineForAsset(assetId);
+    const segments = insertTimelineSegmentsBatch(segmentRows);
+
+    updateProcessingStage(stage.id, {
+      status: 'completed',
+      progress: 100,
+      completedAt: Date.now(),
+    });
+
+    onProgress?.(`Generated ${segments.length} timeline segments.`);
+
+    return { assetId, segmentCount: segments.length, segments };
+  } catch (err) {
+    updateProcessingStage(stage.id, {
+      status: 'failed',
+      lastError: (err as Error).message.slice(0, 500),
+    });
+    throw err;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+interface PendingSegment {
+  assetId: string;
+  startTime: number;
+  endTime: number;
+  segmentType: string;
+  attributes: Record<string, unknown>;
+  confidence: number;
+  frameCount: number;
+}
+
+function createSegmentFromOutput(
+  assetId: string,
+  keyframe: MediaKeyframe,
+  vo: MediaVisionOutput,
+): PendingSegment {
+  const sceneDescription = (vo.output.sceneDescription as string) ?? '';
+  const segmentType = deriveSegmentType(vo.output);
+  return {
+    assetId,
+    startTime: keyframe.timestamp,
+    endTime: keyframe.timestamp,
+    segmentType,
+    attributes: {
+      sceneDescription,
+      subjects: Array.isArray(vo.output.subjects) ? [...(vo.output.subjects as string[])] : [],
+      actions: Array.isArray(vo.output.actions) ? [...(vo.output.actions as string[])] : [],
+      context: (vo.output.context as string) ?? '',
+    },
+    confidence: vo.confidence ?? 0.5,
+    frameCount: 1,
+  };
+}
+
+/**
+ * Derive a generic segment type from vision output.
+ * Uses simple heuristics on the scene description — domain-specific
+ * interpretation belongs in the VLM prompt, not here.
+ */
+function deriveSegmentType(output: Record<string, unknown>): string {
+  const actions = output.actions as string[] | undefined;
+  if (actions && actions.length > 0) {
+    return 'activity';
+  }
+  const subjects = output.subjects as string[] | undefined;
+  if (subjects && subjects.length > 0) {
+    return 'scene';
+  }
+  return 'static';
+}
+
+/**
+ * Decide whether a new vision output is similar enough to merge
+ * into the current segment.
+ *
+ * Uses a simple heuristic: same segment type and overlapping subjects.
+ */
+function shouldMergeIntoSegment(
+  segment: PendingSegment,
+  vo: MediaVisionOutput,
+): boolean {
+  const newType = deriveSegmentType(vo.output);
+  if (newType !== segment.segmentType) return false;
+
+  // Check subject overlap
+  const existingSubjects = new Set(
+    (segment.attributes.subjects as string[]) ?? [],
+  );
+  const newSubjects = (vo.output.subjects as string[]) ?? [];
+
+  if (existingSubjects.size === 0 && newSubjects.length === 0) return true;
+  if (existingSubjects.size === 0 || newSubjects.length === 0) return false;
+
+  const overlap = newSubjects.filter((s) => existingSubjects.has(s)).length;
+  const unionSize = new Set([...existingSubjects, ...newSubjects]).size;
+
+  // Merge if at least 30% overlap (Jaccard similarity)
+  return unionSize > 0 && overlap / unionSize >= 0.3;
+}
+
+function mergeSubjects(
+  attributes: Record<string, unknown>,
+  newOutput: Record<string, unknown>,
+): void {
+  const existing = new Set((attributes.subjects as string[]) ?? []);
+  const incoming = (newOutput.subjects as string[]) ?? [];
+  for (const s of incoming) {
+    existing.add(s);
+  }
+  attributes.subjects = [...existing];
+}
+
+function mergeActions(
+  attributes: Record<string, unknown>,
+  newOutput: Record<string, unknown>,
+): void {
+  const existing = new Set((attributes.actions as string[]) ?? []);
+  const incoming = (newOutput.actions as string[]) ?? [];
+  for (const a of incoming) {
+    existing.add(a);
+  }
+  attributes.actions = [...existing];
+}