@vellumai/assistant 0.3.5 → 0.3.7

package/src/config/bundled-skills/media-processing/services/timeline-service.ts

@@ -1,267 +0,0 @@
-/**
- * Timeline generation service.
- *
- * Aggregates sequential vision outputs into coherent timeline segments.
- * Each segment groups adjacent keyframes that share similar scene characteristics
- * into a single time range with merged attributes.
- */
-
-import {
-  getMediaAssetById,
-  getKeyframesForAsset,
-  getVisionOutputsForAsset,
-  deleteTimelineForAsset,
-  insertTimelineSegmentsBatch,
-  createProcessingStage,
-  updateProcessingStage,
-  getProcessingStagesForAsset,
-  type MediaVisionOutput,
-  type MediaKeyframe,
-  type MediaTimeline,
-} from '../../../../memory/media-store.js';
-
-export interface TimelineGenerationResult {
-  assetId: string;
-  segmentCount: number;
-  segments: MediaTimeline[];
-}
-
-/**
- * Generate a timeline for a media asset from its vision analysis outputs.
- *
- * Groups consecutive keyframes with similar scene descriptions into segments.
- * If a timeline already exists for this asset, it is replaced.
- */
-export function generateTimeline(
-  assetId: string,
-  options?: {
-    analysisType?: string;
-    onProgress?: (message: string) => void;
-  },
-): TimelineGenerationResult {
-  const analysisType = options?.analysisType ?? 'scene_description';
-  const onProgress = options?.onProgress;
-
-  const asset = getMediaAssetById(assetId);
-  if (!asset) {
-    throw new Error(`Media asset not found: ${assetId}`);
-  }
-
-  const keyframes = getKeyframesForAsset(assetId);
-  if (keyframes.length === 0) {
-    throw new Error('No keyframes found for this asset. Run extract_keyframes first.');
-  }
-
-  const visionOutputs = getVisionOutputsForAsset(assetId, analysisType);
-  if (visionOutputs.length === 0) {
-    throw new Error(`No vision outputs found for analysis type "${analysisType}". Run analyze_keyframes first.`);
-  }
-
-  // Find or create the timeline_generation processing stage
-  const existingStages = getProcessingStagesForAsset(assetId);
-  let stage = existingStages.find((s) => s.stage === 'timeline_generation');
-  if (!stage) {
-    stage = createProcessingStage({ assetId, stage: 'timeline_generation' });
-  }
-  updateProcessingStage(stage.id, { status: 'running', startedAt: Date.now() });
-
-  try {
-    // Build a map of keyframeId -> keyframe for timestamp lookup
-    const keyframeMap = new Map<string, MediaKeyframe>();
-    for (const kf of keyframes) {
-      keyframeMap.set(kf.id, kf);
-    }
-
-    // Build a map of keyframeId -> vision output
-    const outputByKeyframe = new Map<string, MediaVisionOutput>();
-    for (const vo of visionOutputs) {
-      outputByKeyframe.set(vo.keyframeId, vo);
-    }
-
-    // Sort keyframes by timestamp to ensure sequential processing
-    const sortedKeyframes = [...keyframes]
-      .filter((kf) => outputByKeyframe.has(kf.id))
-      .sort((a, b) => a.timestamp - b.timestamp);
-
-    if (sortedKeyframes.length === 0) {
-      updateProcessingStage(stage.id, {
-        status: 'completed',
-        progress: 100,
-        completedAt: Date.now(),
-      });
-      return { assetId, segmentCount: 0, segments: [] };
-    }
-
-    onProgress?.('Aggregating vision outputs into timeline segments...');
-
-    // Aggregate consecutive frames into segments based on scene similarity
-    const segmentRows: Array<{
-      assetId: string;
-      startTime: number;
-      endTime: number;
-      segmentType: string;
-      attributes: Record<string, unknown>;
-      confidence: number;
-    }> = [];
-
-    let currentSegment = createSegmentFromOutput(
-      assetId,
-      sortedKeyframes[0],
-      outputByKeyframe.get(sortedKeyframes[0].id)!,
-    );
-
-    for (let i = 1; i < sortedKeyframes.length; i++) {
-      const kf = sortedKeyframes[i];
-      const vo = outputByKeyframe.get(kf.id)!;
-
-      if (shouldMergeIntoSegment(currentSegment, vo)) {
-        // Extend the current segment
-        currentSegment.endTime = kf.timestamp;
-        const newConfidence = vo.confidence ?? 0.5;
-        currentSegment.confidence =
-          (currentSegment.confidence * currentSegment.frameCount + newConfidence) / (currentSegment.frameCount + 1);
-        currentSegment.frameCount++;
-        mergeSubjects(currentSegment.attributes, vo.output);
-        mergeActions(currentSegment.attributes, vo.output);
-      } else {
-        // Finalize current segment and start a new one
-        segmentRows.push(currentSegment);
-        currentSegment = createSegmentFromOutput(assetId, kf, vo);
-      }
-
-      // Update progress
-      const progress = Math.round((i / sortedKeyframes.length) * 100);
-      updateProcessingStage(stage.id, { progress });
-    }
-
-    // Don't forget the last segment
-    segmentRows.push(currentSegment);
-
-    // Clear existing timeline and insert new segments
-    deleteTimelineForAsset(assetId);
-    const segments = insertTimelineSegmentsBatch(segmentRows);
-
-    updateProcessingStage(stage.id, {
-      status: 'completed',
-      progress: 100,
-      completedAt: Date.now(),
-    });
-
-    onProgress?.(`Generated ${segments.length} timeline segments.`);
-
-    return { assetId, segmentCount: segments.length, segments };
-  } catch (err) {
-    updateProcessingStage(stage.id, {
-      status: 'failed',
-      lastError: (err as Error).message.slice(0, 500),
-    });
-    throw err;
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Internal helpers
-// ---------------------------------------------------------------------------
-
-interface PendingSegment {
-  assetId: string;
-  startTime: number;
-  endTime: number;
-  segmentType: string;
-  attributes: Record<string, unknown>;
-  confidence: number;
-  frameCount: number;
-}
-
-function createSegmentFromOutput(
-  assetId: string,
-  keyframe: MediaKeyframe,
-  vo: MediaVisionOutput,
-): PendingSegment {
-  const sceneDescription = (vo.output.sceneDescription as string) ?? '';
-  const segmentType = deriveSegmentType(vo.output);
-  return {
-    assetId,
-    startTime: keyframe.timestamp,
-    endTime: keyframe.timestamp,
-    segmentType,
-    attributes: {
-      sceneDescription,
-      subjects: Array.isArray(vo.output.subjects) ? [...(vo.output.subjects as string[])] : [],
-      actions: Array.isArray(vo.output.actions) ? [...(vo.output.actions as string[])] : [],
-      context: (vo.output.context as string) ?? '',
-    },
-    confidence: vo.confidence ?? 0.5,
-    frameCount: 1,
-  };
-}
-
-/**
- * Derive a generic segment type from vision output.
- * Uses simple heuristics on the scene description — domain-specific
- * interpretation belongs in the VLM prompt, not here.
- */
-function deriveSegmentType(output: Record<string, unknown>): string {
-  const actions = output.actions as string[] | undefined;
-  if (actions && actions.length > 0) {
-    return 'activity';
-  }
-  const subjects = output.subjects as string[] | undefined;
-  if (subjects && subjects.length > 0) {
-    return 'scene';
-  }
-  return 'static';
-}
-
-/**
- * Decide whether a new vision output is similar enough to merge
- * into the current segment.
- *
- * Uses a simple heuristic: same segment type and overlapping subjects.
- */
-function shouldMergeIntoSegment(
-  segment: PendingSegment,
-  vo: MediaVisionOutput,
-): boolean {
-  const newType = deriveSegmentType(vo.output);
-  if (newType !== segment.segmentType) return false;
-
-  // Check subject overlap
-  const existingSubjects = new Set(
-    (segment.attributes.subjects as string[]) ?? [],
-  );
-  const newSubjects = (vo.output.subjects as string[]) ?? [];
-
-  if (existingSubjects.size === 0 && newSubjects.length === 0) return true;
-  if (existingSubjects.size === 0 || newSubjects.length === 0) return false;
-
-  const overlap = newSubjects.filter((s) => existingSubjects.has(s)).length;
-  const unionSize = new Set([...existingSubjects, ...newSubjects]).size;
-
-  // Merge if at least 30% overlap (Jaccard similarity)
-  return unionSize > 0 && overlap / unionSize >= 0.3;
-}
-
-function mergeSubjects(
-  attributes: Record<string, unknown>,
-  newOutput: Record<string, unknown>,
-): void {
-  const existing = new Set((attributes.subjects as string[]) ?? []);
-  const incoming = (newOutput.subjects as string[]) ?? [];
-  for (const s of incoming) {
-    existing.add(s);
-  }
-  attributes.subjects = [...existing];
-}
-
-function mergeActions(
-  attributes: Record<string, unknown>,
-  newOutput: Record<string, unknown>,
-): void {
-  const existing = new Set((attributes.actions as string[]) ?? []);
-  const incoming = (newOutput.actions as string[]) ?? [];
-  for (const a of incoming) {
-    existing.add(a);
-  }
-  attributes.actions = [...existing];
-}

package/src/config/bundled-skills/media-processing/tools/detect-events.ts

@@ -1,110 +0,0 @@
-import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
-import { detectEvents, type DetectionConfig, type DetectionRule } from '../services/event-detection-service.js';
-
-/**
- * Sensible default detection rules for common event types.
- * These are fallbacks when the caller doesn't provide explicit rules —
- * the system is not limited to these event types.
- */
-const DEFAULT_RULES_BY_EVENT_TYPE: Record<string, DetectionRule[]> = {
-  turnover: [
-    { ruleType: 'segment_transition', params: { field: 'subjects' }, weight: 0.5 },
-    { ruleType: 'short_segment', params: { maxDurationSeconds: 5 }, weight: 0.3 },
-    { ruleType: 'attribute_match', params: { field: 'actions', pattern: 'steal|turnover|loss|intercept' }, weight: 0.2 },
-  ],
-  scene_change: [
-    { ruleType: 'segment_transition', params: { field: 'segmentType' }, weight: 1.0 },
-  ],
-  short_play: [
-    { ruleType: 'short_segment', params: { maxDurationSeconds: 3 }, weight: 0.6 },
-    { ruleType: 'segment_transition', params: { field: 'subjects' }, weight: 0.4 },
-  ],
-};
-
-export async function run(
-  input: Record<string, unknown>,
-  context: ToolContext,
-): Promise<ToolExecutionResult> {
-  const assetId = input.asset_id as string | undefined;
-  if (!assetId) {
-    return { content: 'asset_id is required.', isError: true };
-  }
-
-  const eventType = input.event_type as string | undefined;
-  if (!eventType) {
-    return { content: 'event_type is required.', isError: true };
-  }
-
-  // Parse detection rules: use provided rules or fall back to defaults
-  let rules: DetectionRule[];
-  const rawRules = input.detection_rules;
-
-  if (rawRules) {
-    // Accept rules as a JSON string or as an already-parsed array
-    if (typeof rawRules === 'string') {
-      try {
-        const parsed = JSON.parse(rawRules);
-        if (!Array.isArray(parsed)) {
-          return { content: 'detection_rules must be a valid JSON array of rule objects.', isError: true };
-        }
-        rules = parsed as DetectionRule[];
-      } catch {
-        return { content: 'detection_rules must be a valid JSON array of rule objects.', isError: true };
-      }
-    } else if (Array.isArray(rawRules)) {
-      rules = rawRules as DetectionRule[];
-    } else {
-      return { content: 'detection_rules must be an array of rule objects.', isError: true };
-    }
-
-    // Validate each rule has the required shape
-    for (const rule of rules) {
-      if (!rule.ruleType || typeof rule.ruleType !== 'string') {
-        return { content: 'Each detection rule must have a "ruleType" string.', isError: true };
-      }
-      if (rule.weight === undefined || typeof rule.weight !== 'number') {
-        return { content: 'Each detection rule must have a "weight" number.', isError: true };
-      }
-      if (!rule.params || typeof rule.params !== 'object') {
-        return { content: 'Each detection rule must have a "params" object.', isError: true };
-      }
-    }
-  } else {
-    // Use defaults for known event types, or a generic transition-based fallback
-    rules = DEFAULT_RULES_BY_EVENT_TYPE[eventType] ?? [
-      { ruleType: 'segment_transition', params: { field: 'segmentType' }, weight: 0.6 },
-      { ruleType: 'short_segment', params: { maxDurationSeconds: 5 }, weight: 0.4 },
-    ];
-  }
-
-  const config: DetectionConfig = { eventType, rules };
-
-  try {
-    const result = detectEvents(assetId, config, {
-      onProgress: (msg) => context.onOutput?.(`${msg}\n`),
-    });
-
-    return {
-      content: JSON.stringify({
-        message: `Detected ${result.candidateCount} ${result.eventType} events`,
-        assetId: result.assetId,
-        eventType: result.eventType,
-        totalEvents: result.candidateCount,
-        rulesUsed: rules.map((r) => r.ruleType),
-        events: result.events.map((e) => ({
-          id: e.id,
-          startTime: e.startTime,
-          endTime: e.endTime,
-          confidence: e.confidence,
-          reasons: e.reasons,
-        })),
-      }, null, 2),
-      isError: false,
-    };
-  } catch (err) {
-    return {
-      content: `Event detection failed: ${(err as Error).message}`,
-      isError: true,
-    };
-  }
-}

package/src/config/bundled-skills/media-processing/tools/recalibrate.ts

@@ -1,235 +0,0 @@
-/**
- * Recalibration tool for media event detection.
- *
- * Reads all feedback for an asset, analyzes correction patterns, and
- * re-ranks existing events using updated heuristics. Updates confidence
- * scores in the media_events table.
- *
- * All interfaces are generic — works for any event type.
- */
-
-import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
-import { getFeedbackForAsset, type EventFeedback } from '../services/feedback-store.js';
-import { aggregateFeedback } from '../services/feedback-aggregation.js';
-import { getEventsForAsset, getMediaAssetById } from '../../../../memory/media-store.js';
-import { getDb } from '../../../../memory/db.js';
-import { mediaEvents } from '../../../../memory/schema.js';
-import { eq } from 'drizzle-orm';
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
-
-interface RecalibrationAdjustment {
-  eventType: string;
-  action: string;
-  detail: string;
-}
-
-interface EventUpdate {
-  eventId: string;
-  eventType: string;
-  oldConfidence: number;
-  newConfidence: number;
-}
-
-// ---------------------------------------------------------------------------
-// Tool entry point
-// ---------------------------------------------------------------------------
-
-export async function run(
-  input: Record<string, unknown>,
-  context: ToolContext,
-): Promise<ToolExecutionResult> {
-  const assetId = input.asset_id as string | undefined;
-  if (!assetId) {
-    return { content: 'asset_id is required.', isError: true };
-  }
-
-  const asset = getMediaAssetById(assetId);
-  if (!asset) {
-    return { content: `Asset "${assetId}" not found.`, isError: true };
-  }
-
-  const allFeedback = getFeedbackForAsset(assetId);
-  if (allFeedback.length === 0) {
-    return {
-      content: JSON.stringify({
-        message: 'No feedback found for this asset. Submit feedback first, then recalibrate.',
-        assetId,
-      }, null, 2),
-      isError: false,
-    };
-  }
-
-  const aggregation = aggregateFeedback(assetId);
-  const allEvents = getEventsForAsset(assetId);
-
-  // Build a map of event ID to its feedback entries
-  const feedbackByEventId = new Map<string, EventFeedback[]>();
-  for (const fb of allFeedback) {
-    if (!feedbackByEventId.has(fb.eventId)) {
-      feedbackByEventId.set(fb.eventId, []);
-    }
-    feedbackByEventId.get(fb.eventId)!.push(fb);
-  }
-
-  // Build a map of event ID to event type for filtering feedback by type
-  const eventTypeById = new Map<string, string>();
-  for (const ev of allEvents) {
-    eventTypeById.set(ev.id, ev.eventType);
-  }
-
-  const adjustments: RecalibrationAdjustment[] = [];
-  const eventUpdates: EventUpdate[] = [];
-  const db = getDb();
-
-  // Analyze patterns per event type
-  for (const stats of aggregation.statsByEventType) {
-    const { eventType, correct, incorrect, boundaryEdit, missed } = stats;
-    const totalReviewed = correct + incorrect + boundaryEdit + missed;
-    if (totalReviewed === 0) continue;
-
-    // Pattern 1: High false positive rate — penalize low-confidence events
-    const falsePositiveRate = totalReviewed > 0 ? incorrect / totalReviewed : 0;
-    if (falsePositiveRate > 0.3 && incorrect >= 2) {
-      adjustments.push({
-        eventType,
-        action: 'penalize_low_confidence',
-        detail: `False positive rate ${(falsePositiveRate * 100).toFixed(1)}% (${incorrect}/${totalReviewed}) — reducing confidence on unreviewed events of this type`,
-      });
-    }
-
-    // Pattern 2: Many missed events — note for threshold adjustment
-    if (missed >= 2) {
-      adjustments.push({
-        eventType,
-        action: 'note_missed_events',
-        detail: `${missed} missed events reported — consider lowering detection threshold or adding detection rules for this type`,
-      });
-    }
-
-    // Pattern 3: Boundary edits — compute average adjustment
-    if (boundaryEdit >= 1) {
-      const boundaryFeedback = allFeedback.filter(
-        (fb) => fb.feedbackType === 'boundary_edit' && eventTypeById.get(fb.eventId) === eventType,
-      );
-      let startAdjTotal = 0;
-      let endAdjTotal = 0;
-      let startAdjCount = 0;
-      let endAdjCount = 0;
-
-      for (const fb of boundaryFeedback) {
-        if (fb.originalStartTime !== null && fb.correctedStartTime !== null) {
-          startAdjTotal += fb.correctedStartTime - fb.originalStartTime;
-          startAdjCount++;
-        }
-        if (fb.originalEndTime !== null && fb.correctedEndTime !== null) {
-          endAdjTotal += fb.correctedEndTime - fb.originalEndTime;
-          endAdjCount++;
-        }
-      }
-
-      const avgStartAdj = startAdjCount > 0 ? startAdjTotal / startAdjCount : 0;
-      const avgEndAdj = endAdjCount > 0 ? endAdjTotal / endAdjCount : 0;
-
-      if (startAdjCount > 0 || endAdjCount > 0) {
-        adjustments.push({
-          eventType,
-          action: 'boundary_correction_pattern',
-          detail: `Average boundary adjustment: start ${avgStartAdj >= 0 ? '+' : ''}${avgStartAdj.toFixed(2)}s (n=${startAdjCount}), end ${avgEndAdj >= 0 ? '+' : ''}${avgEndAdj.toFixed(2)}s (n=${endAdjCount})`,
-        });
-      }
-    }
-  }
-
-  // Re-rank events: adjust confidence based on feedback
-  for (const event of allEvents) {
-    const eventFeedback = feedbackByEventId.get(event.id);
-    if (!eventFeedback || eventFeedback.length === 0) {
-      // For events without direct feedback, apply type-level adjustments
-      const stats = aggregation.statsByEventType.find((s) => s.eventType === event.eventType);
-      if (stats) {
-        const totalReviewed = stats.correct + stats.incorrect + stats.boundaryEdit + stats.missed;
-        const falsePositiveRate = totalReviewed > 0 ? stats.incorrect / totalReviewed : 0;
-
-        // If high false positive rate for this type, reduce unreviewed event confidence
-        if (falsePositiveRate > 0.3 && totalReviewed >= 3) {
-          const penalty = Math.min(falsePositiveRate * 0.3, 0.2);
-          const newConfidence = Math.max(0.05, event.confidence - penalty);
-          if (newConfidence !== event.confidence) {
-            db.update(mediaEvents)
-              .set({ confidence: newConfidence })
-              .where(eq(mediaEvents.id, event.id))
-              .run();
-            eventUpdates.push({
-              eventId: event.id,
-              eventType: event.eventType,
-              oldConfidence: event.confidence,
-              newConfidence,
-            });
-          }
-        }
-      }
-      continue;
-    }
-
-    // Direct feedback: adjust confidence based on the latest feedback
-    const latestFeedback = eventFeedback.sort((a, b) => b.createdAt - a.createdAt)[0];
-    let newConfidence = event.confidence;
-
-    switch (latestFeedback.feedbackType) {
-      case 'correct':
-        // Boost confidence toward 1.0
-        newConfidence = Math.min(1.0, event.confidence + (1.0 - event.confidence) * 0.3);
-        break;
-      case 'incorrect':
-        // Sharply reduce confidence
-        newConfidence = Math.max(0.05, event.confidence * 0.3);
-        break;
-      case 'boundary_edit':
-        // Slight confidence boost (event was real but boundaries were off)
-        newConfidence = Math.min(1.0, event.confidence + (1.0 - event.confidence) * 0.15);
-        break;
-      case 'missed':
-        // User-reported events keep their initial confidence
-        break;
-    }
-
-    newConfidence = Math.round(newConfidence * 1000) / 1000;
-
-    if (newConfidence !== event.confidence) {
-      db.update(mediaEvents)
-        .set({ confidence: newConfidence })
-        .where(eq(mediaEvents.id, event.id))
-        .run();
-      eventUpdates.push({
-        eventId: event.id,
-        eventType: event.eventType,
-        oldConfidence: event.confidence,
-        newConfidence,
-      });
-    }
-  }
-
-  context.onOutput?.(`Recalibrated ${eventUpdates.length} events based on ${allFeedback.length} feedback entries.\n`);
-
-  return {
-    content: JSON.stringify({
-      message: `Recalibration complete for asset ${assetId}`,
-      assetId,
-      totalFeedbackEntries: allFeedback.length,
-      adjustments,
-      eventsUpdated: eventUpdates.length,
-      eventUpdates: eventUpdates.map((u) => ({
-        eventId: u.eventId,
-        eventType: u.eventType,
-        oldConfidence: u.oldConfidence,
-        newConfidence: u.newConfidence,
-        delta: Math.round((u.newConfidence - u.oldConfidence) * 1000) / 1000,
-      })),
-      aggregation: aggregation.statsByEventType,
-    }, null, 2),
-    isError: false,
-  };
-}