@vellumai/assistant 0.3.5 → 0.3.7

package/src/config/bundled-skills/media-processing/services/event-detection-service.ts

@@ -1,280 +0,0 @@
-/**
- * Generic event detection service.
- *
- * Evaluates configurable detection rules against timeline segments and produces
- * scored event candidates. The rule system is fully pluggable — callers supply
- * a DetectionConfig that specifies which rules to evaluate and how to weight them.
- *
- * Example configurations (not hardcoded — these are passed in by the caller):
- *
- *   Basketball turnovers:
- *     eventType: 'turnover'
- *     rules: [
- *       { ruleType: 'segment_transition', params: { field: 'subjects' }, weight: 0.5 },
- *       { ruleType: 'short_segment', params: { maxDurationSeconds: 5 }, weight: 0.3 },
- *       { ruleType: 'attribute_match', params: { field: 'actions', pattern: 'steal|turnover' }, weight: 0.2 },
- *     ]
- *
- *   Scene changes:
- *     eventType: 'scene_change'
- *     rules: [
- *       { ruleType: 'segment_transition', params: { field: 'segmentType' }, weight: 1.0 },
- *     ]
- */
-
-import {
-  getMediaAssetById,
-  getTimelineForAsset,
-  insertEventsBatch,
-  deleteEventsForAssetByType,
-  type MediaTimeline,
-  type MediaEvent,
-} from '../../../../memory/media-store.js';
-
-// ---------------------------------------------------------------------------
-// Public types
-// ---------------------------------------------------------------------------
-
-export interface DetectionRule {
-  ruleType: string;
-  params: Record<string, unknown>;
-  weight: number;
-}
-
-export interface DetectionConfig {
-  eventType: string;
-  rules: DetectionRule[];
-}
-
-export interface EventCandidate {
-  startTime: number;
-  endTime: number;
-  confidence: number;
-  reasons: string[];
-  metadata: Record<string, unknown>;
-}
-
-export interface DetectionResult {
-  assetId: string;
-  eventType: string;
-  candidateCount: number;
-  events: MediaEvent[];
-}
-
-// ---------------------------------------------------------------------------
-// Rule evaluation
-// ---------------------------------------------------------------------------
-
-interface RuleMatch {
-  matched: boolean;
-  reason: string;
-  metadata?: Record<string, unknown>;
-}
-
-type RuleEvaluator = (
-  segment: MediaTimeline,
-  prevSegment: MediaTimeline | null,
-  nextSegment: MediaTimeline | null,
-  params: Record<string, unknown>,
-) => RuleMatch;
-
-const RULE_EVALUATORS: Record<string, RuleEvaluator> = {
-  /**
-   * Fires when a specified field changes between adjacent segments.
-   * params.field: which attribute to compare ('subjects', 'segmentType', etc.)
-   */
-  segment_transition: (segment, prevSegment, _next, params) => {
-    if (!prevSegment) return { matched: false, reason: '' };
-    const field = (params.field as string) ?? 'segmentType';
-
-    if (field === 'segmentType') {
-      const changed = segment.segmentType !== prevSegment.segmentType;
-      return {
-        matched: changed,
-        reason: changed
-          ? `Segment type changed from "${prevSegment.segmentType}" to "${segment.segmentType}"`
-          : '',
-      };
-    }
-
-    // Compare attribute arrays (e.g., subjects)
-    const prevAttrs = prevSegment.attributes ?? {};
-    const currAttrs = segment.attributes ?? {};
-    const prevValues = new Set(Array.isArray(prevAttrs[field]) ? (prevAttrs[field] as string[]) : []);
-    const currValues = Array.isArray(currAttrs[field]) ? (currAttrs[field] as string[]) : [];
-
-    if (prevValues.size === 0 && currValues.length === 0) {
-      return { matched: false, reason: '' };
-    }
-
-    const overlap = currValues.filter((v) => prevValues.has(v)).length;
-    const unionSize = new Set([...prevValues, ...currValues]).size;
-    const similarity = unionSize > 0 ? overlap / unionSize : 0;
-
-    // A transition is detected when similarity drops below 50%
-    const changed = similarity < 0.5;
-    return {
-      matched: changed,
-      reason: changed
-        ? `${field} changed between segments (similarity: ${(similarity * 100).toFixed(0)}%)`
-        : '',
-      metadata: changed ? { similarity, prevValues: [...prevValues], currValues } : undefined,
-    };
-  },
-
-  /**
-   * Fires when a segment's duration is below a threshold.
-   * params.maxDurationSeconds: maximum segment duration to trigger (default: 5)
-   */
-  short_segment: (segment, _prev, _next, params) => {
-    const maxDuration = (params.maxDurationSeconds as number) ?? 5;
-    const duration = segment.endTime - segment.startTime;
-    const matched = duration > 0 && duration <= maxDuration;
-    return {
-      matched,
-      reason: matched
-        ? `Short segment (${duration.toFixed(1)}s <= ${maxDuration}s threshold)`
-        : '',
-      metadata: matched ? { duration } : undefined,
-    };
-  },
-
-  /**
-   * Fires when a segment's attribute values match a regex pattern.
-   * params.field: which attribute array to search (default: 'actions')
-   * params.pattern: regex pattern to match against values
-   */
-  attribute_match: (segment, _prev, _next, params) => {
-    const field = (params.field as string) ?? 'actions';
-    const pattern = params.pattern as string;
-    if (!pattern) return { matched: false, reason: 'No pattern specified' };
-
-    const attrs = segment.attributes ?? {};
-    const values = Array.isArray(attrs[field]) ? (attrs[field] as string[]) : [];
-    const regex = new RegExp(pattern, 'i');
-    const matchedValues = values.filter((v) => regex.test(v));
-
-    return {
-      matched: matchedValues.length > 0,
-      reason: matchedValues.length > 0
-        ? `${field} matched pattern /${pattern}/: [${matchedValues.join(', ')}]`
-        : '',
-      metadata: matchedValues.length > 0 ? { matchedValues } : undefined,
-    };
-  },
-};
-
-// ---------------------------------------------------------------------------
-// Main detection logic
-// ---------------------------------------------------------------------------
-
-/**
- * Detect events in a media asset's timeline using the provided configuration.
- *
- * For each timeline segment, evaluates all rules and computes a weighted
- * confidence score. Segments where at least one rule matches are emitted
- * as event candidates. Results are stored in the media_events table (previous
- * events of the same type for this asset are replaced).
- */
-export function detectEvents(
-  assetId: string,
-  config: DetectionConfig,
-  options?: { onProgress?: (message: string) => void },
-): DetectionResult {
-  const onProgress = options?.onProgress;
-
-  const asset = getMediaAssetById(assetId);
-  if (!asset) {
-    throw new Error(`Media asset not found: ${assetId}`);
-  }
-
-  const segments = getTimelineForAsset(assetId);
-  if (segments.length === 0) {
-    throw new Error('No timeline segments found. Run timeline generation first.');
-  }
-
-  // Sort segments by start time
-  const sorted = [...segments].sort((a, b) => a.startTime - b.startTime);
-
-  onProgress?.(`Evaluating ${config.rules.length} detection rules against ${sorted.length} segments...`);
-
-  // Normalize weights so they sum to 1
-  const totalWeight = config.rules.reduce((sum, r) => sum + r.weight, 0);
-  const normalizedRules = totalWeight > 0
-    ? config.rules.map((r) => ({ ...r, weight: r.weight / totalWeight }))
-    : config.rules;
-
-  const candidates: EventCandidate[] = [];
-
-  for (let i = 0; i < sorted.length; i++) {
-    const segment = sorted[i];
-    const prev = i > 0 ? sorted[i - 1] : null;
-    const next = i < sorted.length - 1 ? sorted[i + 1] : null;
-
-    let weightedScore = 0;
-    const reasons: string[] = [];
-    const candidateMetadata: Record<string, unknown> = {};
-    let anyMatched = false;
-
-    for (const rule of normalizedRules) {
-      const evaluator = RULE_EVALUATORS[rule.ruleType];
-      if (!evaluator) {
-        reasons.push(`Unknown rule type: ${rule.ruleType}`);
-        continue;
-      }
-
-      const result = evaluator(segment, prev, next, rule.params);
-      if (result.matched) {
-        anyMatched = true;
-        weightedScore += rule.weight;
-        reasons.push(result.reason);
-        if (result.metadata) {
-          candidateMetadata[rule.ruleType] = result.metadata;
-        }
-      }
-    }
-
-    if (anyMatched) {
-      candidates.push({
-        startTime: segment.startTime,
-        endTime: segment.endTime,
-        confidence: Math.min(weightedScore, 1),
-        reasons,
-        metadata: {
-          segmentId: segment.id,
-          segmentType: segment.segmentType,
-          ...candidateMetadata,
-        },
-      });
-    }
-  }
-
-  // Sort by confidence descending
-  candidates.sort((a, b) => b.confidence - a.confidence);
-
-  onProgress?.(`Found ${candidates.length} event candidates. Storing results...`);
-
-  // Replace existing events of this type for the asset (scoped to eventType)
-  deleteEventsForAssetByType(assetId, config.eventType);
-
-  const eventRows = candidates.map((c) => ({
-    assetId,
-    eventType: config.eventType,
-    startTime: c.startTime,
-    endTime: c.endTime,
-    confidence: c.confidence,
-    reasons: c.reasons,
-    metadata: c.metadata,
-  }));
-
-  const events = eventRows.length > 0 ? insertEventsBatch(eventRows) : [];
-
-  onProgress?.(`Stored ${events.length} events.`);
-
-  return {
-    assetId,
-    eventType: config.eventType,
-    candidateCount: candidates.length,
-    events,
-  };
-}

package/src/config/bundled-skills/media-processing/services/feedback-aggregation.ts

@@ -1,144 +0,0 @@
-/**
- * Feedback aggregation service for media events.
- *
- * Computes precision/recall estimates per event type based on user feedback,
- * and provides structured JSON export for offline analysis.
- *
- * All interfaces are generic — works for any event type.
- */
-
-import { getFeedbackForAsset, type EventFeedback } from './feedback-store.js';
-import { getEventsForAsset } from '../../../../memory/media-store.js';
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
-
-export interface EventTypeStats {
-  eventType: string;
-  totalEvents: number;
-  correct: number;
-  incorrect: number;
-  boundaryEdit: number;
-  missed: number;
-  totalFeedback: number;
-  precision: number | null;
-  recall: number | null;
-}
-
-export interface AggregationResult {
-  assetId: string;
-  totalFeedbackEntries: number;
-  statsByEventType: EventTypeStats[];
-}
-
-export interface FeedbackExport {
-  assetId: string;
-  exportedAt: string;
-  totalEntries: number;
-  feedback: EventFeedback[];
-}
-
-// ---------------------------------------------------------------------------
-// Aggregation
-// ---------------------------------------------------------------------------
-
-/**
- * Compute precision/recall estimates per event type for a given asset.
- *
- * precision = correct / (correct + incorrect)
- * recall = (correct + boundary_edit) / (correct + boundary_edit + missed)
- *
- * Returns null for precision/recall when the denominator is zero.
- */
-export function aggregateFeedback(assetId: string): AggregationResult {
-  const allFeedback = getFeedbackForAsset(assetId);
-  const allEvents = getEventsForAsset(assetId);
-
-  // Group feedback by event type — we need event type from the events table
-  const eventTypeById = new Map<string, string>();
-  for (const event of allEvents) {
-    eventTypeById.set(event.id, event.eventType);
-  }
-
-  // Collect counts per event type
-  const countsMap = new Map<string, {
-    correct: number;
-    incorrect: number;
-    boundaryEdit: number;
-    missed: number;
-    totalEvents: number;
-  }>();
-
-  // Initialize with known event types from existing events
-  for (const event of allEvents) {
-    if (!countsMap.has(event.eventType)) {
-      countsMap.set(event.eventType, { correct: 0, incorrect: 0, boundaryEdit: 0, missed: 0, totalEvents: 0 });
-    }
-    countsMap.get(event.eventType)!.totalEvents++;
-  }
-
-  // Tally feedback
-  for (const fb of allFeedback) {
-    const eventType = eventTypeById.get(fb.eventId);
-    if (!eventType) continue;
-
-    if (!countsMap.has(eventType)) {
-      countsMap.set(eventType, { correct: 0, incorrect: 0, boundaryEdit: 0, missed: 0, totalEvents: 0 });
-    }
-
-    const counts = countsMap.get(eventType)!;
-    switch (fb.feedbackType) {
-      case 'correct':
-        counts.correct++;
-        break;
-      case 'incorrect':
-        counts.incorrect++;
-        break;
-      case 'boundary_edit':
-        counts.boundaryEdit++;
-        break;
-      case 'missed':
-        counts.missed++;
-        break;
-    }
-  }
-
-  const statsByEventType: EventTypeStats[] = [];
-  for (const [eventType, counts] of countsMap) {
-    const precisionDenom = counts.correct + counts.incorrect;
-    const recallDenom = counts.correct + counts.boundaryEdit + counts.missed;
-
-    statsByEventType.push({
-      eventType,
-      totalEvents: counts.totalEvents,
-      correct: counts.correct,
-      incorrect: counts.incorrect,
-      boundaryEdit: counts.boundaryEdit,
-      missed: counts.missed,
-      totalFeedback: counts.correct + counts.incorrect + counts.boundaryEdit + counts.missed,
-      precision: precisionDenom > 0 ? counts.correct / precisionDenom : null,
-      recall: recallDenom > 0 ? (counts.correct + counts.boundaryEdit) / recallDenom : null,
-    });
-  }
-
-  return {
-    assetId,
-    totalFeedbackEntries: allFeedback.length,
-    statsByEventType,
-  };
-}
-
-/**
- * Export all feedback for an asset as structured JSON for offline analysis.
- */
-export function exportFeedback(assetId: string): FeedbackExport {
-  const allFeedback = getFeedbackForAsset(assetId);
-
-  return {
-    assetId,
-    exportedAt: new Date().toISOString(),
-    totalEntries: allFeedback.length,
-    feedback: allFeedback,
-  };
-}

package/src/config/bundled-skills/media-processing/services/feedback-store.ts

@@ -1,136 +0,0 @@
-/**
- * Feedback store service for media event corrections.
- *
- * Provides CRUD operations for the media_event_feedback table.
- * All interfaces are generic — works for any event type, not just turnovers.
- */
-
-import { and, eq } from 'drizzle-orm';
-import { v4 as uuid } from 'uuid';
-import { getDb } from '../../../../memory/db.js';
-import { mediaEventFeedback } from '../../../../memory/schema.js';
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
-
-export type FeedbackType = 'correct' | 'incorrect' | 'boundary_edit' | 'missed';
-
-export interface EventFeedback {
-  id: string;
-  assetId: string;
-  eventId: string;
-  feedbackType: FeedbackType;
-  originalStartTime: number | null;
-  originalEndTime: number | null;
-  correctedStartTime: number | null;
-  correctedEndTime: number | null;
-  notes: string | null;
-  createdAt: number;
-}
-
-export interface SubmitFeedbackParams {
-  assetId: string;
-  eventId: string;
-  feedbackType: FeedbackType;
-  originalStartTime?: number;
-  originalEndTime?: number;
-  correctedStartTime?: number;
-  correctedEndTime?: number;
-  notes?: string;
-}
-
-// ---------------------------------------------------------------------------
-// Validation
-// ---------------------------------------------------------------------------
-
-const VALID_FEEDBACK_TYPES: FeedbackType[] = ['correct', 'incorrect', 'boundary_edit', 'missed'];
-
-function isValidFeedbackType(type: string): type is FeedbackType {
-  return VALID_FEEDBACK_TYPES.includes(type as FeedbackType);
-}
-
-// ---------------------------------------------------------------------------
-// CRUD
-// ---------------------------------------------------------------------------
-
-export function submitFeedback(params: SubmitFeedbackParams): EventFeedback {
-  if (!isValidFeedbackType(params.feedbackType)) {
-    throw new Error(`Invalid feedback type "${params.feedbackType}". Must be one of: ${VALID_FEEDBACK_TYPES.join(', ')}`);
-  }
-
-  const db = getDb();
-  const now = Date.now();
-  const record = {
-    id: uuid(),
-    assetId: params.assetId,
-    eventId: params.eventId,
-    feedbackType: params.feedbackType,
-    originalStartTime: params.originalStartTime ?? null,
-    originalEndTime: params.originalEndTime ?? null,
-    correctedStartTime: params.correctedStartTime ?? null,
-    correctedEndTime: params.correctedEndTime ?? null,
-    notes: params.notes ?? null,
-    createdAt: now,
-  };
-
-  db.insert(mediaEventFeedback).values(record).run();
-
-  return record;
-}
-
-export function getFeedbackForAsset(assetId: string): EventFeedback[] {
-  const db = getDb();
-  const rows = db
-    .select()
-    .from(mediaEventFeedback)
-    .where(eq(mediaEventFeedback.assetId, assetId))
-    .all();
-  return rows.map(parseRow);
-}
-
-export function getFeedbackForEvent(eventId: string): EventFeedback[] {
-  const db = getDb();
-  const rows = db
-    .select()
-    .from(mediaEventFeedback)
-    .where(eq(mediaEventFeedback.eventId, eventId))
-    .all();
-  return rows.map(parseRow);
-}
-
-export function getFeedbackByType(assetId: string, feedbackType: FeedbackType): EventFeedback[] {
-  if (!isValidFeedbackType(feedbackType)) {
-    throw new Error(`Invalid feedback type "${feedbackType}". Must be one of: ${VALID_FEEDBACK_TYPES.join(', ')}`);
-  }
-
-  const db = getDb();
-  const rows = db
-    .select()
-    .from(mediaEventFeedback)
-    .where(and(
-      eq(mediaEventFeedback.assetId, assetId),
-      eq(mediaEventFeedback.feedbackType, feedbackType),
-    ))
-    .all();
-  return rows.map(parseRow);
-}
-
-// ---------------------------------------------------------------------------
-// Row parsing
-// ---------------------------------------------------------------------------
-
-function parseRow(row: typeof mediaEventFeedback.$inferSelect): EventFeedback {
-  return {
-    id: row.id,
-    assetId: row.assetId,
-    eventId: row.eventId,
-    feedbackType: row.feedbackType as FeedbackType,
-    originalStartTime: row.originalStartTime,
-    originalEndTime: row.originalEndTime,
-    correctedStartTime: row.correctedStartTime,
-    correctedEndTime: row.correctedEndTime,
-    notes: row.notes,
-    createdAt: row.createdAt,
-  };
-}

package/src/config/bundled-skills/media-processing/services/retrieval-service.ts

@@ -1,95 +0,0 @@
-/**
- * Generic media event retrieval service.
- *
- * Pure data retrieval with configurable filters and ranking — no
- * domain-specific logic. Callers (e.g. a query tool) are responsible
- * for translating domain concepts into filter parameters.
- */
-
-import {
-  getEventsForAsset,
-  type MediaEvent,
-} from '../../../../memory/media-store.js';
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
-
-export interface RetrievalFilters {
-  /** Scope results to a specific media asset. */
-  assetId?: string;
-  /** Filter by event type label. */
-  eventType?: string;
-  /** Minimum confidence threshold (0–1). */
-  minConfidence?: number;
-  /** Maximum number of results to return. */
-  limit?: number;
-  /** Sort order for results. */
-  sortBy?: 'confidence' | 'startTime';
-  /** Only return events that start at or after this time (seconds). */
-  startTimeMin?: number;
-  /** Only return events that start at or before this time (seconds). */
-  startTimeMax?: number;
-}
-
-export interface RetrievalResult {
-  events: MediaEvent[];
-  totalReturned: number;
-  filters: RetrievalFilters;
-}
-
-// ---------------------------------------------------------------------------
-// Retrieval
-// ---------------------------------------------------------------------------
-
-/**
- * Query the media_events table with the given filters and return ranked
- * results with full event metadata.
- *
- * When `assetId` is not provided the function returns an empty result set
- * because the underlying store requires an asset scope. Callers that want
- * cross-asset queries should iterate over asset IDs externally.
- */
-export function retrieveEvents(filters: RetrievalFilters): RetrievalResult {
-  const {
-    assetId,
-    eventType,
-    minConfidence,
-    limit = 10,
-    sortBy = 'confidence',
-    startTimeMin,
-    startTimeMax,
-  } = filters;
-
-  if (!assetId) {
-    return { events: [], totalReturned: 0, filters };
-  }
-
-  // Fetch from the store with the subset of filters it supports natively
-  let events = getEventsForAsset(assetId, {
-    eventType,
-    minConfidence,
-    sortBy,
-    // Fetch more than needed so we can apply time-range filtering locally
-    limit: startTimeMin !== undefined || startTimeMax !== undefined ? undefined : limit,
-  });
-
-  // Apply time-range filters that the store doesn't support directly
-  if (startTimeMin !== undefined) {
-    events = events.filter((e) => e.startTime >= startTimeMin);
-  }
-  if (startTimeMax !== undefined) {
-    events = events.filter((e) => e.startTime <= startTimeMax);
-  }
-
-  // Re-apply limit after local filtering
-  if (limit && events.length > limit) {
-    events = events.slice(0, limit);
-  }
-
-  return {
-    events,
-    totalReturned: events.length,
-    filters,
-  };
-}