@vellumai/assistant 0.3.3 → 0.3.5

package/src/config/bundled-skills/media-processing/TOOLS.json

@@ -0,0 +1,320 @@
+{
+  "version": 1,
+  "tools": [
+    {
+      "name": "ingest_media",
+      "description": "Ingest a media file (video, audio, or image) for processing. Validates the file, detects MIME type, extracts duration for video/audio, registers the asset with content-hash dedup, and enqueues an initial processing job.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "file_path": {
+            "type": "string",
+            "description": "Absolute path to a local media file (video, audio, or image)"
+          },
+          "title": {
+            "type": "string",
+            "description": "Optional human-readable title for the media asset. Defaults to the filename."
+          },
+          "metadata": {
+            "type": "object",
+            "description": "Optional JSON metadata to attach to the asset (e.g., pipeline config, source info)"
+          }
+        },
+        "required": ["file_path"]
+      },
+      "executor": "tools/ingest-media.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "media_status",
+      "description": "Query the processing status of one or more media assets, including per-stage progress details.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "asset_id": {
+            "type": "string",
+            "description": "ID of a specific media asset to query"
+          },
+          "file_path": {
+            "type": "string",
+            "description": "File path to look up a media asset by its original path"
+          },
+          "status_filter": {
+            "type": "string",
+            "enum": ["registered", "processing", "indexed", "failed"],
+            "description": "Filter assets by processing status"
+          }
+        }
+      },
+      "executor": "tools/media-status.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "extract_keyframes",
+      "description": "Extract keyframes from a video asset at regular intervals using ffmpeg. Stores frame images and registers each in the database for subsequent vision analysis.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset (must be a video)"
+          },
+          "interval_seconds": {
+            "type": "number",
+            "description": "Interval between keyframes in seconds. Default: 3"
+          }
+        },
+        "required": ["asset_id"]
+      },
+      "executor": "tools/extract-keyframes.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "analyze_keyframes",
+      "description": "Analyze extracted keyframes using Claude VLM (vision language model). Produces structured scene descriptions with subjects, actions, and context for each frame. Supports resumability — skips already-analyzed frames.",
+      "category": "media",
+      "risk": "medium",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset whose keyframes to analyze"
+          },
+          "analysis_type": {
+            "type": "string",
+            "description": "Type of analysis to perform. Default: 'scene_description'"
+          },
+          "batch_size": {
+            "type": "number",
+            "description": "Number of keyframes to process per batch. Default: 10"
+          }
+        },
+        "required": ["asset_id"]
+      },
+      "executor": "tools/analyze-keyframes.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "detect_events",
+      "description": "Detect events from a media asset's timeline segments using configurable detection rules with weighted confidence ranking. Supports built-in rule types: segment_transition (field changes between adjacent segments), short_segment (segments below a duration threshold), and attribute_match (regex matching on segment attributes). If no rules are provided, sensible defaults are used based on the event type.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset to detect events in"
+          },
+          "event_type": {
+            "type": "string",
+            "description": "Type label for detected events (e.g., 'turnover', 'scene_change', 'highlight')"
+          },
+          "detection_rules": {
+            "type": "array",
+            "description": "Optional array of detection rule objects. Each rule has: ruleType (string: 'segment_transition', 'short_segment', or 'attribute_match'), params (object with rule-specific parameters), and weight (number: contribution to confidence score). If omitted, defaults are used based on event_type.",
+            "items": {
+              "type": "object",
+              "properties": {
+                "ruleType": {
+                  "type": "string",
+                  "description": "Rule type: 'segment_transition', 'short_segment', or 'attribute_match'"
+                },
+                "params": {
+                  "type": "object",
+                  "description": "Rule-specific parameters (e.g., { field: 'subjects' }, { maxDurationSeconds: 5 }, { field: 'actions', pattern: 'steal|turnover' })"
+                },
+                "weight": {
+                  "type": "number",
+                  "description": "Weight for this rule's contribution to the confidence score"
+                }
+              },
+              "required": ["ruleType", "params", "weight"]
+            }
+          }
+        },
+        "required": ["asset_id", "event_type"]
+      },
+      "executor": "tools/detect-events.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "query_media_events",
+      "description": "Query media events using natural language. Parses the query into structured filters (event type, count, confidence threshold, time range) and retrieves matching events ranked by confidence. Supports domain-specific keyword mapping (e.g., 'turnovers' → eventType='turnover').",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "query": {
+            "type": "string",
+            "description": "Natural language query describing the events to find (e.g., 'top 5 turnovers', 'high confidence goals in the first half')"
+          },
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset to search within"
+          }
+        },
+        "required": ["query", "asset_id"]
+      },
+      "executor": "tools/query-media-events.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "select_tracking_profile",
+      "description": "Select and persist a tracking profile for a media asset. When called without capabilities, returns available capabilities organized by tier (ready, beta, experimental). When called with capabilities, validates them against the registry and stores the profile. The capability tier system is generic and extensible across domains.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset to configure tracking for"
+          },
+          "capabilities": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "Optional array of capability names to enable (e.g., ['turnover', 'field_goal']). If omitted, returns the available capabilities for selection."
+          }
+        },
+        "required": ["asset_id"]
+      },
+      "executor": "tools/select-tracking-profile.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "generate_clip",
+      "description": "Extract a video clip from a media asset using ffmpeg. Applies configurable pre/post-roll padding (clamped to file boundaries), outputs the clip as a temporary file, and registers it as an attachment for in-chat delivery.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset (must be a video)"
+          },
+          "start_time": {
+            "type": "number",
+            "description": "Start time of the clip in seconds"
+          },
+          "end_time": {
+            "type": "number",
+            "description": "End time of the clip in seconds"
+          },
+          "pre_roll": {
+            "type": "number",
+            "description": "Seconds of padding before start_time. Default: 3"
+          },
+          "post_roll": {
+            "type": "number",
+            "description": "Seconds of padding after end_time. Default: 2"
+          },
+          "output_format": {
+            "type": "string",
+            "enum": ["mp4", "webm", "mov"],
+            "description": "Output video format. Default: 'mp4'"
+          }
+        },
+        "required": ["asset_id", "start_time", "end_time"]
+      },
+      "executor": "tools/generate-clip.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "submit_feedback",
+      "description": "Submit feedback on a detected media event. Supports four feedback types: 'correct' (confirms accuracy), 'incorrect' (marks false positive), 'boundary_edit' (adjusts start/end times), and 'missed' (reports an event the system failed to detect, creating a new event). Works for any event type.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "event_id": {
+            "type": "string",
+            "description": "ID of the event to provide feedback on. Required for all types except 'missed'."
+          },
+          "feedback_type": {
+            "type": "string",
+            "enum": ["correct", "incorrect", "boundary_edit", "missed"],
+            "description": "Type of feedback: correct, incorrect, boundary_edit, or missed"
+          },
+          "corrected_start_time": {
+            "type": "number",
+            "description": "Corrected start time in seconds (for boundary_edit feedback)"
+          },
+          "corrected_end_time": {
+            "type": "number",
+            "description": "Corrected end time in seconds (for boundary_edit feedback)"
+          },
+          "notes": {
+            "type": "string",
+            "description": "Optional free-text notes about the feedback"
+          },
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset (required for 'missed' type)"
+          },
+          "event_type": {
+            "type": "string",
+            "description": "Event type label for the missed event (required for 'missed' type)"
+          },
+          "start_time": {
+            "type": "number",
+            "description": "Start time in seconds for the missed event (required for 'missed' type)"
+          },
+          "end_time": {
+            "type": "number",
+            "description": "End time in seconds for the missed event (required for 'missed' type)"
+          }
+        },
+        "required": ["feedback_type"]
+      },
+      "executor": "tools/submit-feedback.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "media_diagnostics",
+      "description": "Get a diagnostic report for a media asset including processing stats, per-stage timing, failure reasons, cost estimation, and feedback summary.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset to diagnose"
+          }
+        },
+        "required": ["asset_id"]
+      },
+      "executor": "tools/media-diagnostics.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "recalibrate",
+      "description": "Recalibrate event detection for a media asset based on accumulated feedback. Analyzes correction patterns (false positive rates, missed events, boundary adjustments), re-ranks existing events by adjusting confidence scores, and returns a summary of adjustments made. Works for any event type.",
+      "category": "media",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "asset_id": {
+            "type": "string",
+            "description": "ID of the media asset to recalibrate"
+          }
+        },
+        "required": ["asset_id"]
+      },
+      "executor": "tools/recalibrate.ts",
+      "execution_target": "host"
+    }
+  ]
+}

package/src/config/bundled-skills/media-processing/services/capability-registry.ts

@@ -0,0 +1,137 @@
+/**
+ * Generic capability registry with tier-based classification.
+ *
+ * The registry is domain-agnostic: any domain (sports, surveillance, lecture
+ * recording, etc.) can register its own capabilities. Basketball-specific
+ * capabilities are registered as one example via `registerDefaults()`.
+ */
+
+import type { CapabilityTier } from '../../../../memory/media-store.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface Capability {
+  /** Unique name used as the key in tracking profiles (e.g. 'turnover'). */
+  name: string;
+  /** Human-readable description of what this capability detects/tracks. */
+  description: string;
+  /** Maturity tier governing confidence disclaimers and default inclusion. */
+  tier: CapabilityTier;
+  /** Domain this capability belongs to (e.g. 'basketball', 'surveillance'). */
+  domain: string;
+  /** Granularity level (e.g. 'team', 'per-player'). */
+  granularity?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Registry (singleton in-memory Map)
+// ---------------------------------------------------------------------------
+
+const registry = new Map<string, Capability>();
+
+/**
+ * Register a capability. Overwrites any existing capability with the same name.
+ */
+export function registerCapability(cap: Capability): void {
+  registry.set(cap.name, cap);
+}
+
+/**
+ * Get all registered capabilities, optionally filtered by domain.
+ */
+export function getCapabilities(domain?: string): Capability[] {
+  const all = Array.from(registry.values());
+  if (!domain) return all;
+  return all.filter((c) => c.domain === domain);
+}
+
+/**
+ * Get capabilities filtered by tier.
+ */
+export function getCapabilitiesByTier(tier: CapabilityTier): Capability[] {
+  return Array.from(registry.values()).filter((c) => c.tier === tier);
+}
+
+/**
+ * Look up a single capability by name.
+ */
+export function getCapabilityByName(name: string): Capability | undefined {
+  return registry.get(name);
+}
+
+/**
+ * Get all unique domain names in the registry.
+ */
+export function getRegisteredDomains(): string[] {
+  const domains = new Set<string>();
+  for (const cap of registry.values()) {
+    domains.add(cap.domain);
+  }
+  return Array.from(domains);
+}
+
+// ---------------------------------------------------------------------------
+// Default registrations — basketball as one example domain
+// ---------------------------------------------------------------------------
+
+/**
+ * Register the default basketball capabilities as an example domain.
+ * Other domains should call `registerCapability()` with their own entries.
+ */
+export function registerDefaults(): void {
+  // Ready tier: production-quality detection
+  registerCapability({
+    name: 'turnover',
+    description: 'Team-level turnover detection',
+    tier: 'ready',
+    domain: 'basketball',
+    granularity: 'team',
+  });
+
+  // Beta tier: functional but may have accuracy gaps
+  registerCapability({
+    name: 'field_goal',
+    description: 'Team-level field goal detection',
+    tier: 'beta',
+    domain: 'basketball',
+    granularity: 'team',
+  });
+
+  registerCapability({
+    name: 'rebound',
+    description: 'Team-level rebound detection',
+    tier: 'beta',
+    domain: 'basketball',
+    granularity: 'team',
+  });
+
+  registerCapability({
+    name: 'turnover_per_player',
+    description: 'Per-player turnover attribution',
+    tier: 'beta',
+    domain: 'basketball',
+    granularity: 'per-player',
+  });
+
+  // Experimental tier: early-stage, expect noise
+  registerCapability({
+    name: 'field_goal_per_player',
+    description: 'Per-player field goal attribution',
+    tier: 'experimental',
+    domain: 'basketball',
+    granularity: 'per-player',
+  });
+
+  registerCapability({
+    name: 'rebound_per_player',
+    description: 'Per-player rebound attribution',
+    tier: 'experimental',
+    domain: 'basketball',
+    granularity: 'per-player',
+  });
+}
+
+// Auto-register defaults on first import
+registerDefaults();