@vellumai/assistant 0.3.4 → 0.3.6

package/src/config/bundled-skills/media-processing/tools/generate-clip.ts

@@ -0,0 +1,195 @@
+/**
+ * Clip generation tool — extract a video segment from a media asset.
+ *
+ * Uses ffmpeg to cut a segment with configurable pre/post-roll padding,
+ * then registers the resulting clip as an attachment for in-chat delivery.
+ * This is a generic media-processing primitive with no domain-specific logic.
+ */
+
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { mkdir, unlink, stat, rmdir } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import { getMediaAssetById } from '../../../../memory/media-store.js';
+import { uploadAttachment } from '../../../../memory/attachments-store.js';
+
+const FFMPEG_TIMEOUT_MS = 300_000;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function spawnWithTimeout(
+  cmd: string[],
+  timeoutMs: number,
+): Promise<{ exitCode: number; stdout: string; stderr: string }> {
+  return new Promise((resolve, reject) => {
+    const proc = Bun.spawn(cmd, { stdout: 'pipe', stderr: 'pipe' });
+    const timer = setTimeout(() => {
+      proc.kill();
+      reject(new Error(`Process timed out after ${timeoutMs}ms: ${cmd[0]}`));
+    }, timeoutMs);
+    proc.exited.then(async (exitCode) => {
+      clearTimeout(timer);
+      const stdout = await new Response(proc.stdout).text();
+      const stderr = await new Response(proc.stderr).text();
+      resolve({ exitCode, stdout, stderr });
+    });
+  });
+}
+
+/**
+ * Get the duration of a media file in seconds via ffprobe.
+ */
+async function getMediaDuration(filePath: string): Promise<number> {
+  const result = await spawnWithTimeout([
+    'ffprobe', '-v', 'error',
+    '-show_entries', 'format=duration',
+    '-of', 'csv=p=0',
+    filePath,
+  ], 10_000);
+  if (result.exitCode !== 0) return 0;
+  return parseFloat(result.stdout.trim()) || 0;
+}
+
+function formatTimestamp(seconds: number): string {
+  const hrs = Math.floor(seconds / 3600);
+  const mins = Math.floor((seconds % 3600) / 60);
+  const secs = Math.floor(seconds % 60);
+  return `${hrs.toString().padStart(2, '0')}:${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
+}
+
+const MIME_BY_FORMAT: Record<string, string> = {
+  mp4: 'video/mp4',
+  webm: 'video/webm',
+  mov: 'video/quicktime',
+};
+
+// ---------------------------------------------------------------------------
+// Tool entry point
+// ---------------------------------------------------------------------------
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  if (!assetId) {
+    return { content: 'asset_id is required.', isError: true };
+  }
+
+  const startTime = input.start_time as number | undefined;
+  if (startTime == null) {
+    return { content: 'start_time is required (seconds).', isError: true };
+  }
+
+  const endTime = input.end_time as number | undefined;
+  if (endTime == null) {
+    return { content: 'end_time is required (seconds).', isError: true };
+  }
+
+  if (endTime <= startTime) {
+    return { content: 'end_time must be greater than start_time.', isError: true };
+  }
+
+  const preRoll = (input.pre_roll as number) ?? 3;
+  const postRoll = (input.post_roll as number) ?? 2;
+  const outputFormat = (input.output_format as string) ?? 'mp4';
+
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    return { content: `Media asset not found: ${assetId}`, isError: true };
+  }
+
+  if (asset.mediaType !== 'video') {
+    return { content: `Clip generation requires a video asset. Got: ${asset.mediaType}`, isError: true };
+  }
+
+  // Get the file duration so we can clamp pre/post-roll to file boundaries
+  const fileDuration = asset.durationSeconds ?? await getMediaDuration(asset.filePath);
+
+  // Calculate actual clip boundaries with pre/post-roll, clamped to file
+  const clipStart = Math.max(0, startTime - preRoll);
+  const clipEnd = fileDuration > 0 ? Math.min(fileDuration, endTime + postRoll) : endTime + postRoll;
+  const clipDuration = clipEnd - clipStart;
+
+  // Prepare output path
+  const clipDir = join(tmpdir(), `vellum-clips-${randomUUID()}`);
+  await mkdir(clipDir, { recursive: true });
+
+  const clipFilename = `clip-${formatTimestamp(startTime).replace(/:/g, '')}-${formatTimestamp(endTime).replace(/:/g, '')}.${outputFormat}`;
+  const clipPath = join(clipDir, clipFilename);
+
+  try {
+    context.onOutput?.(`Extracting clip ${formatTimestamp(clipStart)} – ${formatTimestamp(clipEnd)} from ${asset.title}...\n`);
+
+    // Use ffmpeg to extract the segment
+    const ffmpegArgs = [
+      'ffmpeg', '-y',
+      '-ss', String(clipStart),
+      '-i', asset.filePath,
+      '-t', String(clipDuration),
+      '-c', 'copy',
+      '-avoid_negative_ts', 'make_zero',
+      clipPath,
+    ];
+
+    const result = await spawnWithTimeout(ffmpegArgs, FFMPEG_TIMEOUT_MS);
+
+    if (result.exitCode !== 0) {
+      return {
+        content: `ffmpeg clip extraction failed: ${result.stderr.slice(0, 500)}`,
+        isError: true,
+      };
+    }
+
+    // Verify the output file exists and has content
+    const clipStat = await stat(clipPath);
+    if (clipStat.size === 0) {
+      return { content: 'Clip extraction produced an empty file.', isError: true };
+    }
+
+    context.onOutput?.(`Clip extracted (${(clipStat.size / 1024 / 1024).toFixed(1)} MB). Registering as attachment...\n`);
+
+    // Read clip file and register as attachment
+    const clipData = await readFile(clipPath);
+    const clipBase64 = clipData.toString('base64');
+    const mimeType = MIME_BY_FORMAT[outputFormat] ?? 'video/mp4';
+
+    const attachment = uploadAttachment(clipFilename, mimeType, clipBase64);
+
+    context.onOutput?.(`Clip registered as attachment ${attachment.id}.\n`);
+
+    return {
+      content: JSON.stringify({
+        message: `Clip extracted successfully`,
+        attachmentId: attachment.id,
+        filename: clipFilename,
+        mimeType,
+        sizeBytes: attachment.sizeBytes,
+        clipStart,
+        clipEnd,
+        clipDuration,
+        requestedRange: {
+          startTime,
+          endTime,
+          preRoll,
+          postRoll,
+        },
+        assetId,
+      }, null, 2),
+      isError: false,
+    };
+  } catch (err) {
+    return {
+      content: `Clip generation failed: ${(err as Error).message}`,
+      isError: true,
+    };
+  } finally {
+    // Clean up temp file and directory
+    try { await unlink(clipPath); } catch { /* ignore */ }
+    try { await rmdir(clipDir); } catch { /* ignore */ }
+  }
+}

package/src/config/bundled-skills/media-processing/tools/ingest-media.ts

@@ -0,0 +1,197 @@
+import { basename, extname } from 'node:path';
+import { access } from 'node:fs/promises';
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import {
+  registerMediaAsset,
+  getMediaAssetByHash,
+  createProcessingStage,
+  updateMediaAssetStatus,
+  computeFileHash,
+  type MediaType,
+} from '../../../../memory/media-store.js';
+import { enqueueMemoryJob } from '../../../../memory/jobs-store.js';
+
+// ---------------------------------------------------------------------------
+// MIME detection
+// ---------------------------------------------------------------------------
+
+const MIME_MAP: Record<string, string> = {
+  // Video
+  '.mp4': 'video/mp4', '.mov': 'video/quicktime', '.avi': 'video/x-msvideo',
+  '.mkv': 'video/x-matroska', '.webm': 'video/webm', '.m4v': 'video/x-m4v',
+  '.mpeg': 'video/mpeg', '.mpg': 'video/mpeg', '.ts': 'video/mp2t',
+  '.flv': 'video/x-flv', '.wmv': 'video/x-ms-wmv',
+  // Audio
+  '.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.m4a': 'audio/x-m4a',
+  '.aac': 'audio/aac', '.ogg': 'audio/ogg', '.flac': 'audio/flac',
+  '.aiff': 'audio/aiff', '.wma': 'audio/x-ms-wma',
+  // Image
+  '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
+  '.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp',
+  '.tiff': 'image/tiff', '.svg': 'image/svg+xml', '.heic': 'image/heic',
+};
+
+function detectMimeType(filePath: string): string | null {
+  const ext = extname(filePath).toLowerCase();
+  return MIME_MAP[ext] ?? null;
+}
+
+function classifyMediaType(mimeType: string): MediaType | null {
+  if (mimeType.startsWith('video/')) return 'video';
+  if (mimeType.startsWith('audio/')) return 'audio';
+  if (mimeType.startsWith('image/')) return 'image';
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// ffprobe duration extraction
+// ---------------------------------------------------------------------------
+
+function spawnWithTimeout(
+  cmd: string[],
+  timeoutMs: number,
+): Promise<{ exitCode: number; stdout: string; stderr: string }> {
+  return new Promise((resolve, reject) => {
+    const proc = Bun.spawn(cmd, { stdout: 'pipe', stderr: 'pipe' });
+    const timer = setTimeout(() => {
+      proc.kill();
+      reject(new Error(`Process timed out after ${timeoutMs}ms: ${cmd[0]}`));
+    }, timeoutMs);
+    proc.exited.then(async (exitCode) => {
+      clearTimeout(timer);
+      const stdout = await new Response(proc.stdout).text();
+      const stderr = await new Response(proc.stderr).text();
+      resolve({ exitCode, stdout, stderr });
+    });
+  });
+}
+
+async function extractDuration(filePath: string): Promise<number | null> {
+  try {
+    const result = await spawnWithTimeout([
+      'ffprobe', '-v', 'error',
+      '-show_entries', 'format=duration',
+      '-of', 'csv=p=0',
+      filePath,
+    ], 15_000);
+    if (result.exitCode !== 0) return null;
+    const duration = parseFloat(result.stdout.trim());
+    return Number.isFinite(duration) ? duration : null;
+  } catch {
+    // ffprobe not available or timed out
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const filePath = input.file_path as string | undefined;
+  if (!filePath) {
+    return { content: 'file_path is required. Provide an absolute path to a local media file.', isError: true };
+  }
+
+  // Validate file exists
+  try {
+    await access(filePath);
+  } catch {
+    return { content: `File not found: ${filePath}`, isError: true };
+  }
+
+  // Detect MIME type
+  const mimeType = detectMimeType(filePath);
+  if (!mimeType) {
+    return {
+      content: `Unsupported file type: ${extname(filePath)}. Supported: video (mp4, mov, avi, mkv, webm, etc.), audio (mp3, wav, m4a, etc.), image (png, jpg, gif, webp, etc.).`,
+      isError: true,
+    };
+  }
+
+  const mediaType = classifyMediaType(mimeType);
+  if (!mediaType) {
+    return { content: `Could not classify media type for MIME: ${mimeType}`, isError: true };
+  }
+
+  // Compute content hash for dedup – uses the same Bun.hash (wyhash) as
+  // media-store.ts so that hashes are consistent across ingest and lookup.
+  context.onOutput?.('Computing content hash...\n');
+  const fileBytes = await Bun.file(filePath).arrayBuffer();
+  const fileHash = computeFileHash(new Uint8Array(fileBytes));
+
+  // Check for existing asset with same hash
+  const existingAsset = getMediaAssetByHash(fileHash);
+  if (existingAsset) {
+    return {
+      content: JSON.stringify({
+        message: 'Media asset already registered (duplicate detected by content hash)',
+        asset: existingAsset,
+        deduplicated: true,
+      }, null, 2),
+      isError: false,
+    };
+  }
+
+  // Extract duration for video/audio
+  let durationSeconds: number | null = null;
+  if (mediaType === 'video' || mediaType === 'audio') {
+    context.onOutput?.('Extracting duration via ffprobe...\n');
+    durationSeconds = await extractDuration(filePath);
+  }
+
+  // Determine title
+  const title = (input.title as string) || basename(filePath);
+
+  // Parse optional metadata
+  let metadata: Record<string, unknown> | undefined;
+  if (input.metadata && typeof input.metadata === 'object') {
+    metadata = input.metadata as Record<string, unknown>;
+  }
+
+  // Register the asset
+  const asset = registerMediaAsset({
+    title,
+    filePath,
+    mimeType,
+    durationSeconds,
+    fileHash,
+    mediaType,
+    metadata,
+  });
+
+  // Create an initial processing stage
+  createProcessingStage({
+    assetId: asset.id,
+    stage: 'ingest',
+  });
+
+  // Update status to processing
+  updateMediaAssetStatus(asset.id, 'processing');
+
+  // Enqueue a processing job via the existing jobs framework
+  enqueueMemoryJob('media_processing', {
+    mediaAssetId: asset.id,
+    stage: 'ingest',
+    filePath,
+    mimeType,
+    mediaType,
+  });
+
+  context.onOutput?.(`Registered media asset: ${asset.id}\n`);
+
+  return {
+    content: JSON.stringify({
+      message: 'Media asset registered and processing enqueued',
+      asset: {
+        ...asset,
+        status: 'processing',
+      },
+      deduplicated: false,
+    }, null, 2),
+    isError: false,
+  };
+}

package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts

@@ -0,0 +1,143 @@
+/**
+ * Media diagnostics tool.
+ *
+ * Surfaces processing stats, per-stage timing, failure reasons,
+ * and cost estimation for a media asset.
+ * All metrics are generic media-processing infrastructure.
+ */
+
+import { join, dirname } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import {
+  getMediaAssetById,
+  getProcessingStagesForAsset,
+  getKeyframesForAsset,
+  type ProcessingStage,
+} from '../../../../memory/media-store.js';
+import type { PreprocessManifest } from '../services/preprocess.js';
+// ---------------------------------------------------------------------------
+// Cost estimation constants (Gemini 2.5 Flash pricing)
+// ---------------------------------------------------------------------------
+
+/** Estimated cost per segment Map call (Gemini 2.5 Flash with ~10 frames). */
+const ESTIMATED_COST_PER_SEGMENT_USD = 0.001;
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+interface StageDiagnostic {
+  stage: string;
+  status: string;
+  progress: number;
+  durationMs: number | null;
+  lastError: string | null;
+}
+
+interface DiagnosticReport {
+  assetId: string;
+  assetTitle: string;
+  assetStatus: string;
+  mediaType: string;
+  durationSeconds: number | null;
+  processingStats: {
+    totalKeyframes: number;
+  };
+  stages: StageDiagnostic[];
+  costEstimate: {
+    keyframeCount: number;
+    estimatedSegments: number;
+    estimatedCostPerSegment: number;
+    estimatedTotalCost: number;
+    currency: string;
+    note: string;
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function computeStageDuration(stage: ProcessingStage): number | null {
+  if (stage.startedAt == null) return null;
+  if (stage.completedAt != null) return stage.completedAt - stage.startedAt;
+  // Only use Date.now() as a fallback for currently running stages
+  if (stage.status === 'running') return Date.now() - stage.startedAt;
+  // For failed/pending stages without completedAt, duration is unknown
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+export async function run(
+  input: Record<string, unknown>,
+  _context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  if (!assetId) {
+    return { content: 'asset_id is required.', isError: true };
+  }
+
+  const asset = getMediaAssetById(assetId);
+  if (!asset) {
+    return { content: `Media asset not found: ${assetId}`, isError: true };
+  }
+
+  // Gather processing stats
+  const keyframes = getKeyframesForAsset(assetId);
+
+  // Per-stage diagnostics (new pipeline: preprocess, map, reduce)
+  const stages = getProcessingStagesForAsset(assetId);
+  const stageDiagnostics: StageDiagnostic[] = stages.map((s) => ({
+    stage: s.stage,
+    status: s.status,
+    progress: s.progress,
+    durationMs: computeStageDuration(s),
+    lastError: s.lastError,
+  }));
+
+  // Cost estimation: Gemini 2.5 Flash is ~$0.001 per segment (~10 frames each)
+  const keyframeCount = keyframes.length;
+
+  // Prefer actual segment count from preprocess manifest when available
+  let estimatedSegments: number;
+  const manifestPath = join(dirname(asset.filePath), 'pipeline', asset.id, 'manifest.json');
+  try {
+    const raw = await readFile(manifestPath, 'utf-8');
+    const manifest: PreprocessManifest = JSON.parse(raw);
+    estimatedSegments = manifest.segments.length;
+  } catch {
+    // Manifest doesn't exist yet (preprocess hasn't run) — fall back to estimation
+    estimatedSegments = Math.ceil(keyframeCount / 10);
+  }
+
+  const estimatedTotalCost = estimatedSegments * ESTIMATED_COST_PER_SEGMENT_USD;
+
+  const report: DiagnosticReport = {
+    assetId: asset.id,
+    assetTitle: asset.title,
+    assetStatus: asset.status,
+    mediaType: asset.mediaType,
+    durationSeconds: asset.durationSeconds,
+    processingStats: {
+      totalKeyframes: keyframeCount,
+    },
+    stages: stageDiagnostics,
+    costEstimate: {
+      keyframeCount,
+      estimatedSegments,
+      estimatedCostPerSegment: ESTIMATED_COST_PER_SEGMENT_USD,
+      estimatedTotalCost: Math.round(estimatedTotalCost * 1000) / 1000,
+      currency: 'USD',
+      note: 'Gemini 2.5 Flash for Map phase; Claude for Reduce phase (additional cost).',
+    },
+  };
+
+  return {
+    content: JSON.stringify(report, null, 2),
+    isError: false,
+  };
+}

package/src/config/bundled-skills/media-processing/tools/media-status.ts

@@ -0,0 +1,75 @@
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import {
+  getMediaAssetById,
+  getMediaAssetByFilePath,
+  getMediaAssetsByStatus,
+  getProcessingStagesForAsset,
+  type MediaAssetStatus,
+} from '../../../../memory/media-store.js';
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+export async function run(
+  input: Record<string, unknown>,
+  _context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  const filePath = input.file_path as string | undefined;
+  const statusFilter = input.status_filter as MediaAssetStatus | undefined;
+
+  // Query by asset ID
+  if (assetId) {
+    const asset = getMediaAssetById(assetId);
+    if (!asset) {
+      return { content: `Media asset not found: ${assetId}`, isError: true };
+    }
+    const stages = getProcessingStagesForAsset(asset.id);
+    return {
+      content: JSON.stringify({ asset, stages }, null, 2),
+      isError: false,
+    };
+  }
+
+  // Query by file path
+  if (filePath) {
+    const asset = getMediaAssetByFilePath(filePath);
+    if (!asset) {
+      return { content: `No media asset found for path: ${filePath}`, isError: true };
+    }
+    const stages = getProcessingStagesForAsset(asset.id);
+    return {
+      content: JSON.stringify({ asset, stages }, null, 2),
+      isError: false,
+    };
+  }
+
+  // Query by status filter
+  if (statusFilter) {
+    const validStatuses: MediaAssetStatus[] = ['registered', 'processing', 'indexed', 'failed'];
+    if (!validStatuses.includes(statusFilter)) {
+      return {
+        content: `Invalid status filter: ${statusFilter}. Valid values: ${validStatuses.join(', ')}`,
+        isError: true,
+      };
+    }
+    const assets = getMediaAssetsByStatus(statusFilter);
+    const results = assets.map((asset) => ({
+      asset,
+      stages: getProcessingStagesForAsset(asset.id),
+    }));
+    return {
+      content: JSON.stringify({
+        count: results.length,
+        assets: results,
+      }, null, 2),
+      isError: false,
+    };
+  }
+
+  return {
+    content: 'Provide at least one query parameter: asset_id, file_path, or status_filter.',
+    isError: true,
+  };
+}

package/src/config/bundled-skills/media-processing/tools/query-media-events.ts

@@ -0,0 +1,65 @@
+/**
+ * Query media tool — sends natural language queries against video
+ * analysis data (map output) via Claude for intelligent answers.
+ *
+ * Replaces the old keyword-matching approach with an LLM-powered
+ * reduce/query step that can answer arbitrary questions about the
+ * video content.
+ */
+
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import { reduceForAsset, type ReduceOptions, type ReduceResult } from '../services/reduce.js';
+
+// ---------------------------------------------------------------------------
+// Exported function for job handler use (one-shot merge mode)
+// ---------------------------------------------------------------------------
+
+export { reduceForAsset } from '../services/reduce.js';
+
+// ---------------------------------------------------------------------------
+// Tool entry point
+// ---------------------------------------------------------------------------
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const assetId = input.asset_id as string | undefined;
+  if (!assetId) {
+    return { content: 'asset_id is required.', isError: true };
+  }
+
+  const query = input.query as string | undefined;
+  if (!query) {
+    return { content: 'query is required.', isError: true };
+  }
+
+  const systemPrompt = input.system_prompt as string | undefined;
+  const model = input.model as string | undefined;
+
+  const options: ReduceOptions = {
+    query,
+    systemPrompt,
+    model,
+  };
+
+  try {
+    const result: ReduceResult = await reduceForAsset(assetId, options, context.onOutput);
+
+    return {
+      content: JSON.stringify({
+        query,
+        answer: result.answer,
+        model: result.model,
+        usage: {
+          inputTokens: result.inputTokens,
+          outputTokens: result.outputTokens,
+        },
+      }, null, 2),
+      isError: false,
+    };
+  } catch (err) {
+    const msg = (err as Error).message;
+    return { content: msg, isError: true };
+  }
+}