@vellumai/assistant 0.3.0

package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts

@@ -0,0 +1,370 @@
+import { tmpdir } from 'node:os';
+import { join, extname } from 'node:path';
+import { writeFile, unlink, access, readFile, mkdir, readdir } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
+import { getAttachmentsByIds } from '../../../../memory/attachments-store.js';
+import { getConfig } from '../../../../config/loader.js';
+
+const VIDEO_EXTENSIONS = new Set(['.mp4', '.mov', '.avi', '.mkv', '.webm', '.m4v', '.mpeg', '.mpg']);
+const AUDIO_EXTENSIONS = new Set(['.mp3', '.wav', '.m4a', '.aac', '.ogg', '.flac', '.aiff', '.wma']);
+
+/** Timeout for ffmpeg operations. */
+const FFMPEG_TIMEOUT_MS = 120_000;
+
+/** Max file size for a single OpenAI Whisper API request (25MB). */
+const WHISPER_API_MAX_BYTES = 25 * 1024 * 1024;
+
+/** Duration per chunk when splitting for the API (10 minutes — stays well under 25MB as WAV). */
+const API_CHUNK_DURATION_SECS = 600;
+
+/** Timeout for a single Whisper API request. */
+const API_REQUEST_TIMEOUT_MS = 300_000;
+
+/** Timeout for a single whisper.cpp chunk transcription. */
+const LOCAL_CHUNK_TIMEOUT_MS = 600_000;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function spawnWithTimeout(
+  cmd: string[],
+  timeoutMs: number,
+): Promise<{ exitCode: number; stdout: string; stderr: string }> {
+  return new Promise((resolve, reject) => {
+    const proc = Bun.spawn(cmd, { stdout: 'pipe', stderr: 'pipe' });
+    const timer = setTimeout(() => {
+      proc.kill();
+      reject(new Error(`Process timed out after ${timeoutMs}ms: ${cmd[0]}`));
+    }, timeoutMs);
+    proc.exited.then(async (exitCode) => {
+      clearTimeout(timer);
+      const stdout = await new Response(proc.stdout).text();
+      const stderr = await new Response(proc.stderr).text();
+      resolve({ exitCode, stdout, stderr });
+    });
+  });
+}
+
+async function getAudioDuration(audioPath: string): Promise<number> {
+  const result = await spawnWithTimeout([
+    'ffprobe', '-v', 'error',
+    '-show_entries', 'format=duration',
+    '-of', 'csv=p=0',
+    audioPath,
+  ], 10_000);
+  if (result.exitCode !== 0) return 0;
+  return parseFloat(result.stdout.trim()) || 0;
+}
+
+async function splitAudio(
+  audioPath: string,
+  chunkDir: string,
+  chunkDurationSecs: number,
+): Promise<string[]> {
+  const chunkPattern = join(chunkDir, 'chunk-%03d.wav');
+  const result = await spawnWithTimeout([
+    'ffmpeg', '-y',
+    '-i', audioPath,
+    '-f', 'segment',
+    '-segment_time', String(chunkDurationSecs),
+    '-acodec', 'pcm_s16le',
+    '-ar', '16000',
+    '-ac', '1',
+    chunkPattern,
+  ], FFMPEG_TIMEOUT_MS);
+  if (result.exitCode !== 0) {
+    throw new Error(`Failed to split audio: ${result.stderr.slice(0, 300)}`);
+  }
+  const files = await readdir(chunkDir);
+  return files
+    .filter(f => f.startsWith('chunk-') && f.endsWith('.wav'))
+    .sort()
+    .map(f => join(chunkDir, f));
+}
+
+// ---------------------------------------------------------------------------
+// Source resolution
+// ---------------------------------------------------------------------------
+
+async function resolveSource(
+  input: Record<string, unknown>,
+): Promise<{ inputPath: string; isVideo: boolean; tempFile: string | null } | ToolExecutionResult> {
+  const filePath = input.file_path as string | undefined;
+  const attachmentId = input.attachment_id as string | undefined;
+
+  if (filePath) {
+    try { await access(filePath); } catch {
+      return { content: `File not found: ${filePath}`, isError: true };
+    }
+    const ext = extname(filePath).toLowerCase();
+    const isVideo = VIDEO_EXTENSIONS.has(ext);
+    const isAudio = AUDIO_EXTENSIONS.has(ext);
+    if (!isVideo && !isAudio) {
+      return { content: `Unsupported file type: ${ext}. Only video and audio files can be transcribed.`, isError: true };
+    }
+    return { inputPath: filePath, isVideo, tempFile: null };
+  }
+
+  if (attachmentId) {
+    const attachments = getAttachmentsByIds([attachmentId]);
+    if (attachments.length === 0) {
+      return { content: `Attachment not found: ${attachmentId}`, isError: true };
+    }
+    const attachment = attachments[0];
+    const mime = attachment.mimeType;
+    if (!mime.startsWith('video/') && !mime.startsWith('audio/')) {
+      return { content: `Unsupported file type: ${mime}. Only video and audio files can be transcribed.`, isError: true };
+    }
+    const ext = mime.startsWith('video/') ? '.mp4' : '.m4a';
+    const tempPath = join(tmpdir(), `vellum-transcribe-in-${randomUUID()}${ext}`);
+    await writeFile(tempPath, Buffer.from(attachment.dataBase64, 'base64'));
+    return { inputPath: tempPath, isVideo: mime.startsWith('video/'), tempFile: tempPath };
+  }
+
+  return { content: 'Provide either file_path or attachment_id.', isError: true };
+}
+
+/** Convert source to 16kHz mono WAV for consistent processing. */
+async function toWav(inputPath: string, isVideo: boolean): Promise<string> {
+  const wavPath = join(tmpdir(), `vellum-transcribe-${randomUUID()}.wav`);
+  const args = ['ffmpeg', '-y', '-i', inputPath];
+  if (isVideo) args.push('-vn');
+  args.push('-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1', wavPath);
+  const result = await spawnWithTimeout(args, FFMPEG_TIMEOUT_MS);
+  if (result.exitCode !== 0) {
+    throw new Error(`ffmpeg failed: ${result.stderr.slice(0, 500)}`);
+  }
+  return wavPath;
+}
+
+// ---------------------------------------------------------------------------
+// API mode — OpenAI Whisper API
+// ---------------------------------------------------------------------------
+
+async function transcribeViaApi(
+  audioPath: string,
+  apiKey: string,
+  context: ToolContext,
+): Promise<string> {
+  const duration = await getAudioDuration(audioPath);
+  const fileSize = Bun.file(audioPath).size;
+
+  // If small enough, send directly
+  if (fileSize <= WHISPER_API_MAX_BYTES) {
+    return await whisperApiRequest(audioPath, apiKey);
+  }
+
+  // Split into chunks for large files
+  const chunkDir = join(tmpdir(), `vellum-transcribe-api-chunks-${randomUUID()}`);
+  await mkdir(chunkDir, { recursive: true });
+
+  try {
+    context.onOutput?.(`Large file (${Math.round(duration / 60)}min) — splitting into chunks...\n`);
+    const chunks = await splitAudio(audioPath, chunkDir, API_CHUNK_DURATION_SECS);
+    const parts: string[] = [];
+
+    for (let i = 0; i < chunks.length; i++) {
+      if (context.signal?.aborted) throw new Error('Cancelled');
+      context.onOutput?.(`  Transcribing chunk ${i + 1}/${chunks.length}...\n`);
+      const text = await whisperApiRequest(chunks[i], apiKey);
+      if (text) parts.push(text);
+    }
+
+    return parts.join(' ');
+  } finally {
+    const { rm } = await import('node:fs/promises');
+    await rm(chunkDir, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
+async function whisperApiRequest(audioPath: string, apiKey: string): Promise<string> {
+  const audioData = await readFile(audioPath);
+  const formData = new FormData();
+  formData.append('file', new Blob([audioData], { type: 'audio/wav' }), 'audio.wav');
+  formData.append('model', 'whisper-1');
+
+  const response = await fetch('https://api.openai.com/v1/audio/transcriptions', {
+    method: 'POST',
+    headers: { 'Authorization': `Bearer ${apiKey}` },
+    body: formData,
+    signal: AbortSignal.timeout(API_REQUEST_TIMEOUT_MS),
+  });
+
+  if (!response.ok) {
+    const body = await response.text().catch(() => '');
+    throw new Error(`Whisper API error (${response.status}): ${body.slice(0, 300)}`);
+  }
+
+  const result = await response.json() as { text?: string };
+  return result.text?.trim() ?? '';
+}
+
+// ---------------------------------------------------------------------------
+// Local mode — whisper.cpp
+// ---------------------------------------------------------------------------
+
+async function transcribeViaLocal(
+  audioPath: string,
+  context: ToolContext,
+): Promise<string> {
+  // Check if whisper-cpp is installed
+  const whichResult = await spawnWithTimeout(['which', 'whisper-cpp'], 5_000);
+  if (whichResult.exitCode !== 0) {
+    throw new Error(
+      'whisper-cpp is not installed. Install it with: brew install whisper-cpp'
+    );
+  }
+
+  // Resolve model path — use the base model, download if needed
+  const modelPath = await resolveWhisperModel(context);
+
+  const duration = await getAudioDuration(audioPath);
+
+  if (duration > 0 && duration <= 1800) {
+    // Under 30 minutes — transcribe directly (whisper.cpp handles long files well)
+    context.onOutput?.(`Transcribing ${Math.round(duration / 60)}min of audio locally...\n`);
+    return await whisperCppRun(audioPath, modelPath);
+  }
+
+  // Very long files — split into 10-minute chunks to show progress
+  const chunkDir = join(tmpdir(), `vellum-transcribe-local-chunks-${randomUUID()}`);
+  await mkdir(chunkDir, { recursive: true });
+
+  try {
+    context.onOutput?.(`Large file (${Math.round(duration / 60)}min) — splitting into chunks...\n`);
+    const chunks = await splitAudio(audioPath, chunkDir, 600);
+    const parts: string[] = [];
+
+    for (let i = 0; i < chunks.length; i++) {
+      if (context.signal?.aborted) throw new Error('Cancelled');
+      context.onOutput?.(`  Transcribing chunk ${i + 1}/${chunks.length}...\n`);
+      const text = await whisperCppRun(chunks[i], modelPath);
+      if (text) parts.push(text);
+    }
+
+    return parts.join(' ');
+  } finally {
+    const { rm } = await import('node:fs/promises');
+    await rm(chunkDir, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
+async function resolveWhisperModel(context: ToolContext): Promise<string> {
+  // Check common locations for the base model
+  const homeDir = process.env.HOME ?? '/tmp';
+  const candidates = [
+    join(homeDir, '.vellum', 'models', 'ggml-base.en.bin'),
+    join(homeDir, '.vellum', 'models', 'ggml-base.bin'),
+    '/usr/local/share/whisper-cpp/models/ggml-base.en.bin',
+    '/opt/homebrew/share/whisper-cpp/models/ggml-base.en.bin',
+  ];
+
+  for (const p of candidates) {
+    try { await access(p); return p; } catch { /* next */ }
+  }
+
+  // Download the base.en model (~140MB)
+  const modelDir = join(homeDir, '.vellum', 'models');
+  await mkdir(modelDir, { recursive: true });
+  const modelPath = join(modelDir, 'ggml-base.en.bin');
+  const modelUrl = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin';
+
+  context.onOutput?.('Downloading Whisper base.en model (~140MB)...\n');
+
+  const response = await fetch(modelUrl);
+  if (!response.ok) {
+    throw new Error(`Failed to download model: ${response.status}`);
+  }
+
+  const data = Buffer.from(await response.arrayBuffer());
+  await writeFile(modelPath, data);
+  context.onOutput?.('Model downloaded.\n');
+
+  return modelPath;
+}
+
+async function whisperCppRun(audioPath: string, modelPath: string): Promise<string> {
+  const result = await spawnWithTimeout([
+    'whisper-cpp',
+    '-m', modelPath,
+    '-f', audioPath,
+    '--no-timestamps',
+  ], LOCAL_CHUNK_TIMEOUT_MS);
+
+  if (result.exitCode !== 0) {
+    throw new Error(`whisper-cpp failed: ${result.stderr.slice(0, 300)}`);
+  }
+
+  // whisper-cpp outputs transcription to stderr with some logging, and
+  // the actual text lines to stdout. Clean up whitespace.
+  return result.stdout
+    .split('\n')
+    .map(l => l.trim())
+    .filter(l => l.length > 0)
+    .join(' ')
+    .trim();
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  const mode = input.mode as 'api' | 'local';
+  if (!mode || (mode !== 'api' && mode !== 'local')) {
+    return {
+      content: "Please specify mode: 'api' (OpenAI cloud) or 'local' (whisper.cpp on-device). Ask the user which they prefer.",
+      isError: true,
+    };
+  }
+
+  // Validate API key for api mode
+  if (mode === 'api') {
+    const config = getConfig();
+    const apiKey = config.apiKeys.openai;
+    if (!apiKey) {
+      return {
+        content: 'No OpenAI API key configured. Set your OpenAI API key to use cloud transcription, or use mode "local" for on-device transcription with whisper.cpp.',
+        isError: true,
+      };
+    }
+  }
+
+  const source = await resolveSource(input);
+  if ('isError' in source) return source;
+
+  const { inputPath, isVideo, tempFile } = source;
+  let wavPath: string | null = null;
+
+  try {
+    // Convert to WAV
+    wavPath = await toWav(inputPath, isVideo);
+
+    let text: string;
+    if (mode === 'api') {
+      const config = getConfig();
+      text = await transcribeViaApi(wavPath, config.apiKeys.openai!, context);
+    } else {
+      text = await transcribeViaLocal(wavPath, context);
+    }
+
+    if (!text.trim()) {
+      return { content: 'No speech detected in the audio.', isError: false };
+    }
+
+    return { content: text, isError: false };
+  } catch (err) {
+    return {
+      content: `Transcription failed: ${(err as Error).message}`,
+      isError: true,
+    };
+  } finally {
+    if (tempFile) { try { await unlink(tempFile); } catch { /* ignore */ } }
+    if (wavPath) { try { await unlink(wavPath); } catch { /* ignore */ } }
+  }
+}

package/src/config/bundled-skills/twitter/SKILL.md

@@ -0,0 +1,220 @@
+---
+name: "X"
+description: "Read and post on X (formerly Twitter) via OAuth or browser session"
+user-invocable: true
+metadata: {"vellum": {"emoji": "𝕏"}}
+---
+
+You are an X (formerly Twitter) assistant. Use the `execute_bash` tool to run `vellum x` CLI commands.
+
+## Connection Options
+
+There are two supported ways to connect to X. Both are fully functional; choose whichever fits the user's situation.
+
+### OAuth (recommended with X developer credentials)
+
+OAuth uses the official X API v2. It is the most reliable connection method and does not depend on browser sessions.
+
+- Supports: **post** and **reply**
+- Read-only operations (timeline, search, home, bookmarks, notifications, likes, followers, following, media) always use the browser path directly, regardless of the strategy setting.
+- Setup: The user connects OAuth credentials through the Settings UI or the `twitter_auth_start` IPC flow.
+- Set the strategy: `vellum x strategy set oauth`
+
+### Browser session (no developer credentials needed)
+
+The browser path is quick to start and useful when the user does not have X developer app credentials. It captures auth cookies from Chrome and uses them to interact with X.
+
+- Supports: **all operations** (post, reply, timeline, search, home, bookmarks, notifications, likes, followers, following, media)
+- Setup: Run `vellum x refresh` to open Chrome and capture session cookies automatically.
+- Set the strategy: `vellum x strategy set browser`
+
+### Auto mode (default)
+
+When the strategy is `auto` (the default), the router tries OAuth first for supported operations if credentials are available, then falls back to the browser path. This gives the best of both worlds without requiring manual switching.
+
+- Set auto mode: `vellum x strategy set auto`
+
+## First-Use Decision Flow
+
+When the user triggers a Twitter operation and no strategy has been configured yet, follow these steps:
+
+1. **Check current status:**
+   ```bash
+   vellum x status --json
+   ```
+   Look at `oauthConnected`, `browserSessionActive`, `preferredStrategy`, and `strategyConfigured` in the response. If `strategyConfigured` is `false`, the user has not yet chosen a strategy and should be guided through setup.
+
+2. **Present both options with trade-offs:**
+   - **OAuth**: Most reliable and official. Requires X developer app credentials (OAuth Client ID and optional Client Secret). Supports posting and replying. Set up through Settings UI.
+   - **Browser session**: Quick to start, no developer credentials needed. Supports all operations including reading timelines and searching. Set up with `vellum x refresh`.
+
+3. **Ask the user which they prefer.** Do not choose for them.
+
+4. **Execute setup for the chosen path:**
+   - If OAuth: Guide the user to the Settings UI to connect their X developer credentials, or initiate the `twitter_auth_start` IPC flow.
+   - If browser: Run `vellum x refresh` to capture session cookies from Chrome.
+
+5. **Set the preferred strategy:**
+   ```bash
+   vellum x strategy set <oauth|browser|auto>
+   ```
+
+## Failure Recovery Flow
+
+When a Twitter operation fails, follow these steps:
+
+1. **Detect the failure type from the error output:**
+   - `session_expired` or `SessionExpiredError` — the browser session cookies have expired.
+   - `OAuth is not configured` — the user chose OAuth but credentials are not set up.
+   - `Twitter API error (401)` — OAuth token may be expired or revoked.
+   - `UnsupportedOAuthOperationError` — the requested write operation is not available via OAuth.
+   - `Cannot connect to daemon` — the Vellum daemon is not running.
+
+2. **Explain the likely cause clearly** to the user.
+
+3. **Suggest trying the other path as an alternative:**
+   - If the browser session expired: suggest setting up OAuth for post/reply operations, or refresh the browser session with `vellum x refresh`.
+   - If OAuth failed or is not configured: suggest using the browser path with `vellum x strategy set browser` and `vellum x refresh`.
+   - If the operation is unsupported via OAuth: explain that this write operation is not yet supported via OAuth, and suggest using the browser path with `vellum x strategy set browser`.
+
+4. **Offer concrete steps to switch:**
+   ```bash
+   # Switch to the other strategy
+   vellum x strategy set <oauth|browser|auto>
+
+   # If switching to browser, refresh the session
+   vellum x refresh
+   ```
+
+## Strategy Management Commands
+
+```bash
+# Check current strategy
+vellum x strategy
+
+# Set strategy to OAuth, browser, or auto
+vellum x strategy set <oauth|browser|auto>
+
+# Check full status (session, OAuth, and strategy info)
+vellum x status --json
+```
+
+## Posting
+
+```bash
+vellum x post "The post text here"
+```
+
+Returns JSON with `ok`, `tweetId`, `text`, `url`, and `pathUsed` fields. The `pathUsed` field indicates whether the post was sent via `oauth` or `browser`. Share the URL with the user so they can verify the post.
+
+The `post` command routes through the strategy router: it uses OAuth if configured and available, otherwise falls back to the browser path.
+
+## Replying
+
+```bash
+vellum x reply <tweetUrl> "The reply text here"
+```
+
+The first argument is a tweet URL (e.g. `https://x.com/user/status/123456`) or a bare tweet ID.
+
+Like `post`, the `reply` command routes through the strategy router and returns a `pathUsed` field.
+
+## Reading
+
+Read-only operations always use the browser path directly, regardless of the strategy setting. They work the same whether the strategy is `oauth`, `browser`, or `auto` — the strategy only affects `post` and `reply` commands.
+
+### User timeline
+```bash
+vellum x timeline <screenName> [--count N]
+```
+Returns `user` and `tweets` array.
+
+### Single tweet + replies
+```bash
+vellum x tweet <tweetIdOrUrl>
+```
+Returns the focal tweet and its reply thread.
+
+### Search
+```bash
+vellum x search "query" [--count N] [--product Top|Latest|People|Media]
+```
+
+### Home timeline
+```bash
+vellum x home [--count N]
+```
+
+### Bookmarks
+```bash
+vellum x bookmarks [--count N]
+```
+
+### Notifications
+```bash
+vellum x notifications [--count N]
+```
+Returns `notifications` array with `id`, `message`, `timestamp`, `url`.
+
+### Likes
+```bash
+vellum x likes <screenName> [--count N]
+```
+
+### Followers / Following
+```bash
+vellum x followers <screenName> [--count N]
+vellum x following <screenName> [--count N]
+```
+Returns `user` and `followers`/`following` array (userId, screenName, name).
+
+### Media
+```bash
+vellum x media <screenName> [--count N]
+```
+Returns tweets that contain media from the user's profile.
+
+## Workflows
+
+### Check Mentions
+
+When the user asks to check mentions, check X, or see what's happening:
+
+1. Fetch notifications: `vellum x notifications --count 20 --json`
+2. Fetch their recent tweets to see replies: `vellum x timeline <theirScreenName> --count 10 --json`
+3. Summarize what needs attention:
+   - Group by type: replies to their tweets, likes, new followers, mentions
+   - For anything that looks like it needs a reply, fetch the full thread with `vellum x tweet <tweetId>` to understand context
+   - Prioritize: direct questions > mentions > engagement notifications
+4. For items that need replies, draft a response and ask the user to approve before sending with `vellum x reply`
+
+Present the summary as a scannable list, not a wall of text. Lead with action items.
+
+### Research a Topic
+
+When the user wants to understand what people are saying about something:
+
+1. Search: `vellum x search "topic" --count 20 --json`
+2. For the most interesting tweets, fetch threads: `vellum x tweet <tweetId>`
+3. Summarize: key themes, notable voices, sentiment, and any emerging consensus
+4. If the user wants to engage, draft a post or reply that adds to the conversation
+
+### Engagement Check
+
+When the user wants to see how their posts are performing:
+
+1. Fetch their recent tweets: `vellum x timeline <screenName> --count 20 --json`
+2. For each tweet, note engagement signals from the text/metadata
+3. Fetch notifications to see who's interacting: `vellum x notifications --count 20 --json`
+4. Summarize: which posts got traction, who's engaging, any conversations worth continuing
+
+## Tips
+
+- Keep posts under 280 characters
+- All `screenName` arguments should be without the `@` prefix
+- All commands return JSON with an `ok` field
+- When drafting replies, match the tone of the conversation — casual threads get casual replies
+- Always show the user what you're about to post and get approval before sending
+- If a browser session is expired, refresh it with `vellum x refresh` before retrying, or suggest switching to OAuth for post/reply operations
+- If an operation fails, check `vellum x status --json` to diagnose the issue before retrying
+- The `post` and `reply` commands include a `pathUsed` field in their response so you can tell the user which connection method was used

package/src/config/bundled-skills/watcher/SKILL.md

@@ -0,0 +1,27 @@
+---
+name: "Watcher"
+description: "Polling watcher system for monitoring external sources"
+metadata: {"vellum": {"emoji": "👀"}}
+---
+
+Create and manage watchers that poll external services for events and process them with an action prompt.
+
+## Concepts
+
+- **Provider** — The external service to poll (e.g. "gmail"). Each provider defines how to fetch and parse events.
+- **Action prompt** — LLM instructions for handling detected events. Sent along with event data to a background conversation.
+- **Poll interval** — How often to check for new events (minimum 15 seconds, default 60 seconds).
+- **Digest** — Summary of recent watcher activity, grouped by watcher with time-based filtering.
+
+## Lifecycle
+
+1. Create a watcher with a name, provider, and action prompt.
+2. The system polls the provider at the configured interval.
+3. Detected events are processed according to the action prompt.
+4. Use `watcher_digest` to review recent activity.
+
+## Usage Notes
+
+- Use `watcher_create` when the user wants to monitor an external source (e.g. "watch my Gmail for important emails").
+- `watcher_digest` is the go-to tool when the user asks "what happened with my email?" or similar questions about watcher activity.
+- Watchers can be enabled/disabled via `watcher_update` without deleting them.