@vellumai/assistant 0.4.42 → 0.4.43

package/src/config/bundled-skills/doordash/doordash-cli.ts

@@ -5,7 +5,8 @@
  * All commands output JSON to stdout. Use --json for machine-readable output.
  */
 
-import * as net from "node:net";
+import { statSync } from "node:fs";
+import { join } from "node:path";
 
 import { Command } from "commander";
 
@@ -35,9 +36,8 @@ import {
   importFromRecording,
   loadSession,
 } from "./lib/session.js";
-import { createMessageParser, serialize } from "./lib/shared/ipc.js";
 import { NetworkRecorder } from "./lib/shared/network-recorder.js";
-import { getSocketPath, readSessionToken } from "./lib/shared/platform.js";
+import { buildDaemonUrl, getDataDir, getHttpPort, readHttpToken } from "./lib/shared/platform.js";
 import { loadRecording, saveRecording } from "./lib/shared/recording-store.js";
 import type { SessionRecording } from "./lib/shared/recording-types.js";
 
@@ -958,101 +958,135 @@ async function startLearnSession(
     startUrl: "https://www.doordash.com/consumer/login/",
   });
 
-  // Step 2: Connect to daemon and start recording
-  return new Promise((resolve, reject) => {
-    const socketPath = getSocketPath();
-    const sessionToken = readSessionToken();
-    const socket = net.createConnection(socketPath);
-    const parser = createMessageParser();
-
-    socket.on("error", (err) => {
-      reject(
-        new Error(
-          `Cannot connect to assistant: ${err.message}. Is the assistant running?`,
-        ),
-      );
-    });
+  // Step 2: Start ride-shotgun learn session via HTTP
+  const port = getHttpPort();
+  const baseUrl = buildDaemonUrl(port);
+  const token = readHttpToken();
 
-    // Timeout safety — unref so it doesn't keep process alive
-    const timeoutHandle = setTimeout(
-      () => {
-        socket.destroy();
-        reject(
-          new Error(`Learn session timed out after ${durationSeconds + 30}s`),
-        );
-      },
-      (durationSeconds + 30) * 1000,
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+  };
+  if (token) {
+    headers["Authorization"] = `Bearer ${token}`;
+  }
+
+  const startResponse = await fetch(
+    `${baseUrl}/v1/computer-use/ride-shotgun/start`,
+    {
+      method: "POST",
+      headers,
+      body: JSON.stringify({
+        durationSeconds,
+        intervalSeconds: 5,
+        mode: "learn",
+        targetDomain: "doordash.com",
+      }),
+    },
+  );
+
+  if (!startResponse.ok) {
+    const errorBody = await startResponse.text();
+    throw new Error(
+      `Failed to start ride-shotgun session (HTTP ${startResponse.status}): ${errorBody}`,
     );
-    timeoutHandle.unref();
+  }
 
-    let authenticated = !sessionToken; // If no token needed, consider already authenticated
+  const startResult = (await startResponse.json()) as {
+    watchId: string;
+    sessionId: string;
+  };
 
-    const sendStartCommand = () => {
-      socket.write(
-        serialize({
-          type: "ride_shotgun_start",
-          durationSeconds,
-          intervalSeconds: 5,
-          mode: "learn",
-          targetDomain: "doordash.com",
-        } as Record<string, unknown>),
-      );
-    };
+  if (!startResult.watchId) {
+    throw new Error("Ride-shotgun start response missing watchId");
+  }
+
+  // Step 3: Poll session status endpoint for completion or failure, then
+  // look for the correlated recording file using the recordingId from the session.
+  const { watchId } = startResult;
+  const statusUrl = `${baseUrl}/v1/computer-use/ride-shotgun/status/${watchId}`;
+  const timeoutMs = (durationSeconds + 30) * 1000;
+  const pollIntervalMs = 2000;
+  const startTime = Date.now();
+
+  return new Promise<LearnResult>((resolve, reject) => {
+    const pollOnce = async () => {
+      if (Date.now() - startTime > timeoutMs) {
+        reject(
+          new Error(`Learn session timed out after ${durationSeconds + 30}s`),
+        );
+        return;
+      }
 
-    socket.on("data", (chunk) => {
-      const messages = parser.feed(chunk.toString("utf-8"));
-      for (const msg of messages) {
-        const m = msg as unknown as Record<string, unknown>;
+      // Poll session status to detect failures early
+      try {
+        const fetchAbort = AbortSignal.timeout(10_000);
+        const statusRes = await fetch(statusUrl, { headers, signal: fetchAbort });
+        if (statusRes.ok) {
+          const status = (await statusRes.json()) as {
+            status: string;
+            recordingId?: string;
+            savedRecordingPath?: string;
+            bootstrapFailureReason?: string;
+          };
 
-        // Handle auth handshake
-        if (!authenticated && m.type === "auth_result") {
-          if ((m as { success: boolean }).success) {
-            authenticated = true;
-            sendStartCommand();
-          } else {
-            clearTimeout(timeoutHandle);
-            socket.destroy();
-            reject(new Error("Authentication failed"));
+          // Session failed without producing a recording
+          if (status.bootstrapFailureReason) {
+            reject(
+              new Error(
+                `Learn session failed: ${status.bootstrapFailureReason}`,
+              ),
+            );
+            return;
           }
-          continue;
-        }
 
-        // Skip duplicate auth_result after already authenticated
-        if (m.type === "auth_result") {
-          continue;
-        }
+          // Session completed — check for the correlated recording file
+          if (status.status === "completed") {
+            if (status.savedRecordingPath && status.recordingId) {
+              resolve({
+                recordingId: status.recordingId,
+                recordingPath: status.savedRecordingPath,
+              });
+              return;
+            }
 
-        if (m.type === "ride_shotgun_error") {
-          clearTimeout(timeoutHandle);
-          socket.destroy();
-          reject(new Error((m as { message: string }).message));
-          continue;
-        }
+            // Recording path not in status — fall back to filesystem lookup
+            // using the recordingId for correlation
+            if (status.recordingId) {
+              const recordingsDir = join(getDataDir(), "recordings");
+              const expectedPath = join(
+                recordingsDir,
+                `${status.recordingId}.json`,
+              );
+              try {
+                statSync(expectedPath);
+                resolve({
+                  recordingId: status.recordingId,
+                  recordingPath: expectedPath,
+                });
+                return;
+              } catch {
+                // Recording file not yet written — continue polling
+                setTimeout(pollOnce, pollIntervalMs);
+                return;
+              }
+            }
 
-        if (m.type === "ride_shotgun_result") {
-          clearTimeout(timeoutHandle);
-          socket.destroy();
-          resolve({
-            recordingId: m.recordingId as string | undefined,
-            recordingPath: m.recordingPath as string | undefined,
-          });
+            // Completed but no recordingId — cannot correlate
+            reject(
+              new Error(
+                "Learn session completed but no recording was saved.",
+              ),
+            );
+            return;
+          }
         }
+      } catch {
+        // Status endpoint not reachable — continue polling
       }
-    });
 
-    socket.on("connect", () => {
-      if (sessionToken) {
-        // Send auth and wait for auth_result before sending the command
-        socket.write(
-          serialize({
-            type: "auth",
-            token: sessionToken,
-          } as Record<string, unknown>),
-        );
-      } else {
-        // No auth needed, send command immediately
-        sendStartCommand();
-      }
-    });
+      setTimeout(pollOnce, pollIntervalMs);
+    };
+
+    setTimeout(pollOnce, pollIntervalMs);
   });
 }

package/src/config/bundled-skills/doordash/lib/session.ts

@@ -13,7 +13,6 @@ import {
 import { join } from "node:path";
 
 import { ConfigError } from "./shared/errors.js";
-import { getDataDir } from "./shared/platform.js";
 import type {
   ExtractedCredential,
   SessionRecording,
@@ -26,7 +25,7 @@ export interface DoorDashSession {
 }
 
 function getSessionDir(): string {
-  return join(getDataDir(), "doordash");
+  return join(process.env.VELLUM_DATA_DIR!, "doordash");
 }
 
 function getSessionPath(): string {

package/src/config/bundled-skills/doordash/lib/shared/platform.ts

@@ -16,19 +16,36 @@ export function getDataDir(): string {
   return join(getRootDir(), "workspace", "data");
 }
 
-export function getSocketPath(): string {
-  const override = process.env.VELLUM_DAEMON_SOCKET?.trim();
-  if (override) {
-    if (override === "~") return homedir();
-    if (override.startsWith("~/")) return join(homedir(), override.slice(2));
-    return override;
+/** Default daemon HTTP port — matches cli/src/lib/constants.ts. */
+const DEFAULT_DAEMON_PORT = 7821;
+
+/**
+ * Resolve the daemon HTTP port from RUNTIME_HTTP_PORT env var or default.
+ */
+export function getHttpPort(): number {
+  const envPort = process.env.RUNTIME_HTTP_PORT;
+  if (envPort) {
+    const parsed = parseInt(envPort, 10);
+    if (!isNaN(parsed)) return parsed;
   }
-  return join(getRootDir(), "vellum.sock");
+  return DEFAULT_DAEMON_PORT;
 }
 
-export function readSessionToken(): string | null {
+/**
+ * Build the base URL for the daemon HTTP server.
+ */
+export function buildDaemonUrl(port?: number): string {
+  return `http://127.0.0.1:${port ?? getHttpPort()}`;
+}
+
+/**
+ * Read the HTTP bearer token from `<rootDir>/http-token`.
+ * Returns null if the token file doesn't exist or is empty.
+ */
+export function readHttpToken(): string | null {
   try {
-    return readFileSync(join(getRootDir(), "session-token"), "utf-8").trim();
+    const token = readFileSync(join(getRootDir(), "http-token"), "utf-8").trim();
+    return token || null;
   } catch {
     return null;
   }

package/src/config/bundled-skills/elevenlabs-voice/SKILL.md

@@ -0,0 +1,140 @@
+---
+name: elevenlabs-voice
+description: Select and tune an ElevenLabs TTS voice — curated voice list, custom/cloned voices via API key, and tuning parameters
+compatibility: "Designed for Vellum personal assistants"
+metadata: {"emoji":"🗣️","vellum":{"display-name":"ElevenLabs Voice","user-invocable":true}}
+---
+
+## Overview
+
+ElevenLabs provides text-to-speech voices for both **in-app TTS** and **phone calls**. The shared config key `elevenlabs.voiceId` controls the voice across all channels. Use the `voice_config_update` tool to change the voice — it writes to the config file and pushes to the macOS app via SSE in one call.
+
+## Choose a Voice
+
+Pick a voice that matches the your identity and the user's preferences. Offer to show the full list if they want to choose themselves.
+
+### Female voices
+
+| Voice     | Style                      | Voice ID               |
+| --------- | -------------------------- | ---------------------- |
+| Rachel    | Calm, warm, conversational | `21m00Tcm4TlvDq8ikWAM` |
+| Sarah     | Soft, young, approachable  | `EXAVITQu4vr4xnSDxMaL` |
+| Charlotte | Warm, Swedish-accented     | `XB0fDUnXU5powFXDhCwa` |
+| Alice     | Confident, British         | `Xb7hH8MSUJpSbSDYk0k2` |
+| Matilda   | Warm, friendly, young      | `XrExE9yKIg1WjnnlVkGX` |
+| Lily      | Warm, British              | `pFZP5JQG7iQjIQuC4Bku` |
+
+### Male voices
+
+| Voice   | Style                           | Voice ID               |
+| ------- | ------------------------------- | ---------------------- |
+| Antoni  | Warm, well-rounded              | `ErXwobaYiN019PkySvjV` |
+| Josh    | Deep, young, clear              | `TxGEqnHWrfWFTfGW9XjX` |
+| Arnold  | Crisp, narrative                | `VR6AewLTigWG4xSOukaG` |
+| Adam    | Deep, middle-aged, professional | `pNInz6obpgDQGcFmaJgB` |
+| Bill    | Trustworthy, American           | `pqHfZKP75CvOlQylNhV4` |
+| George  | Warm, British, distinguished    | `JBFqnCBsd6RMkjVDRZzb` |
+| Daniel  | Authoritative, British          | `onwK4e9ZLuTAKqWW03F9` |
+| Charlie | Casual, Australian              | `IKne3meq5aSn9XLyUdCD` |
+| Liam    | Young, articulate               | `TX3LPaxmHKxFdv7VOQHJ` |
+
+### Setting the voice
+
+To set the chosen voice, use `voice_config_update`. This writes to the config file (`elevenlabs.voiceId`) for phone calls **and** pushes to the macOS app via SSE (`ttsVoiceId`) for in-app TTS in one call:
+
+```
+voice_config_update setting="tts_voice_id" value="<selected-voice-id>"
+```
+
+Verify it worked:
+
+```bash
+assistant config get elevenlabs.voiceId
+```
+
+Tell the user what voice you chose and why, but also offer to show all available voices so they can choose for themselves.
+
+## ElevenLabs API Key Setup
+
+For advanced voice selection (browsing the full library, custom/cloned voices), the user needs an ElevenLabs API key. A free tier is available at https://elevenlabs.io.
+
+To collect the API key securely:
+
+```
+credential_store action="prompt" service="elevenlabs" field="api_key"
+```
+
+## Advanced Voice Selection (with API key)
+
+Users with an ElevenLabs API key can go beyond the curated list above.
+
+### Check for an existing key
+
+```bash
+assistant credentials inspect elevenlabs:api_key --json
+```
+
+### Browse the voice library
+
+```bash
+curl -s "https://api.elevenlabs.io/v2/voices?category=premade&page_size=50" \
+  -H "xi-api-key: $(assistant credentials reveal elevenlabs:api_key)" | python3 -m json.tool
+```
+
+### Search for a specific style
+
+```bash
+curl -s "https://api.elevenlabs.io/v2/voices?search=warm+female&page_size=10" \
+  -H "xi-api-key: $(assistant credentials reveal elevenlabs:api_key)" | python3 -m json.tool
+```
+
+### Custom and cloned voices
+
+If the user has created a custom voice or voice clone in their ElevenLabs account, they can use its voice ID directly. These voices work in both in-app TTS and Twilio ConversationRelay.
+
+### Preview voices
+
+Each voice in the API response includes a `preview_url` with an audio sample the user can listen to before deciding.
+
+### Set the chosen voice
+
+After the user picks a voice from the library:
+
+```
+voice_config_update setting="tts_voice_id" value="<selected-voice-id>"
+```
+
+## Voice Tuning
+
+Fine-tune how the selected voice sounds. These parameters apply to all ElevenLabs modes (in-app TTS and phone calls):
+
+```bash
+# Playback speed (0.7 = slower, 1.0 = normal, 1.2 = faster)
+assistant config set elevenlabs.speed 1.0
+
+# Stability (0.0 = more expressive/variable, 1.0 = more consistent/monotone)
+assistant config set elevenlabs.stability 0.5
+
+# Similarity boost (0.0 = more creative, 1.0 = closer to original voice)
+assistant config set elevenlabs.similarityBoost 0.75
+```
+
+Lower stability makes the voice more expressive but less predictable — good for conversational calls. Higher stability is better for scripted or formal contexts.
+
+## Voice Model Tuning
+
+By default, the system sends a **bare** `voiceId` to Twilio ConversationRelay (no model/tuning suffix). This is the safest default across voice IDs.
+
+To optionally force Twilio's extended voice spec, set a model ID:
+
+```bash
+assistant config set elevenlabs.voiceModelId "flash_v2_5"
+```
+
+When `voiceModelId` is set, the emitted voice string becomes: `voiceId-model-speed_stability_similarity`.
+
+To clear and revert to the bare voiceId default:
+
+```bash
+assistant config set elevenlabs.voiceModelId ""
+```

package/src/config/bundled-skills/email-setup/SKILL.md

@@ -1,8 +1,8 @@
 ---
-name: "Email Setup"
-description: "Create the assistant's own email address via the Vellum hosted API (one-time setup)"
-user-invocable: true
-metadata: { "vellum": { "emoji": "📧" } }
+name: email-setup
+description: Create the assistant's own email address via the Vellum hosted API (one-time setup)
+compatibility: "Designed for Vellum personal assistants"
+metadata: {"emoji":"📧","vellum":{"display-name":"Email Setup","user-invocable":true}}
 ---
 
 You are setting up your own personal email address. This is a one-time operation — once you have an email, you do not need to run this again.

package/src/config/bundled-skills/followups/SKILL.md

@@ -1,7 +1,8 @@
 ---
-name: "Followups"
-description: "Track sent messages awaiting responses across communication channels"
-metadata: {"vellum": {"emoji": "\ud83d\udce8"}}
+name: followups
+description: Track sent messages awaiting responses across communication channels
+compatibility: "Designed for Vellum personal assistants"
+metadata: {"emoji":"📨","vellum":{"display-name":"Followups"}}
 ---
 
 Track messages sent on external channels (email, Slack, WhatsApp, etc.) that are awaiting a response.

package/src/config/bundled-skills/frontend-design/SKILL.md

@@ -1,7 +1,9 @@
 ---
 name: frontend-design
 description: Create distinctive, production-grade frontend interfaces with high design quality. Use this skill when the user asks to build web components, pages, artifacts, posters, or applications (examples include websites, landing pages, dashboards, React components, HTML/CSS layouts, or when styling/beautifying any web UI). Generates creative, polished code and UI design that avoids generic AI aesthetics.
+compatibility: "Designed for Vellum personal assistants"
 license: Complete terms in LICENSE.txt
+metadata: {"emoji":"🎨"}
 ---
 
 This skill guides creation of distinctive, production-grade frontend interfaces that avoid generic "AI slop" aesthetics. Implement real working code with exceptional attention to aesthetic details and creative choices.

package/src/config/bundled-skills/google-calendar/SKILL.md

@@ -1,8 +1,8 @@
 ---
-name: "Google Calendar"
-description: "View, create, and manage Google Calendar events and check availability"
-user-invocable: true
-metadata: { "vellum": { "emoji": "📅" } }
+name: google-calendar
+description: View, create, and manage Google Calendar events and check availability
+compatibility: "Designed for Vellum personal assistants"
+metadata: {"emoji":"📅","vellum":{"display-name":"Google Calendar","user-invocable":true}}
 ---
 
 You are a Google Calendar assistant with full access to the user's calendar. Use the Calendar tools to help them view, create, and manage events.

package/src/config/bundled-skills/google-oauth-setup/SKILL.md

@@ -1,10 +1,8 @@
 ---
-name: "Google OAuth Setup"
-description: "Set up Google Cloud OAuth credentials for Gmail and Calendar using browser automation"
-user-invocable: true
-credential-setup-for: "gmail"
-includes: ["browser", "public-ingress"]
-metadata: { "vellum": { "emoji": "\ud83d\udd11" } }
+name: google-oauth-setup
+description: Set up Google Cloud OAuth credentials for Gmail and Calendar using browser automation
+compatibility: "Designed for Vellum personal assistants"
+metadata: {"emoji":"🔑","vellum":{"display-name":"Google OAuth Setup","user-invocable":true,"includes":["browser","public-ingress"],"credential-setup-for":"gmail"}}
 ---
 
 You are helping your user set up Google Cloud OAuth credentials so Gmail and Google Calendar integrations can connect.