@vellumai/assistant 0.6.3 → 0.6.4

package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts

@@ -0,0 +1,82 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+import { getWorkspaceDir } from "../../../../util/platform.js";
+
+interface GmailPreferences {
+  /** Sender emails archived in previous sessions — auto-archive candidates. */
+  blocklist: string[];
+  /** Sender emails the user explicitly kept (deselected) — exclude from future tables. */
+  safelist: string[];
+}
+
+const PREFS_FILENAME = "gmail-preferences.json";
+
+function getPrefsPath(): string {
+  return join(getWorkspaceDir(), "data", PREFS_FILENAME);
+}
+
+export function loadPreferences(): GmailPreferences {
+  try {
+    const raw = readFileSync(getPrefsPath(), "utf-8");
+    const parsed = JSON.parse(raw) as Partial<GmailPreferences>;
+    return {
+      blocklist: Array.isArray(parsed.blocklist) ? parsed.blocklist : [],
+      safelist: Array.isArray(parsed.safelist) ? parsed.safelist : [],
+    };
+  } catch {
+    return { blocklist: [], safelist: [] };
+  }
+}
+
+function savePreferences(prefs: GmailPreferences): void {
+  const path = getPrefsPath();
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, JSON.stringify(prefs, null, 2));
+}
+
+/** Add sender emails to the blocklist (deduplicated). */
+export function addToBlocklist(emails: string[]): void {
+  const prefs = loadPreferences();
+  const existing = new Set(prefs.blocklist);
+  for (const email of emails) {
+    const normalized = email.toLowerCase();
+    existing.add(normalized);
+    // If a sender is blocklisted, remove them from safelist
+    const safeIdx = prefs.safelist.indexOf(normalized);
+    if (safeIdx !== -1) prefs.safelist.splice(safeIdx, 1);
+  }
+  prefs.blocklist = [...existing];
+  savePreferences(prefs);
+}
+
+/** Add sender emails to the safelist (deduplicated). */
+export function addToSafelist(emails: string[]): void {
+  const prefs = loadPreferences();
+  const existing = new Set(prefs.safelist);
+  for (const email of emails) {
+    const normalized = email.toLowerCase();
+    existing.add(normalized);
+    // If a sender is safelisted, remove them from blocklist
+    const blockIdx = prefs.blocklist.indexOf(normalized);
+    if (blockIdx !== -1) prefs.blocklist.splice(blockIdx, 1);
+  }
+  prefs.safelist = [...existing];
+  savePreferences(prefs);
+}
+
+/** Remove sender emails from the blocklist. */
+export function removeFromBlocklist(emails: string[]): void {
+  const prefs = loadPreferences();
+  const toRemove = new Set(emails.map((e) => e.toLowerCase()));
+  prefs.blocklist = prefs.blocklist.filter((e) => !toRemove.has(e));
+  savePreferences(prefs);
+}
+
+/** Remove sender emails from the safelist. */
+export function removeFromSafelist(emails: string[]): void {
+  const prefs = loadPreferences();
+  const toRemove = new Set(emails.map((e) => e.toLowerCase()));
+  prefs.safelist = prefs.safelist.filter((e) => !toRemove.has(e));
+  savePreferences(prefs);
+}

package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts

@@ -11,9 +11,31 @@ import type {
 import { storeScanResult } from "./scan-result-store.js";
 import { err, ok } from "./shared.js";
 
-const MAX_MESSAGES_CAP = 10000;
+function isRateLimitError(e: unknown): boolean {
+  if (!(e instanceof Error)) return false;
+  return /\b429\b/.test(e.message);
+}
+
+function isAbortError(e: unknown): boolean {
+  if (e instanceof DOMException && e.name === "AbortError") return true;
+  if (e instanceof Error && e.message === "fetch deadline exceeded") return true;
+  // AbortSignal.reason propagated through promise chains
+  if (
+    e instanceof Error &&
+    "cause" in e &&
+    e.cause instanceof Error &&
+    e.cause.message === "fetch deadline exceeded"
+  )
+    return true;
+  return false;
+}
+
+const MAX_MESSAGES_CAP = 5000;
 const MAX_IDS_PER_SENDER = 5000;
 const MAX_SAMPLE_SUBJECTS = 3;
+const MAX_SENDERS_CAP = 75;
+const MAX_SUBJECT_LENGTH = 80;
+const MAX_RESULT_BYTES = 24_000;
 
 interface SenderAggregation {
   displayName: string;
@@ -52,10 +74,13 @@ export async function run(
   const query =
     (input.query as string) ?? "in:inbox category:promotions newer_than:90d";
   const maxMessages = Math.min(
-    (input.max_messages as number) ?? 5000,
+    (input.max_messages as number) ?? 2000,
     MAX_MESSAGES_CAP,
   );
-  const maxSenders = (input.max_senders as number) ?? 50;
+  const maxSenders = Math.min(
+    (input.max_senders as number) ?? 50,
+    MAX_SENDERS_CAP,
+  );
   const inputPageToken = input.page_token as string | undefined;
 
   try {
@@ -66,6 +91,7 @@ export async function run(
     // overlapping fetch latency with pagination latency
     const allMessageIds: string[] = [];
     const fetchPromises: Promise<GmailMessage[]>[] = [];
+    const fetchAbort = new AbortController();
     let pageToken: string | undefined = inputPageToken;
     let truncated = false;
     let timeBudgetExceeded = false;
@@ -73,6 +99,8 @@ export async function run(
     const startTime = Date.now();
     const TIME_BUDGET_MS = 90_000;
 
+    let rateLimited = false;
+
     while (allMessageIds.length < maxMessages) {
       if (Date.now() - startTime > TIME_BUDGET_MS) {
         timeBudgetExceeded = true;
@@ -80,12 +108,22 @@ export async function run(
         break;
       }
       const pageSize = Math.min(100, maxMessages - allMessageIds.length);
-      const listResp = await listMessages(
-        connection,
-        query,
-        pageSize,
-        pageToken,
-      );
+      let listResp;
+      try {
+        listResp = await listMessages(
+          connection,
+          query,
+          pageSize,
+          pageToken,
+        );
+      } catch (e) {
+        if (isRateLimitError(e)) {
+          rateLimited = true;
+          truncated = true;
+          break;
+        }
+        throw e;
+      }
       const ids = (listResp.messages ?? []).map((m) => m.id);
       if (ids.length === 0) break;
       allMessageIds.push(...ids);
@@ -96,6 +134,7 @@ export async function run(
           "metadata",
           metadataHeaders,
           "id,internalDate,payload/headers",
+          fetchAbort.signal,
         ),
       );
       pageToken = listResp.nextPageToken ?? undefined;
@@ -108,6 +147,18 @@ export async function run(
     }
 
     if (allMessageIds.length === 0) {
+      if (rateLimited) {
+        return ok(
+          JSON.stringify({
+            senders: [],
+            total_scanned: 0,
+            rate_limited: true,
+            truncated: true,
+            message:
+              "Rate limited before any messages could be fetched. Try again later or reduce max_messages.",
+          }),
+        );
+      }
       return ok(
         JSON.stringify({
           senders: [],
@@ -118,7 +169,30 @@ export async function run(
       );
     }
 
-    const messages = (await Promise.all(fetchPromises)).flat();
+    // Settle all fetch promises — collect successes and tolerate 429 failures.
+    // Abort in-flight requests when the deadline fires so they don't continue
+    // consuming API quota in the background.
+    const elapsedMs = Date.now() - startTime;
+    const settleDeadlineMs = Math.max(TIME_BUDGET_MS - elapsedMs, 5_000);
+    const deadlineTimer = setTimeout(() => {
+      fetchAbort.abort(new Error("fetch deadline exceeded"));
+    }, settleDeadlineMs);
+    const settled = await Promise.allSettled(fetchPromises);
+    clearTimeout(deadlineTimer);
+    const messages: GmailMessage[] = [];
+    for (const result of settled) {
+      if (result.status === "fulfilled") {
+        messages.push(...result.value);
+      } else if (isRateLimitError(result.reason)) {
+        rateLimited = true;
+        truncated = true;
+      } else if (isAbortError(result.reason)) {
+        timeBudgetExceeded = true;
+        truncated = true;
+      } else {
+        throw result.reason;
+      }
+    }
 
     // Group by sender email
     const senderMap = new Map<string, SenderAggregation>();
@@ -221,17 +295,38 @@ export async function run(
       newest_date: s.newestDate,
       // Preserve original query filters so follow-up searches stay scoped
       search_query: `from:${s.email} ${query}`,
-      sample_subjects: s.sampleSubjects,
+      sample_subjects: s.sampleSubjects.map((subj) =>
+        subj.length > MAX_SUBJECT_LENGTH
+          ? subj.slice(0, MAX_SUBJECT_LENGTH) + "…"
+          : subj,
+      ),
     }));
 
+    // Trim senders if the serialized result would exceed the byte budget.
+    // Senders are already sorted by message_count desc, so we drop the
+    // least-active ones first.
+    while (resultSenders.length > 1) {
+      const probe = JSON.stringify({ senders: resultSenders });
+      if (probe.length <= MAX_RESULT_BYTES) break;
+      resultSenders.pop();
+    }
+
+    // Build a set of sender IDs that survived the trim so the scan store
+    // only holds entries the LLM can reference.
+    const keptSenderIds = new Set(resultSenders.map((s) => s.id));
+
     // Store message IDs server-side to keep them out of LLM context
     const scanId = storeScanResult(
-      sorted.map((s) => ({
-        id: Buffer.from(s.email).toString("base64url"),
-        messageIds: s.messageIds,
-        newestMessageId: s.newestMessageId,
-        newestUnsubscribableMessageId: s.newestUnsubscribableMessageId,
-      })),
+      sorted
+        .filter((s) =>
+          keptSenderIds.has(Buffer.from(s.email).toString("base64url")),
+        )
+        .map((s) => ({
+          id: Buffer.from(s.email).toString("base64url"),
+          messageIds: s.messageIds,
+          newestMessageId: s.newestMessageId,
+          newestUnsubscribableMessageId: s.newestUnsubscribableMessageId,
+        })),
     );
 
     return ok(
@@ -242,6 +337,7 @@ export async function run(
         query_used: query,
         ...(truncated ? { truncated: true } : {}),
         ...(timeBudgetExceeded ? { time_budget_exceeded: true } : {}),
+        ...(rateLimited ? { rate_limited: true } : {}),
         note: `message_count reflects emails found per sender within the ${allMessageIds.length} messages scanned. Use scan_id with gmail_archive to archive messages (pass scan_id + sender_ids instead of message_ids).`,
       }),
     );

package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts

@@ -19,9 +19,9 @@ export async function run(
   context: ToolContext,
 ): Promise<ToolExecutionResult> {
   const account = input.account as string | undefined;
-  if (!context.triggeredBySurfaceAction) {
+  if (!context.triggeredBySurfaceAction && !context.batchAuthorizedByTask) {
     return err(
-      "This tool requires user confirmation via a surface action. Present results in a selection table with action buttons and wait for the user to click before proceeding.",
+      "This tool requires either a surface action or a scheduled task run with this tool in required_tools. Present results in a selection table with action buttons and wait for the user to click before proceeding.",
     );
   }
 

package/src/config/bundled-skills/media-processing/SKILL.md

@@ -45,8 +45,7 @@ Parameters:
 - `section_config` - Path to a JSON file with manual section boundaries.
 - `detect_dead_time` - Whether to detect and skip dead time (default: false). Dead-time detection can be too aggressive for continuous action video like sports - it may incorrectly skip live play. Enable only for content with clear idle periods (e.g., lectures, surveillance footage).
 - `short_edge` - Short edge resolution for downscaled frames in pixels (default: 480).
-- `include_audio` - Whether to extract and transcribe audio for each segment (default: false). When enabled, each segment's audio is transcribed and stored alongside visual frames.
-- `transcription_mode` - Transcription backend: `'api'` (OpenAI Whisper cloud) or `'local'` (whisper.cpp on-device). Default: `'local'`. The `'api'` mode requires an OpenAI API key configured in settings.
+- `include_audio` - Whether to extract and transcribe audio for each segment (default: false). When enabled, each segment's audio is transcribed using the configured STT service and stored alongside visual frames.
 
 ### analyze_keyframes
 
@@ -121,7 +120,7 @@ Tracks estimated API costs during pipeline execution.
 
 ## Audio + Vision Multimodal Analysis
 
-When `include_audio` is enabled on `extract_keyframes`, the pipeline transcribes each segment's audio track and attaches the transcript to the segment data. During the Map phase (`analyze_keyframes`), Gemini receives both the visual frames and the audio transcript for each segment, enabling multimodal analysis that combines what is seen with what is said.
+When `include_audio` is enabled on `extract_keyframes`, the pipeline transcribes each segment's audio track using the configured STT service and attaches the transcript to the segment data. During the Map phase (`analyze_keyframes`), Gemini receives both the visual frames and the audio transcript for each segment, enabling multimodal analysis that combines what is seen with what is said.
 
 This is useful for:
 
@@ -130,12 +129,7 @@ This is useful for:
 - **Meetings and interviews**: Pair facial expressions and gestures with spoken dialogue.
 - **Tutorials and demos**: Link on-screen actions with verbal instructions.
 
-Transcription modes:
-
-- **`local`** (default): Uses whisper.cpp for on-device transcription. Requires `whisper-cpp` to be installed (`brew install whisper-cpp`). No API costs, but slower.
-- **`api`**: Uses the OpenAI Whisper API for cloud-based transcription. Faster and more accurate, but requires an OpenAI API key and incurs per-minute costs.
-
-The audio transcription degrades gracefully - if transcription fails for a segment (missing tools, no audio track, API errors), the segment proceeds with visual-only analysis.
+Audio transcription uses the STT service configured in assistant settings. If no STT service is configured or transcription fails for a segment (no audio track, service errors), the segment gracefully degrades to visual-only analysis.
 
 ## Best Practices
 

package/src/config/bundled-skills/media-processing/TOOLS.json

@@ -99,12 +99,7 @@
           },
           "include_audio": {
             "type": "boolean",
-            "description": "Whether to extract and transcribe audio for each segment. Default: false."
-          },
-          "transcription_mode": {
-            "type": "string",
-            "enum": ["api", "local"],
-            "description": "Transcription backend: 'api' (OpenAI Whisper cloud) or 'local' (whisper.cpp). Default: 'local'."
+            "description": "Whether to extract and transcribe audio for each segment using the configured STT service. Default: false."
           },
           "activity": {
             "type": "string",

package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts

@@ -0,0 +1,125 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { BatchTranscriber } from "../../../../stt/types.js";
+
+// ---------------------------------------------------------------------------
+// Mocks — must be declared before the subject import
+// ---------------------------------------------------------------------------
+
+let mockTranscriber: BatchTranscriber | null = null;
+
+mock.module("../../../../providers/speech-to-text/resolve.js", () => ({
+  resolveBatchTranscriber: async () => mockTranscriber,
+}));
+
+let spawnResult: { exitCode: number; stdout: string; stderr: string } = {
+  exitCode: 0,
+  stdout: "",
+  stderr: "",
+};
+
+mock.module("../../../../util/spawn.js", () => ({
+  spawnWithTimeout: async () => spawnResult,
+}));
+
+let mockFileContents: Buffer = Buffer.alloc(0);
+
+mock.module("node:fs/promises", () => ({
+  readFile: async () => mockFileContents,
+  unlink: async () => {},
+}));
+
+// ---------------------------------------------------------------------------
+// Subject import (after mocks)
+// ---------------------------------------------------------------------------
+
+import { transcribeSegmentAudio } from "../services/audio-transcribe.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeMockTranscriber(text: string): BatchTranscriber {
+  return {
+    providerId: "openai-whisper",
+    boundaryId: "daemon-batch",
+    transcribe: async () => ({ text }),
+  };
+}
+
+function makeFailingTranscriber(error: Error): BatchTranscriber {
+  return {
+    providerId: "openai-whisper",
+    boundaryId: "daemon-batch",
+    transcribe: async () => {
+      throw error;
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("transcribeSegmentAudio", () => {
+  beforeEach(() => {
+    mockTranscriber = null;
+    spawnResult = { exitCode: 0, stdout: "", stderr: "" };
+    mockFileContents = Buffer.from("fake-wav-data");
+  });
+
+  test("returns transcript text on successful transcription", async () => {
+    mockTranscriber = makeMockTranscriber("Hello, this is a test transcript.");
+
+    const result = await transcribeSegmentAudio("/tmp/video.mp4", 10, 15);
+
+    expect(result).toBe("Hello, this is a test transcript.");
+  });
+
+  test("returns empty string when no STT provider is configured", async () => {
+    mockTranscriber = null;
+
+    const result = await transcribeSegmentAudio("/tmp/video.mp4", 0, 30);
+
+    expect(result).toBe("");
+  });
+
+  test("returns empty string when ffmpeg extraction fails", async () => {
+    mockTranscriber = makeMockTranscriber("should not reach here");
+    spawnResult = {
+      exitCode: 1,
+      stdout: "",
+      stderr: "ffmpeg: error extracting audio",
+    };
+
+    const result = await transcribeSegmentAudio("/tmp/video.mp4", 5, 10);
+
+    expect(result).toBe("");
+  });
+
+  test("returns empty string when provider throws an error", async () => {
+    mockTranscriber = makeFailingTranscriber(
+      new Error("Provider API rate limited"),
+    );
+
+    const result = await transcribeSegmentAudio("/tmp/video.mp4", 0, 20);
+
+    expect(result).toBe("");
+  });
+
+  test("trims whitespace from transcript result", async () => {
+    mockTranscriber = makeMockTranscriber("  trimmed text  ");
+
+    const result = await transcribeSegmentAudio("/tmp/video.mp4", 0, 10);
+
+    expect(result).toBe("trimmed text");
+  });
+
+  test("returns empty string when provider returns empty text", async () => {
+    mockTranscriber = makeMockTranscriber("");
+
+    const result = await transcribeSegmentAudio("/tmp/video.mp4", 0, 10);
+
+    expect(result).toBe("");
+  });
+});

package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts

@@ -0,0 +1,181 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { PreprocessManifest } from "../services/preprocess.js";
+
+// ---------------------------------------------------------------------------
+// Mocks — must be declared before the subject import
+// ---------------------------------------------------------------------------
+
+let lastPreprocessOptions: Record<string, unknown> | undefined;
+let mockManifest: PreprocessManifest;
+let mockPreprocessError: Error | null = null;
+
+mock.module("../services/preprocess.js", () => ({
+  preprocessForAsset: async (
+    _assetId: string,
+    options: Record<string, unknown>,
+  ) => {
+    lastPreprocessOptions = options;
+    if (mockPreprocessError) throw mockPreprocessError;
+    return mockManifest;
+  },
+}));
+
+mock.module("../../../../memory/media-store.js", () => ({
+  getMediaAssetById: () => ({ filePath: "/tmp/videos/test.mp4" }),
+  getKeyframesForAsset: () => [{ id: "kf-1" }, { id: "kf-2" }, { id: "kf-3" }],
+}));
+
+// ---------------------------------------------------------------------------
+// Subject import (after mocks)
+// ---------------------------------------------------------------------------
+
+import { run } from "../tools/extract-keyframes.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeManifest(
+  overrides: Partial<PreprocessManifest> = {},
+): PreprocessManifest {
+  return {
+    assetId: "asset-1",
+    videoPath: "/tmp/videos/test.mp4",
+    durationSeconds: 60,
+    segments: [
+      {
+        id: "seg-001",
+        startSeconds: 0,
+        endSeconds: 15,
+        framePaths: ["/tmp/frame-1.jpg"],
+        frameTimestamps: [0],
+      },
+    ],
+    deadTimeRanges: [],
+    subjectRegistry: { groups: [] },
+    sectionBoundaries: [],
+    config: {
+      intervalSeconds: 1,
+      segmentDuration: 15,
+      deadTimeThreshold: 0.02,
+      shortEdge: 480,
+    },
+    ...overrides,
+  };
+}
+
+function makeContext() {
+  return {
+    workingDir: "/tmp",
+    conversationId: "conv-1",
+    trustClass: "guardian" as const,
+    onOutput: () => {},
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("extract_keyframes tool", () => {
+  beforeEach(() => {
+    lastPreprocessOptions = undefined;
+    mockPreprocessError = null;
+    mockManifest = makeManifest();
+  });
+
+  test("returns error when asset_id is missing", async () => {
+    const result = await run({}, makeContext());
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toBe("asset_id is required.");
+  });
+
+  test("passes include_audio through to preprocessForAsset", async () => {
+    await run({ asset_id: "asset-1", include_audio: true }, makeContext());
+
+    expect(lastPreprocessOptions?.includeAudio).toBe(true);
+  });
+
+  test("defaults include_audio to false when not provided", async () => {
+    await run({ asset_id: "asset-1" }, makeContext());
+
+    expect(lastPreprocessOptions?.includeAudio).toBe(false);
+  });
+
+  test("maps all numeric option fields from tool input", async () => {
+    await run(
+      {
+        asset_id: "asset-1",
+        interval_seconds: 2,
+        segment_duration: 30,
+        dead_time_threshold: 0.05,
+        short_edge: 720,
+      },
+      makeContext(),
+    );
+
+    expect(lastPreprocessOptions?.intervalSeconds).toBe(2);
+    expect(lastPreprocessOptions?.segmentDuration).toBe(30);
+    expect(lastPreprocessOptions?.deadTimeThreshold).toBe(0.05);
+    expect(lastPreprocessOptions?.shortEdge).toBe(720);
+  });
+
+  test("passes detect_dead_time and section_config through", async () => {
+    await run(
+      {
+        asset_id: "asset-1",
+        detect_dead_time: true,
+        section_config: "/tmp/sections.json",
+      },
+      makeContext(),
+    );
+
+    expect(lastPreprocessOptions?.detectDeadTime).toBe(true);
+    expect(lastPreprocessOptions?.sectionConfigPath).toBe("/tmp/sections.json");
+  });
+
+  test("does not pass transcriptionMode or openaiApiKey", async () => {
+    await run(
+      {
+        asset_id: "asset-1",
+        include_audio: true,
+        transcription_mode: "api",
+      },
+      makeContext(),
+    );
+
+    expect(lastPreprocessOptions).toBeDefined();
+    expect("transcriptionMode" in lastPreprocessOptions!).toBe(false);
+    expect("openaiApiKey" in lastPreprocessOptions!).toBe(false);
+  });
+
+  test("returns successful result with manifest summary", async () => {
+    const result = await run({ asset_id: "asset-1" }, makeContext());
+
+    expect(result.isError).toBe(false);
+    const parsed = JSON.parse(result.content);
+    expect(parsed.assetId).toBe("asset-1");
+    expect(parsed.segmentCount).toBe(1);
+    expect(parsed.keyframeCount).toBe(3);
+  });
+
+  test("returns error content for known preprocess failures", async () => {
+    mockPreprocessError = new Error("Media asset not found: asset-bad");
+
+    const result = await run({ asset_id: "asset-bad" }, makeContext());
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toBe("Media asset not found: asset-bad");
+  });
+
+  test("wraps unknown errors in generic message", async () => {
+    mockPreprocessError = new Error("Something unexpected");
+
+    const result = await run({ asset_id: "asset-1" }, makeContext());
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toBe("Preprocess failed: Something unexpected");
+  });
+});