@vellumai/assistant 0.3.18 → 0.3.20

package/src/tools/browser/browser-manager.ts

@@ -10,6 +10,11 @@ import { checkBrowserRuntime } from './runtime-check.js';
 
 const log = getLogger('browser-manager');
 
+// Screencast capture dimensions — used by coordinate math across the browser module
+// to map between page coordinates and screencast-frame coordinates.
+export const SCREENCAST_WIDTH = 800;
+export const SCREENCAST_HEIGHT = 600;
+
 function getDownloadsDir(): string {
   const dir = join(getDataDir(), 'browser-downloads');
   mkdirSync(dir, { recursive: true });
@@ -20,6 +25,7 @@ export type DownloadInfo = { path: string; filename: string };
 
 type BrowserContext = {
   newPage(): Promise<Page>;
+  pages?(): Page[];
   close(): Promise<void>;
 };
 
@@ -253,32 +259,11 @@ class BrowserManager {
           }
         }
 
-        // If a client is connected, launch headed Chromium (minimized) so the user
-        // can interact directly when handoff triggers (e.g. CAPTCHAs).
-        // The window stays offscreen until bringToFront() is called during handoff.
-        const hasSender = !!(invokingSessionId && this.sessionSenders.get(invokingSessionId));
-        if (hasSender && this._browserMode === 'headless') {
-          try {
-            const pw2 = await import('playwright');
-            const headedBrowser = await pw2.chromium.launch({
-              channel: 'chrome',
-              headless: false,
-              args: [
-                '--window-position=-32000,-32000',
-                '--window-size=1,1',
-                '--disable-blink-features=AutomationControlled',
-              ],
-            });
-            const ctx = headedBrowser.contexts()[0] || await headedBrowser.newContext();
-            this.cdpBrowser = headedBrowser as unknown as typeof this.cdpBrowser;
-            this._browserLaunched = true;
-            this.setBrowserMode('cdp');
-            await this.initBrowserCdpSession();
-            log.info('Launched headed Chromium (minimized) for interactive handoff support');
-            return ctx as unknown as BrowserContext;
-          } catch (err2) {
-            log.warn({ err: err2 }, 'Headed Chromium launch failed, falling back to headless');
-          }
+        if (invokingSessionId && this.sessionSenders.get(invokingSessionId) && this._browserMode === 'headless') {
+          log.info(
+            { sessionId: invokingSessionId },
+            'CDP unavailable/declined; staying in headless mode (no visible browser window will be auto-launched)',
+          );
         }
       }
 
@@ -380,7 +365,26 @@ class BrowserManager {
     this.snapshotMaps.delete(sessionId);
     await this.stopScreencast(sessionId);
 
-    const page = await context.newPage();
+    let page: Page | undefined;
+
+    // In connectOverCDP mode, Chrome often starts with a pre-opened blank tab.
+    // Only reuse blank/new-tab pages to avoid hijacking active user tabs, which
+    // could cause user-visible disruption or data loss when the session closes.
+    if (this._browserMode === 'cdp' && !this._browserLaunched && typeof context.pages === 'function') {
+      const BLANK_TAB_URLS = new Set(['about:blank', 'chrome://newtab/', 'chrome://new-tab-page/']);
+      const claimedPages = new Set(this.pages.values());
+      const reusable = context.pages().find((p) => {
+        if (p.isClosed() || claimedPages.has(p)) return false;
+        const url = p.url();
+        return BLANK_TAB_URLS.has(url) || url === '';
+      });
+      if (reusable) {
+        page = reusable;
+        log.debug({ sessionId, url: reusable.url() }, 'Reusing blank CDP tab instead of creating a new page');
+      }
+    }
+
+    page ??= await context.newPage();
     this.pages.set(sessionId, page);
     this.rawPages.set(sessionId, page);
 
@@ -509,17 +513,27 @@ class BrowserManager {
     this.cdpSessions.set(sessionId, cdp);
     this.screencastCallbacks.set(sessionId, onFrame);
 
+    // Keep screencast intentionally low-frequency to avoid Chrome renderer /
+    // WindowServer spikes while users type in interactive auth flows.
+    const MIN_FRAME_INTERVAL_MS = 1000;
+    let lastFrameTime = 0;
+
     cdp.on('Page.screencastFrame', (params) => {
-      onFrame({ data: params.data as string, metadata: params.metadata as ScreencastFrameMetadata });
+      const now = Date.now();
+      if (now - lastFrameTime >= MIN_FRAME_INTERVAL_MS) {
+        lastFrameTime = now;
+        onFrame({ data: params.data as string, metadata: params.metadata as ScreencastFrameMetadata });
+      }
+      // Always ack so CDP continues delivering frames (otherwise it stalls)
       silentlyWithLog(cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId }), 'screencast frame ack');
     });
 
     await cdp.send('Page.startScreencast', {
       format: 'jpeg',
-      quality: 60,
-      maxWidth: 1280,
-      maxHeight: 960,
-      everyNthFrame: 1,
+      quality: 30,
+      maxWidth: SCREENCAST_WIDTH,
+      maxHeight: SCREENCAST_HEIGHT,
+      everyNthFrame: 4,
     });
   }
 

package/src/tools/browser/browser-screencast.ts

@@ -1,7 +1,7 @@
 import { v4 as uuid } from 'uuid';
 
 import type { BrowserFrame, BrowserViewSurfaceData, ServerMessage } from '../../daemon/ipc-contract.js';
-import { browserManager } from './browser-manager.js';
+import { browserManager, SCREENCAST_HEIGHT,SCREENCAST_WIDTH } from './browser-manager.js';
 
 // Track active screencast sessions
 const activeScreencasts = new Map<string, { surfaceId: string }>();
@@ -161,7 +161,7 @@ export async function getElementBounds(
       })()
     `) as { x: number; y: number; w: number; h: number; vw: number; vh: number } | null;
     if (!result) return null;
-    const scale = Math.min(1280 / result.vw, 960 / result.vh);
+    const scale = Math.min(SCREENCAST_WIDTH / result.vw, SCREENCAST_HEIGHT / result.vh);
     return {
       x: result.x * scale,
       y: result.y * scale,

package/src/tools/calls/call-start.ts

@@ -23,7 +23,7 @@ export async function executeCallStart(
       return {
         content: [
           'Error: A guardian voice verification call is already active for this number.',
-          'Use the guardian outbound verification flow (`/v1/integrations/guardian/outbound/start` or `/resend`) and wait for completion before using `call_start`.',
+          'Use the guardian outbound verification flow via the gateway API (`/v1/integrations/guardian/outbound/start` or `/resend`) and wait for completion before using `call_start`.',
         ].join(' '),
         isError: true,
       };

package/src/tools/executor.ts

@@ -58,7 +58,7 @@ export class ToolExecutor {
 
     // Run pre-execution approval gates (abort, parental controls, guardian
     // policy, allowed-tool-set, task-run preflight, tool registry lookup).
-    const gateResult = this.approvalHandler.checkPreExecutionGates(
+    const gateResult = await this.approvalHandler.checkPreExecutionGates(
       name, input, context, executionTarget, riskLevel, startTime,
       (event) => emitLifecycleEvent(context, event),
     );
@@ -70,24 +70,29 @@ export class ToolExecutor {
     const tool = gateResult.tool;
 
     try {
-      // Check permissions via the extracted PermissionChecker
-      const permResult = await this.permissionChecker.checkPermission(
-        name,
-        input,
-        tool,
-        context,
-        executionTarget,
-        (event) => emitLifecycleEvent(context, event),
-        sanitizeToolInput,
-        startTime,
-        computePreviewDiff,
-      );
+      // A consumed scoped grant is a complete authorization — skip the
+      // interactive permission/prompt flow so non-interactive sessions
+      // don't auto-deny prompt-gated tools and burn the one-time grant.
+      if (!gateResult.grantConsumed) {
+        // Check permissions via the extracted PermissionChecker
+        const permResult = await this.permissionChecker.checkPermission(
+          name,
+          input,
+          tool,
+          context,
+          executionTarget,
+          (event) => emitLifecycleEvent(context, event),
+          sanitizeToolInput,
+          startTime,
+          computePreviewDiff,
+        );
 
-      riskLevel = permResult.riskLevel;
-      decision = permResult.decision;
+        riskLevel = permResult.riskLevel;
+        decision = permResult.decision;
 
-      if (!permResult.allowed) {
-        return { content: permResult.content, isError: true };
+        if (!permResult.allowed) {
+          return { content: permResult.content, isError: true };
+        }
       }
 
       const hookResult = await getHookManager().trigger('pre-tool-execute', {

package/src/tools/network/script-proxy/session-manager.ts

@@ -373,7 +373,6 @@ export async function stopSession(sessionId: ProxySessionId): Promise<void> {
  */
 export function getSessionEnv(
   sessionId: ProxySessionId,
-  options?: { dockerMode?: boolean },
 ): ProxyEnvVars {
   const managed = sessions.get(sessionId);
   if (!managed) throw new Error(`Session not found: ${sessionId}`);
@@ -384,10 +383,7 @@ export function getSessionEnv(
   // Touch the idle timer on access
   resetIdleTimer(managed);
 
-  // Inside Docker, 127.0.0.1 is the container's own loopback — use
-  // host.docker.internal so traffic reaches the host-side proxy.
-  const host = options?.dockerMode ? 'host.docker.internal' : '127.0.0.1';
-  const proxyUrl = `http://${host}:${managed.session.port}`;
+  const proxyUrl = `http://127.0.0.1:${managed.session.port}`;
   const env: ProxyEnvVars = {
     HTTP_PROXY: proxyUrl,
     HTTPS_PROXY: proxyUrl,

package/src/tools/skills/load.ts

@@ -1,3 +1,6 @@
+import { isAssistantFeatureFlagEnabled } from '../../config/assistant-feature-flags.js';
+import { getConfig } from '../../config/loader.js';
+import { skillFlagKey } from '../../config/skill-state.js';
 import type { SkillSummary } from '../../config/skills.js';
 import { loadSkillBySelector, loadSkillCatalog } from '../../config/skills.js';
 import { RiskLevel } from '../../permissions/types.js';
@@ -46,6 +49,15 @@ export class SkillLoadTool implements Tool {
 
     const skill = loaded.skill;
 
+    // Assistant feature flag gate: reject loading if the skill's flag is OFF
+    const config = getConfig();
+    if (!isAssistantFeatureFlagEnabled(skillFlagKey(skill.id), config)) {
+      return {
+        content: `Error: skill "${skill.id}" is currently unavailable (disabled by feature flag)`,
+        isError: true,
+      };
+    }
+
     // Load catalog for include validation and child metadata output
     let catalogIndex: Map<string, SkillSummary> | undefined;
     if (skill.includes && skill.includes.length > 0) {
@@ -83,14 +95,15 @@ export class SkillLoadTool implements Tool {
       const childLines: string[] = [];
       for (const childId of skill.includes) {
         const child = catalogIndex.get(childId);
-        if (child) {
-          childLines.push(`  - ${child.id}: ${child.name} — ${child.description} (${child.skillFilePath})`);
-
-          // Load the included skill's body content
-          const childLoaded = loadSkillBySelector(childId);
-          if (childLoaded.skill && childLoaded.skill.body.length > 0) {
-            includedBodies.push(`--- Included Skill: ${childLoaded.skill.name} (${childId}) ---\n${childLoaded.skill.body}`);
-          }
+        if (!child) continue;
+        if (!isAssistantFeatureFlagEnabled(skillFlagKey(childId), config)) continue;
+
+        childLines.push(`  - ${child.id}: ${child.name} — ${child.description} (${child.skillFilePath})`);
+
+        // Load the included skill's body content
+        const childLoaded = loadSkillBySelector(childId);
+        if (childLoaded.skill && childLoaded.skill.body.length > 0) {
+          includedBodies.push(`--- Included Skill: ${childLoaded.skill.name} (${childId}) ---\n${childLoaded.skill.body}`);
         }
       }
       immediateChildrenSection = `Included Skills (immediate):\n${childLines.join('\n')}`;
@@ -113,6 +126,7 @@ export class SkillLoadTool implements Tool {
       for (const childId of skill.includes) {
         const child = catalogIndex.get(childId);
         if (!child) continue;
+        if (!isAssistantFeatureFlagEnabled(skillFlagKey(childId), config)) continue;
         let childHash: string | undefined;
         try {
           childHash = computeSkillVersionHash(child.directoryPath);

package/src/tools/system/avatar-generator.ts

@@ -0,0 +1,119 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+import { getConfig } from '../../config/loader.js';
+import { generateImage, mapGeminiError } from '../../media/gemini-image-service.js';
+import { RiskLevel } from '../../permissions/types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import { getLogger } from '../../util/logger.js';
+import { getWorkspaceDir } from '../../util/platform.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+
+const log = getLogger('avatar-generator');
+
+const TOOL_NAME = 'set_avatar';
+
+/** Canonical path where the custom avatar PNG is stored. */
+function getAvatarPath(): string {
+  return join(getWorkspaceDir(), 'data', 'avatar', 'custom-avatar.png');
+}
+
+export const setAvatarTool: Tool = {
+  name: TOOL_NAME,
+  description:
+    'Generate a custom avatar image from a text description. ' +
+    'Saves the result as the assistant\'s avatar.',
+  category: 'system',
+  defaultRiskLevel: RiskLevel.Low,
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: TOOL_NAME,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          description: {
+            type: 'string',
+            description:
+              'A text description of the desired avatar appearance, ' +
+              'e.g. "a friendly purple cat with green eyes wearing a tiny hat".',
+          },
+        },
+        required: ['description'],
+      },
+    };
+  },
+
+  async execute(
+    input: Record<string, unknown>,
+    _context: ToolContext,
+  ): Promise<ToolExecutionResult> {
+    const description = input.description;
+    if (typeof description !== 'string' || description.trim() === '') {
+      return {
+        content: 'Error: description is required and must be a non-empty string.',
+        isError: true,
+      };
+    }
+
+    const config = getConfig();
+    const apiKey = config.apiKeys.gemini ?? process.env.GEMINI_API_KEY;
+    if (!apiKey) {
+      return {
+        content: 'No Gemini API key configured. Please add your Gemini API key in Settings → Models & Services, or set the GEMINI_API_KEY environment variable.',
+        isError: true,
+      };
+    }
+
+    try {
+      log.info({ description: description.trim() }, 'Generating avatar via Gemini');
+
+      const prompt =
+        `Create an avatar image based on this description: ${description.trim()}\n\n` +
+        'Style: cute, friendly, work-safe illustration. ' +
+        'Vibrant but soft colors. Simple and recognizable at small sizes (28px). ' +
+        'Circular or rounded composition filling the canvas. ' +
+        'Subtle background color (not white or transparent).';
+
+      const result = await generateImage(apiKey, {
+        prompt,
+        mode: 'generate',
+        model: config.imageGenModel,
+      });
+
+      if (result.images.length === 0) {
+        return {
+          content: 'Error: Gemini returned no image data. Please try again.',
+          isError: true,
+        };
+      }
+
+      const image = result.images[0];
+      const pngBuffer = Buffer.from(image.dataBase64, 'base64');
+
+      const avatarPath = getAvatarPath();
+      const avatarDir = dirname(avatarPath);
+
+      mkdirSync(avatarDir, { recursive: true });
+      writeFileSync(avatarPath, pngBuffer);
+
+      log.info({ avatarPath }, 'Avatar saved successfully');
+
+      // Side-effect hook in tool-side-effects.ts broadcasts avatar_updated to all clients.
+
+      return {
+        content: 'Avatar updated! Your new avatar will appear shortly.',
+        isError: false,
+      };
+    } catch (error) {
+      const message = mapGeminiError(error);
+      log.error({ error: message }, 'Avatar generation failed');
+
+      return {
+        content: `Avatar generation failed: ${message}`,
+        isError: true,
+      };
+    }
+  },
+};

package/src/tools/system/navigate-settings.ts

@@ -0,0 +1,65 @@
+import { RiskLevel } from '../../permissions/types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+
+const SETTINGS_TABS = [
+  'Voice',
+  'Connect',
+  'Trust',
+  'Model',
+  'Scheduling',
+  'Parental',
+] as const;
+
+type SettingsTab = (typeof SETTINGS_TABS)[number];
+
+export class NavigateSettingsTabTool implements Tool {
+  name = 'navigate_settings_tab';
+  description =
+    'Open the Vellum settings panel to a specific tab (e.g. Voice, Connect, Trust). ' +
+    'Use this when the user needs to review or adjust settings visually.';
+  category = 'system';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          tab: {
+            type: 'string',
+            enum: [...SETTINGS_TABS],
+            description: 'The settings tab to navigate to',
+          },
+        },
+        required: ['tab'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    const tab = input.tab as string;
+    if (!SETTINGS_TABS.includes(tab as SettingsTab)) {
+      return {
+        content: `Error: unknown tab "${tab}". Valid tabs: ${SETTINGS_TABS.join(', ')}`,
+        isError: true,
+      };
+    }
+
+    if (context.sendToClient) {
+      context.sendToClient({
+        type: 'navigate_settings',
+        tab,
+      });
+    }
+
+    return {
+      content: `Opened settings to the ${tab} tab.`,
+      isError: false,
+    };
+  }
+}
+
+export const navigateSettingsTabTool = new NavigateSettingsTabTool();

package/src/tools/system/open-system-settings.ts

@@ -0,0 +1,75 @@
+import { RiskLevel } from '../../permissions/types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+
+const PANES = {
+  microphone: {
+    url: 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone',
+    label: 'Microphone privacy',
+    instruction: 'Please toggle Vellum Assistant on.',
+  },
+  speech_recognition: {
+    url: 'x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition',
+    label: 'Speech Recognition privacy',
+    instruction: 'Please toggle Vellum Assistant on.',
+  },
+} as const;
+
+type PaneName = keyof typeof PANES;
+
+const VALID_PANES = Object.keys(PANES) as PaneName[];
+
+export class OpenSystemSettingsTool implements Tool {
+  name = 'open_system_settings';
+  description =
+    'Open a specific macOS System Settings pane (e.g. Microphone or Speech Recognition privacy). ' +
+    'Use this to guide the user through granting permissions that can only be toggled in System Settings.';
+  category = 'system';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          pane: {
+            type: 'string',
+            enum: [...VALID_PANES],
+            description: 'The System Settings pane to open',
+          },
+        },
+        required: ['pane'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    const pane = input.pane as string;
+    if (!VALID_PANES.includes(pane as PaneName)) {
+      return {
+        content: `Error: unknown pane "${pane}". Valid panes: ${VALID_PANES.join(', ')}`,
+        isError: true,
+      };
+    }
+
+    const meta = PANES[pane as PaneName];
+
+    // Send open_url IPC to the client — the x-apple.systempreferences: scheme
+    // opens System Settings directly without a browser confirmation dialog.
+    if (context.sendToClient) {
+      context.sendToClient({
+        type: 'open_url',
+        url: meta.url,
+      });
+    }
+
+    return {
+      content: `Opened System Settings to ${meta.label}. ${meta.instruction}`,
+      isError: false,
+    };
+  }
+}
+
+export const openSystemSettingsTool = new OpenSystemSettingsTool();