@vellumai/assistant 0.3.0

package/src/tools/computer-use/definitions.ts

@@ -0,0 +1,443 @@
+/**
+ * Computer-use tool definitions.
+ *
+ * These tools mirror the macOS client's ToolDefinitions.swift schemas, prefixed
+ * with `computer_use_` to avoid collisions with existing daemon tools.  They are all
+ * proxy tools — execution is forwarded to a connected macOS client and never
+ * handled locally by the daemon.
+ */
+
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function proxyExecute(): Promise<ToolExecutionResult> {
+  throw new Error('Proxy tool: execution must be forwarded to the connected client');
+}
+
+function makeClickTool(name: string, verb: string): Tool {
+  return {
+    name,
+    description: `${verb} on a UI element by its [ID] from the accessibility tree, or at raw screen coordinates as fallback.`,
+    category: 'computer-use',
+    defaultRiskLevel: RiskLevel.Low,
+    executionMode: 'proxy',
+
+    getDefinition(): ToolDefinition {
+      return {
+        name: this.name,
+        description: this.description,
+        input_schema: {
+          type: 'object',
+          properties: {
+            element_id: {
+              type: 'integer',
+              description: 'The [ID] number of the element from the accessibility tree (preferred)',
+            },
+            x: {
+              type: 'integer',
+              description: 'X coordinate on screen (fallback when no element_id)',
+            },
+            y: {
+              type: 'integer',
+              description: 'Y coordinate on screen (fallback when no element_id)',
+            },
+            reasoning: {
+              type: 'string',
+              description: `Explanation of what you see and why you are ${verb.toLowerCase()}ing here`,
+            },
+          },
+          required: ['reasoning'],
+        },
+      };
+    },
+
+    execute: proxyExecute,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Click variants
+// ---------------------------------------------------------------------------
+
+export const computerUseClickTool = makeClickTool('computer_use_click', 'Click');
+export const computerUseDoubleClickTool = makeClickTool('computer_use_double_click', 'Double-click');
+export const computerUseRightClickTool = makeClickTool('computer_use_right_click', 'Right-click');
+
+// ---------------------------------------------------------------------------
+// type_text
+// ---------------------------------------------------------------------------
+
+export const computerUseTypeTextTool: Tool = {
+  name: 'computer_use_type_text',
+  description: 'Type text at the current cursor position. The target field must already be focused (click it first).',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          text: {
+            type: 'string',
+            description: 'The text to type',
+          },
+          reasoning: {
+            type: 'string',
+            description: 'Explanation of what you are typing and why',
+          },
+        },
+        required: ['text', 'reasoning'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// key
+// ---------------------------------------------------------------------------
+
+export const computerUseKeyTool: Tool = {
+  name: 'computer_use_key',
+  description: 'Press a key or keyboard shortcut. Supported: enter, tab, escape, backspace, delete, up, down, left, right, space, cmd+a, cmd+c, cmd+v, cmd+z, cmd+tab, cmd+w, shift+tab, option+tab',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          key: {
+            type: 'string',
+            description: 'Key or shortcut to press (e.g. enter, tab, cmd+c, cmd+v)',
+          },
+          reasoning: {
+            type: 'string',
+            description: 'Explanation of why you are pressing this key',
+          },
+        },
+        required: ['key', 'reasoning'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// scroll
+// ---------------------------------------------------------------------------
+
+export const computerUseScrollTool: Tool = {
+  name: 'computer_use_scroll',
+  description: 'Scroll within an element by its [ID], or at raw screen coordinates as fallback.',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          element_id: {
+            type: 'integer',
+            description: 'The [ID] number of the element to scroll within (preferred)',
+          },
+          x: {
+            type: 'integer',
+            description: 'X coordinate on screen (fallback when no element_id)',
+          },
+          y: {
+            type: 'integer',
+            description: 'Y coordinate on screen (fallback when no element_id)',
+          },
+          direction: {
+            type: 'string',
+            enum: ['up', 'down', 'left', 'right'],
+            description: 'Scroll direction',
+          },
+          amount: {
+            type: 'integer',
+            description: 'Scroll amount (1-10)',
+          },
+          reasoning: {
+            type: 'string',
+            description: 'Explanation of why you are scrolling',
+          },
+        },
+        required: ['direction', 'amount', 'reasoning'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// drag
+// ---------------------------------------------------------------------------
+
+export const computerUseDragTool: Tool = {
+  name: 'computer_use_drag',
+  description: 'Drag from one element or position to another. Use for moving files, resizing windows, rearranging items, or adjusting sliders.',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          element_id: {
+            type: 'integer',
+            description: 'The [ID] of the source element to drag from (preferred)',
+          },
+          x: {
+            type: 'integer',
+            description: 'Source X coordinate (fallback when no element_id)',
+          },
+          y: {
+            type: 'integer',
+            description: 'Source Y coordinate (fallback when no element_id)',
+          },
+          to_element_id: {
+            type: 'integer',
+            description: 'The [ID] of the destination element to drag to (preferred)',
+          },
+          to_x: {
+            type: 'integer',
+            description: 'Destination X coordinate (fallback when no to_element_id)',
+          },
+          to_y: {
+            type: 'integer',
+            description: 'Destination Y coordinate (fallback when no to_element_id)',
+          },
+          reasoning: {
+            type: 'string',
+            description: 'Explanation of what you are dragging and why',
+          },
+        },
+        required: ['reasoning'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// wait
+// ---------------------------------------------------------------------------
+
+export const computerUseWaitTool: Tool = {
+  name: 'computer_use_wait',
+  description: 'Wait for the UI to update',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          duration_ms: {
+            type: 'integer',
+            description: 'Milliseconds to wait',
+          },
+          reasoning: {
+            type: 'string',
+            description: 'Explanation of what you are waiting for',
+          },
+        },
+        required: ['duration_ms', 'reasoning'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// open_app
+// ---------------------------------------------------------------------------
+
+export const computerUseOpenAppTool: Tool = {
+  name: 'computer_use_open_app',
+  description: 'Open or switch to a macOS application by name. Preferred over cmd+tab for switching apps — more reliable and explicit.',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          app_name: {
+            type: 'string',
+            description: 'The name of the application to open (e.g. "Slack", "Safari", "Google Chrome", "VS Code")',
+          },
+          reasoning: {
+            type: 'string',
+            description: 'Explanation of why you need to open or switch to this app',
+          },
+        },
+        required: ['app_name', 'reasoning'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// run_applescript
+// ---------------------------------------------------------------------------
+
+export const computerUseRunAppleScriptTool: Tool = {
+  name: 'computer_use_run_applescript',
+  description:
+    'Execute an AppleScript to control applications via Apple\'s scripting bridge. ' +
+    'Use this for operations that are more reliable through scripting than UI interaction: ' +
+    'setting a browser URL directly, navigating Finder to a path, querying app state ' +
+    '(tab count, window titles, document status), or clicking deeply nested menu items. ' +
+    'The script\'s return value (if any) will be reported back. ' +
+    'NEVER use "do shell script" — it is blocked for security. ' +
+    'Keep scripts short and targeted to a single operation.',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          script: {
+            type: 'string',
+            description: 'The AppleScript source code to execute',
+          },
+          reasoning: {
+            type: 'string',
+            description: 'Explanation of what this script does and why AppleScript is better than UI interaction for this step',
+          },
+        },
+        required: ['script', 'reasoning'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// done
+// ---------------------------------------------------------------------------
+
+export const computerUseDoneTool: Tool = {
+  name: 'computer_use_done',
+  description: 'Task is complete',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          summary: {
+            type: 'string',
+            description: 'Human-readable summary of what was accomplished',
+          },
+        },
+        required: ['summary'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// respond
+// ---------------------------------------------------------------------------
+
+export const computerUseRespondTool: Tool = {
+  name: 'computer_use_respond',
+  description: 'Respond directly to the user with a text answer. Use this when the user is asking a question (about their schedule, meetings, calendar, etc.) rather than asking you to control the computer.',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          answer: {
+            type: 'string',
+            description: 'The text answer to display to the user',
+          },
+          reasoning: {
+            type: 'string',
+            description: 'Explanation of how you determined the answer',
+          },
+        },
+        required: ['answer', 'reasoning'],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
+// ---------------------------------------------------------------------------
+// All tools exported as array for convenience
+// ---------------------------------------------------------------------------
+
+export const allComputerUseTools: Tool[] = [
+  computerUseClickTool,
+  computerUseDoubleClickTool,
+  computerUseRightClickTool,
+  computerUseTypeTextTool,
+  computerUseKeyTool,
+  computerUseScrollTool,
+  computerUseDragTool,
+  computerUseWaitTool,
+  computerUseOpenAppTool,
+  computerUseRunAppleScriptTool,
+  computerUseDoneTool,
+  computerUseRespondTool,
+];

package/src/tools/computer-use/registry.ts

@@ -0,0 +1,22 @@
+/**
+ * Registers computer-use tools with the daemon's tool registry.
+ *
+ * The 12 computer_use_* action tools and the computer_use_request_control
+ * escalation tool are now provided by the bundled computer-use skill.
+ * This module retains registerComputerUseActionTools() for backward
+ * compatibility (used by tests), but it is no longer called during
+ * normal startup.
+ */
+
+import { registerTool } from '../registry.js';
+import { allComputerUseTools } from './definitions.js';
+
+/**
+ * Register the 12 `computer_use_*` action proxy tools.
+ * After cutover these are provided by the bundled computer-use skill instead.
+ */
+export function registerComputerUseActionTools(): void {
+  for (const tool of allComputerUseTools) {
+    registerTool(tool);
+  }
+}

package/src/tools/computer-use/request-computer-control.ts

@@ -0,0 +1,53 @@
+/**
+ * computer_use_request_control tool definition.
+ *
+ * This tool allows a text_qa session to escalate to foreground computer use
+ * when the user explicitly requests it (e.g. "go ahead and do it", "take over
+ * for 20 seconds"). It is a proxy tool — execution is handled by the session's
+ * surfaceProxyResolver, which creates a CU session and sends a task_routed
+ * message to the client.
+ *
+ * This tool is only available to text_qa sessions. It must NOT be added to
+ * CU sessions (that would be recursive).
+ *
+ * Part of the bundled computer-use skill. The definition here is imported by
+ * buildToolDefinitions() so text_qa sessions can include it without
+ * preactivating the entire skill.
+ */
+
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+
+export const requestComputerControlTool: Tool = {
+  name: 'computer_use_request_control',
+  description:
+    'Escalate to foreground computer use. Call this when the user explicitly asks you to ' +
+    'take control of their computer to perform a task (e.g. "go ahead and do it", ' +
+    '"take over", "open that for me"). Provide a concise description of the task ' +
+    'that computer use should accomplish.',
+  category: 'computer-use',
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: 'proxy',
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          task: {
+            type: 'string',
+            description: 'Concise description of what computer use should accomplish',
+          },
+        },
+        required: ['task'],
+      },
+    };
+  },
+
+  execute(): Promise<ToolExecutionResult> {
+    throw new Error('Proxy tool: execution must be forwarded via surfaceProxyResolver');
+  },
+};

package/src/tools/computer-use/skill-proxy-bridge.ts

@@ -0,0 +1,28 @@
+/**
+ * Shared helper for computer-use skill wrapper scripts.
+ *
+ * Each wrapper calls forwardComputerUseProxyTool() to delegate execution to
+ * the proxy resolver, which forwards the call to the connected macOS client.
+ */
+
+import type { ToolContext, ToolExecutionResult } from '../types.js';
+
+/**
+ * Forward a computer-use proxy tool call through the context's proxyToolResolver.
+ *
+ * Returns a clear error result if the resolver is missing (e.g. when the tool
+ * is invoked outside a session with a connected client).
+ */
+export function forwardComputerUseProxyTool(
+  toolName: string,
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  if (!context.proxyToolResolver) {
+    return Promise.resolve({
+      content: `Cannot execute ${toolName}: no proxy resolver available. This tool requires a connected macOS client.`,
+      isError: true,
+    });
+  }
+  return context.proxyToolResolver(toolName, input);
+}

package/src/tools/credentials/account-registry.ts

@@ -0,0 +1,127 @@
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import {
+  createAccount,
+  listAccounts,
+  getAccount,
+  updateAccount,
+} from '../../memory/account-store.js';
+
+class AccountManageTool implements Tool {
+  name = 'account_manage';
+  description = 'Create, list, get, or update account records';
+  category = 'credentials';
+  defaultRiskLevel = RiskLevel.Low;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          action: {
+            type: 'string',
+            enum: ['create', 'list', 'get', 'update'],
+            description: 'CRUD operation',
+          },
+          id: {
+            type: 'string',
+            description: 'Account ID (for get/update)',
+          },
+          service: {
+            type: 'string',
+            description: 'Service name',
+          },
+          username: { type: 'string' },
+          email: { type: 'string' },
+          display_name: { type: 'string' },
+          status: {
+            type: 'string',
+            enum: ['active', 'pending_verification', 'suspended'],
+          },
+          credential_ref: {
+            type: 'string',
+            description: 'Service name linking to credential vault',
+          },
+          metadata: { type: 'object' },
+        },
+        required: ['action'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, _context: ToolContext): Promise<ToolExecutionResult> {
+    const action = input.action as string;
+
+    switch (action) {
+      case 'create': {
+        const service = input.service as string | undefined;
+        if (!service || typeof service !== 'string') {
+          return { content: 'Error: service is required for create action', isError: true };
+        }
+
+        const record = createAccount({
+          service,
+          username: input.username as string | undefined,
+          email: input.email as string | undefined,
+          displayName: input.display_name as string | undefined,
+          status: input.status as string | undefined,
+          credentialRef: input.credential_ref as string | undefined,
+          metadata: input.metadata as Record<string, unknown> | undefined,
+        });
+
+        return { content: JSON.stringify(record, null, 2), isError: false };
+      }
+
+      case 'list': {
+        const records = listAccounts({
+          service: input.service as string | undefined,
+          status: input.status as string | undefined,
+        });
+        return { content: JSON.stringify(records, null, 2), isError: false };
+      }
+
+      case 'get': {
+        const id = input.id as string | undefined;
+        if (!id || typeof id !== 'string') {
+          return { content: 'Error: id is required for get action', isError: true };
+        }
+
+        const record = getAccount(id);
+        if (!record) {
+          return { content: `Error: account not found: ${id}`, isError: true };
+        }
+        return { content: JSON.stringify(record, null, 2), isError: false };
+      }
+
+      case 'update': {
+        const id = input.id as string | undefined;
+        if (!id || typeof id !== 'string') {
+          return { content: 'Error: id is required for update action', isError: true };
+        }
+
+        const updated = updateAccount(id, {
+          service: input.service as string | undefined,
+          username: input.username as string | undefined,
+          email: input.email as string | undefined,
+          displayName: input.display_name as string | undefined,
+          status: input.status as string | undefined,
+          credentialRef: input.credential_ref as string | undefined,
+          metadata: input.metadata as Record<string, unknown> | undefined,
+        });
+
+        if (!updated) {
+          return { content: `Error: account not found: ${id}`, isError: true };
+        }
+        return { content: JSON.stringify(updated, null, 2), isError: false };
+      }
+
+      default:
+        return { content: `Error: unknown action "${action}"`, isError: true };
+    }
+  }
+}
+
+export const accountManageTool = new AccountManageTool();