@vellumai/assistant 0.4.48 → 0.4.49

package/src/tools/computer-use/definitions.ts

@@ -34,7 +34,7 @@ const reasonProperty = {
 export const computerUseClickTool: Tool = {
   name: "computer_use_click",
   description:
-    "Click on a UI element by its [ID] from the accessibility tree, or at raw screen coordinates as fallback. Supports single click, double-click, and right-click via the click_type parameter.",
+    "Click an element on screen. Prefer element_id (from the accessibility tree) over x/y coordinates.",
   category: "computer-use",
   defaultRiskLevel: RiskLevel.Low,
   executionMode: "proxy",
@@ -86,7 +86,7 @@ export const computerUseClickTool: Tool = {
 export const computerUseTypeTextTool: Tool = {
   name: "computer_use_type_text",
   description:
-    "Type text at the current cursor position. The target field must already be focused (click it first).",
+    "Type text at the current cursor position. First click a text field (by element_id) to focus it, then call this tool. If a field shows 'FOCUSED', skip the click.",
   category: "computer-use",
   defaultRiskLevel: RiskLevel.Low,
   executionMode: "proxy",
@@ -352,13 +352,7 @@ export const computerUseOpenAppTool: Tool = {
 export const computerUseRunAppleScriptTool: Tool = {
   name: "computer_use_run_applescript",
   description:
-    "Execute an AppleScript to control applications via Apple's scripting bridge. " +
-    "Use this for operations that are more reliable through scripting than UI interaction: " +
-    "setting a browser URL directly, navigating Finder to a path, querying app state " +
-    "(tab count, window titles, document status), or clicking deeply nested menu items. " +
-    "The script's return value (if any) will be reported back. " +
-    'NEVER use "do shell script" — it is blocked for security. ' +
-    "Keep scripts short and targeted to a single operation.",
+    "Run an AppleScript command. Prefer this over click/type when possible — it doesn't move the cursor or interrupt the user. Never use 'do shell script' inside AppleScript (blocked for security).",
   category: "computer-use",
   defaultRiskLevel: RiskLevel.Low,
   executionMode: "proxy",
@@ -395,7 +389,8 @@ export const computerUseRunAppleScriptTool: Tool = {
 
 export const computerUseDoneTool: Tool = {
   name: "computer_use_done",
-  description: "Task is complete",
+  description:
+    "Signal that the computer use task is complete. Provide a summary of what was accomplished. This ends the computer use session.",
   category: "computer-use",
   defaultRiskLevel: RiskLevel.Low,
   executionMode: "proxy",
@@ -428,7 +423,7 @@ export const computerUseDoneTool: Tool = {
 export const computerUseRespondTool: Tool = {
   name: "computer_use_respond",
   description:
-    "Respond directly to the user with a text answer. Use this when the user is asking a question (about their schedule, meetings, calendar, etc.) rather than asking you to control the computer.",
+    "Respond to the user with a text answer instead of performing computer actions. Use this when you can answer directly without interacting with the screen.",
   category: "computer-use",
   defaultRiskLevel: RiskLevel.Low,
   executionMode: "proxy",
@@ -458,11 +453,41 @@ export const computerUseRespondTool: Tool = {
   execute: proxyExecute,
 };
 
+// ---------------------------------------------------------------------------
+// observe
+// ---------------------------------------------------------------------------
+
+export const computerUseObserveTool: Tool = {
+  name: "computer_use_observe",
+  description:
+    "Capture the current screen state. Returns the accessibility tree with [ID] element references and optionally a screenshot.\n\nThe accessibility tree shows interactive elements like [3] AXButton 'Save' or [17] AXTextField 'Search'. Use element_id to target these elements in subsequent actions — this is much more reliable than pixel coordinates.\n\nCall this before your first computer use action, or to check screen state without acting.",
+  category: "computer-use",
+  defaultRiskLevel: RiskLevel.Low,
+  executionMode: "proxy",
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: "object",
+        properties: {
+          reason: reasonProperty,
+        },
+        required: ["reason"],
+      },
+    };
+  },
+
+  execute: proxyExecute,
+};
+
 // ---------------------------------------------------------------------------
 // All tools exported as array for convenience
 // ---------------------------------------------------------------------------
 
 export const allComputerUseTools: Tool[] = [
+  computerUseObserveTool,
   computerUseClickTool,
   computerUseTypeTextTool,
   computerUseKeyTool,
@@ -474,3 +499,13 @@ export const allComputerUseTools: Tool[] = [
   computerUseDoneTool,
   computerUseRespondTool,
 ];
+
+/**
+ * Tools safe for the legacy fallback path (no skill projection).
+ *
+ * Excludes `computer_use_observe` because the macOS client doesn't handle it
+ * in the legacy code path — it falls back to `.done` which skips sending an
+ * observation, causing the daemon to block on `pendingObservation` until timeout.
+ */
+export const legacyFallbackComputerUseTools: Tool[] =
+  allComputerUseTools.filter((t) => t.name !== "computer_use_observe");

package/src/tools/computer-use/registry.ts

@@ -1,11 +1,10 @@
 /**
  * Registers computer-use tools with the daemon's tool registry.
  *
- * The 12 computer_use_* action tools and the computer_use_request_control
- * escalation tool are now provided by the bundled computer-use skill.
- * This module retains registerComputerUseActionTools() for backward
- * compatibility (used by tests), but it is no longer called during
- * normal startup.
+ * The computer_use_* action tools are now provided by the bundled
+ * computer-use skill. This module retains registerComputerUseActionTools()
+ * for backward compatibility (used by tests), but it is no longer called
+ * during normal startup.
  */
 
 import { registerTool } from "../registry.js";

package/src/tools/credentials/broker.ts

@@ -1,6 +1,6 @@
 import { v4 as uuid } from "uuid";
 
-import { credentialKey, migrateKeys } from "../../security/credential-key.js";
+import { credentialKey } from "../../security/credential-key.js";
 import { getSecureKey } from "../../security/secure-keys.js";
 import { getLogger } from "../../util/logger.js";
 import type {
@@ -60,7 +60,6 @@ export class CredentialBroker {
    * Returns a single-use token on success, or a denial reason on failure.
    */
   authorize(request: AuthorizeRequest): AuthorizeResult {
-    migrateKeys();
     const metadata = getCredentialMetadata(request.service, request.field);
     if (!metadata) {
       return {

package/src/tools/credentials/metadata-store.ts

@@ -4,6 +4,11 @@
  * Persists non-secret metadata about credentials (policy, timestamps, IDs)
  * in a versioned JSON file under protected storage. Secret values remain
  * in the secure key backend only.
+ *
+ * OAuth-specific fields (expiresAt, grantedScopes, oauth2TokenUrl,
+ * oauth2ClientId, oauth2TokenEndpointAuthMethod, hasRefreshToken) are now
+ * exclusively managed by the SQLite oauth-store and have been removed
+ * from this interface as of v5.
  */
 
 import { randomUUID } from "node:crypto";
@@ -21,26 +26,16 @@ export interface CredentialMetadata {
   allowedTools: string[];
   allowedDomains: string[];
   usageDescription?: string;
-  expiresAt?: number;
-  grantedScopes?: string[];
-  /** OAuth2 token endpoint — enables autonomous token refresh without an IntegrationDefinition. */
-  oauth2TokenUrl?: string;
-  /** OAuth2 client ID — paired with oauth2TokenUrl for refresh. */
-  oauth2ClientId?: string;
-  /** How the client authenticates at the token endpoint (client_secret_basic or client_secret_post). */
-  oauth2TokenEndpointAuthMethod?: string;
   /** Human-friendly name for this credential (e.g. "fal-primary"). */
   alias?: string;
   /** Templates describing how to inject this credential into proxied requests. */
   injectionTemplates?: CredentialInjectionTemplate[];
-  /** Whether a refresh token exists in the secure store for this service. */
-  hasRefreshToken?: boolean;
   createdAt: number;
   updatedAt: number;
 }
 
 /** Current on-disk schema version. */
-const CURRENT_VERSION = 4;
+const CURRENT_VERSION = 5;
 
 interface MetadataFile {
   version: typeof CURRENT_VERSION;
@@ -88,9 +83,10 @@ function isValidCredentialRecord(
 }
 
 /**
- * Migrate a v1 record to v2 by backfilling new optional fields with defaults.
+ * Migrate any record to v5 by stripping OAuth-specific fields that are
+ * now exclusively managed by the SQLite oauth-store.
  */
-function migrateRecordV1toV2(
+function migrateRecordToV5(
   record: Record<string, unknown>,
 ): CredentialMetadata {
   return {
@@ -107,73 +103,15 @@ function migrateRecordV1toV2(
       typeof record.usageDescription === "string"
         ? record.usageDescription
         : undefined,
-    expiresAt:
-      typeof record.expiresAt === "number" ? record.expiresAt : undefined,
-    grantedScopes: Array.isArray(record.grantedScopes)
-      ? (record.grantedScopes as string[])
-      : undefined,
-    oauth2TokenUrl:
-      typeof record.oauth2TokenUrl === "string"
-        ? record.oauth2TokenUrl
-        : undefined,
-    oauth2ClientId:
-      typeof record.oauth2ClientId === "string"
-        ? record.oauth2ClientId
-        : undefined,
-    oauth2TokenEndpointAuthMethod:
-      typeof record.oauth2TokenEndpointAuthMethod === "string"
-        ? record.oauth2TokenEndpointAuthMethod
-        : undefined,
     alias: typeof record.alias === "string" ? record.alias : undefined,
     injectionTemplates: Array.isArray(record.injectionTemplates)
       ? (record.injectionTemplates as CredentialInjectionTemplate[])
       : undefined,
-    hasRefreshToken:
-      typeof record.hasRefreshToken === "boolean"
-        ? record.hasRefreshToken
-        : undefined,
     createdAt: record.createdAt as number,
     updatedAt: record.updatedAt as number,
   };
 }
 
-/**
- * Migrate a v2 record to v3 by stripping the oauth2ClientSecret field.
- * Client secrets are now read exclusively from the secure key store.
- */
-function migrateRecordV2toV3(record: CredentialMetadata): CredentialMetadata {
-  const { oauth2ClientSecret: _removed, ...rest } =
-    record as CredentialMetadata & { oauth2ClientSecret?: string };
-  return rest;
-}
-
-/**
- * Migrate v3 credentials to v4:
- * - Delete ghost `refresh_token` metadata records
- * - Set `hasRefreshToken: true` on corresponding `access_token` records
- */
-function migrateV3toV4(
-  credentials: CredentialMetadata[],
-): CredentialMetadata[] {
-  // Collect services that had refresh_token ghost records
-  const servicesWithRefresh = new Set<string>();
-  for (const c of credentials) {
-    if (c.field === "refresh_token") {
-      servicesWithRefresh.add(c.service);
-    }
-  }
-
-  // Remove all refresh_token records and set hasRefreshToken on access_token records
-  const filtered = credentials.filter((c) => c.field !== "refresh_token");
-  for (const c of filtered) {
-    if (c.field === "access_token" && servicesWithRefresh.has(c.service)) {
-      c.hasRefreshToken = true;
-    }
-  }
-
-  return filtered;
-}
-
 function loadFile(): LoadResult {
   const raw = readTextFileSync(getMetadataPath());
   if (raw == null) {
@@ -189,7 +127,8 @@ function loadFile(): LoadResult {
       fileVersion !== 1 &&
       fileVersion !== 2 &&
       fileVersion !== 3 &&
-      fileVersion !== 4
+      fileVersion !== 4 &&
+      fileVersion !== 5
     ) {
       // Unrecognized version (future, fractional, negative, zero) — refuse to touch it
       return { unknownVersion: true };
@@ -201,24 +140,12 @@ function loadFile(): LoadResult {
     const validRecords = rawCredentials.filter(isValidCredentialRecord);
 
     if (fileVersion < CURRENT_VERSION) {
-      // Apply migrations in sequence: v1→v2→v3→v4
-      let credentials: CredentialMetadata[];
-      if (fileVersion === 1) {
-        credentials = migrateV3toV4(
-          validRecords.map(migrateRecordV1toV2).map(migrateRecordV2toV3),
-        );
-      } else if (fileVersion === 2) {
-        credentials = migrateV3toV4(
-          (validRecords as unknown as CredentialMetadata[]).map(
-            migrateRecordV2toV3,
-          ),
-        );
-      } else {
-        // fileVersion === 3
-        credentials = migrateV3toV4(
-          validRecords as unknown as CredentialMetadata[],
-        );
-      }
+      // Migrate all older versions to v5 by stripping OAuth-specific fields
+      // and removing ghost refresh_token records
+      const filtered = validRecords.filter(
+        (r) => (r as Record<string, unknown>).field !== "refresh_token",
+      );
+      const credentials = filtered.map(migrateRecordToV5);
       const migrated: MetadataFile = { version: CURRENT_VERSION, credentials };
       try {
         saveFile(migrated);
@@ -272,17 +199,10 @@ export function upsertCredentialMetadata(
     allowedTools?: string[];
     allowedDomains?: string[];
     usageDescription?: string;
-    /** Pass `null` to explicitly clear a previously-set expiry. */
-    expiresAt?: number | null;
-    grantedScopes?: string[];
-    oauth2TokenUrl?: string;
-    oauth2ClientId?: string;
-    oauth2TokenEndpointAuthMethod?: string;
     /** Pass `null` to explicitly clear a previously-set alias. */
     alias?: string | null;
     /** Pass `null` to explicitly clear injection templates. */
     injectionTemplates?: CredentialInjectionTemplate[] | null;
-    hasRefreshToken?: boolean;
   },
 ): CredentialMetadata {
   const result = loadFile();
@@ -305,22 +225,6 @@ export function upsertCredentialMetadata(
       existing.allowedDomains = policy.allowedDomains;
     if (policy?.usageDescription !== undefined)
       existing.usageDescription = policy.usageDescription;
-    if (policy?.expiresAt !== undefined) {
-      if (policy.expiresAt == null) {
-        delete existing.expiresAt;
-      } else {
-        existing.expiresAt = policy.expiresAt;
-      }
-    }
-    if (policy?.grantedScopes !== undefined)
-      existing.grantedScopes = policy.grantedScopes;
-    if (policy?.oauth2TokenUrl !== undefined)
-      existing.oauth2TokenUrl = policy.oauth2TokenUrl;
-    if (policy?.oauth2ClientId !== undefined)
-      existing.oauth2ClientId = policy.oauth2ClientId;
-    if (policy?.oauth2TokenEndpointAuthMethod !== undefined)
-      existing.oauth2TokenEndpointAuthMethod =
-        policy.oauth2TokenEndpointAuthMethod;
     if (policy?.alias !== undefined) {
       if (policy.alias == null) {
         delete existing.alias;
@@ -335,8 +239,6 @@ export function upsertCredentialMetadata(
         existing.injectionTemplates = policy.injectionTemplates;
       }
     }
-    if (policy?.hasRefreshToken !== undefined)
-      existing.hasRefreshToken = policy.hasRefreshToken;
     existing.updatedAt = now;
     saveFile(data);
     return existing;
@@ -349,14 +251,8 @@ export function upsertCredentialMetadata(
     allowedTools: policy?.allowedTools ?? [],
     allowedDomains: policy?.allowedDomains ?? [],
     usageDescription: policy?.usageDescription,
-    expiresAt: policy?.expiresAt ?? undefined,
-    grantedScopes: policy?.grantedScopes,
-    oauth2TokenUrl: policy?.oauth2TokenUrl,
-    oauth2ClientId: policy?.oauth2ClientId,
-    oauth2TokenEndpointAuthMethod: policy?.oauth2TokenEndpointAuthMethod,
     alias: policy?.alias ?? undefined,
     injectionTemplates: policy?.injectionTemplates ?? undefined,
-    hasRefreshToken: policy?.hasRefreshToken,
     createdAt: now,
     updatedAt: now,
   };