@vellumai/assistant 0.7.1 → 0.7.3

package/src/config/bundled-skills/app-control/TOOLS.json

@@ -0,0 +1,299 @@
+{
+  "version": 1,
+  "tools": [
+    {
+      "name": "app_control_start",
+      "description": "Start (launch or focus) the target application. Optionally pass command-line arguments. Begins an app-control session targeting this app.",
+      "category": "app-control",
+      "risk": "medium",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Bundle ID (preferred, e.g. 'com.apple.Safari') or process name of the target application"
+          },
+          "args": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "Optional command-line arguments to launch the app with"
+          },
+          "reasoning": {
+            "type": "string",
+            "description": "Explanation of why you are starting this app"
+          }
+        },
+        "required": ["app", "reasoning"]
+      },
+      "executor": "tools/app-control-start.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "app_control_observe",
+      "description": "Capture the current window state of the target application — returns lifecycle state (running/missing/minimized), an optional screenshot, and window bounds. Use this before issuing input actions, or to check progress without acting.",
+      "category": "app-control",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Bundle ID (preferred, e.g. 'com.apple.Safari') or process name of the target application"
+          },
+          "settle_ms": {
+            "type": "integer",
+            "description": "Milliseconds to wait before capturing so the target app and the WindowServer can flush pending input + composite a fresh frame. Default ~200ms (sized for emulator-class apps at 60fps). Pass 0 for static UIs; raise for slow-feedback apps."
+          }
+        },
+        "required": ["app"]
+      },
+      "executor": "tools/app-control-observe.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "app_control_press",
+      "description": "Press a single key in the target application, with optional modifiers (cmd/shift/option/ctrl) and a hold duration in milliseconds.",
+      "category": "app-control",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Bundle ID (preferred, e.g. 'com.apple.Safari') or process name of the target application"
+          },
+          "key": {
+            "type": "string",
+            "description": "Key identifier, e.g. \"return\", \"a\", \"f12\""
+          },
+          "modifiers": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "Modifier list, e.g. [\"cmd\", \"shift\"]. Omit for no modifiers."
+          },
+          "duration_ms": {
+            "type": "integer",
+            "description": "Hold duration in milliseconds"
+          },
+          "reasoning": {
+            "type": "string",
+            "description": "Explanation of why you are pressing this key"
+          }
+        },
+        "required": ["app", "key", "reasoning"]
+      },
+      "executor": "tools/app-control-press.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "app_control_combo",
+      "description": "Press multiple keys simultaneously in the target application (e.g. cmd+shift+4). Use for keyboard shortcuts where every key is held at once.",
+      "category": "app-control",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Bundle ID (preferred, e.g. 'com.apple.Safari') or process name of the target application"
+          },
+          "keys": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "Sequence of keys pressed simultaneously, e.g. [\"cmd\", \"shift\", \"4\"]"
+          },
+          "duration_ms": {
+            "type": "integer",
+            "description": "Hold duration in milliseconds"
+          },
+          "reasoning": {
+            "type": "string",
+            "description": "Explanation of why you are pressing this combo"
+          }
+        },
+        "required": ["app", "keys", "reasoning"]
+      },
+      "executor": "tools/app-control-combo.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "app_control_sequence",
+      "description": "Send an ordered batch of single-key presses to the target application. The target app is activated once at the start, then each step is pressed serially with optional inter-step gaps. PREFER this over multiple back-to-back app_control_press calls when sending an ordered batch (e.g. menu navigation, repeated movement) — sequence runs in a single round-trip with no window for keyboard focus to drift between presses.",
+      "category": "app-control",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Bundle ID (preferred, e.g. 'com.apple.Safari') or process name of the target application"
+          },
+          "steps": {
+            "type": "array",
+            "description": "Ordered list of single-key presses to execute serially.",
+            "items": {
+              "type": "object",
+              "properties": {
+                "key": {
+                  "type": "string",
+                  "description": "Key identifier, e.g. \"return\", \"a\", \"right\""
+                },
+                "modifiers": {
+                  "type": "array",
+                  "items": { "type": "string" },
+                  "description": "Modifier list for this step, e.g. [\"cmd\", \"shift\"]. Omit for no modifiers."
+                },
+                "duration_ms": {
+                  "type": "integer",
+                  "description": "Hold duration for this key in milliseconds (default 50)"
+                },
+                "gap_ms": {
+                  "type": "integer",
+                  "description": "Pause after this step before starting the next, in milliseconds (default 30)"
+                }
+              },
+              "required": ["key"]
+            }
+          },
+          "reasoning": {
+            "type": "string",
+            "description": "Explanation of what this sequence does and why"
+          }
+        },
+        "required": ["app", "steps", "reasoning"]
+      },
+      "executor": "tools/app-control-sequence.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "app_control_type",
+      "description": "Type literal text into the target application at the current focus. Ensure the intended field is focused before calling.",
+      "category": "app-control",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Bundle ID (preferred, e.g. 'com.apple.Safari') or process name of the target application"
+          },
+          "text": {
+            "type": "string",
+            "description": "The text to type"
+          },
+          "reasoning": {
+            "type": "string",
+            "description": "Explanation of what you are typing and why"
+          }
+        },
+        "required": ["app", "text", "reasoning"]
+      },
+      "executor": "tools/app-control-type.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "app_control_click",
+      "description": "Click at the given (x, y) coordinates inside the target application's window. Defaults to a single left-click; pass `button` and/or `double` to vary.",
+      "category": "app-control",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Bundle ID (preferred, e.g. 'com.apple.Safari') or process name of the target application"
+          },
+          "x": {
+            "type": "integer",
+            "description": "X coordinate (window-relative)"
+          },
+          "y": {
+            "type": "integer",
+            "description": "Y coordinate (window-relative)"
+          },
+          "button": {
+            "type": "string",
+            "enum": ["left", "right", "middle"],
+            "description": "Mouse button (default: \"left\")"
+          },
+          "double": {
+            "type": "boolean",
+            "description": "When true, performs a double-click"
+          },
+          "reasoning": {
+            "type": "string",
+            "description": "Explanation of what you see and why you are clicking here"
+          }
+        },
+        "required": ["app", "x", "y", "reasoning"]
+      },
+      "executor": "tools/app-control-click.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "app_control_drag",
+      "description": "Drag from (from_x, from_y) to (to_x, to_y) inside the target application's window. Defaults to left button.",
+      "category": "app-control",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Bundle ID (preferred, e.g. 'com.apple.Safari') or process name of the target application"
+          },
+          "from_x": {
+            "type": "integer",
+            "description": "Source X coordinate (window-relative)"
+          },
+          "from_y": {
+            "type": "integer",
+            "description": "Source Y coordinate (window-relative)"
+          },
+          "to_x": {
+            "type": "integer",
+            "description": "Destination X coordinate (window-relative)"
+          },
+          "to_y": {
+            "type": "integer",
+            "description": "Destination Y coordinate (window-relative)"
+          },
+          "button": {
+            "type": "string",
+            "enum": ["left", "right", "middle"],
+            "description": "Mouse button (default: \"left\")"
+          },
+          "reasoning": {
+            "type": "string",
+            "description": "Explanation of what you are dragging and why"
+          }
+        },
+        "required": ["app", "from_x", "from_y", "to_x", "to_y", "reasoning"]
+      },
+      "executor": "tools/app-control-drag.ts",
+      "execution_target": "host"
+    },
+    {
+      "name": "app_control_stop",
+      "description": "Stop the current app-control session. When `app` is omitted, stops whichever app currently holds the session. This is the terminal action for an app-control flow.",
+      "category": "app-control",
+      "risk": "low",
+      "input_schema": {
+        "type": "object",
+        "properties": {
+          "app": {
+            "type": "string",
+            "description": "Optional bundle ID or process name. When omitted, stops whichever app currently holds the session."
+          },
+          "reason": {
+            "type": "string",
+            "description": "Free-form reason, surfaced for logging"
+          }
+        },
+        "required": []
+      },
+      "executor": "tools/app-control-stop.ts",
+      "execution_target": "host"
+    }
+  ]
+}

package/src/config/bundled-skills/app-control/tools/app-control-click.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_click", input, context);
+}

package/src/config/bundled-skills/app-control/tools/app-control-combo.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_combo", input, context);
+}

package/src/config/bundled-skills/app-control/tools/app-control-drag.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_drag", input, context);
+}

package/src/config/bundled-skills/app-control/tools/app-control-observe.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_observe", input, context);
+}

package/src/config/bundled-skills/app-control/tools/app-control-press.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_press", input, context);
+}

package/src/config/bundled-skills/app-control/tools/app-control-sequence.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_sequence", input, context);
+}

package/src/config/bundled-skills/app-control/tools/app-control-start.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_start", input, context);
+}

package/src/config/bundled-skills/app-control/tools/app-control-stop.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_stop", input, context);
+}

package/src/config/bundled-skills/app-control/tools/app-control-type.ts

@@ -0,0 +1,12 @@
+import { forwardAppControlProxyTool } from "../../../../tools/app-control/skill-proxy-bridge.js";
+import type {
+  ToolContext,
+  ToolExecutionResult,
+} from "../../../../tools/types.js";
+
+export async function run(
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return forwardAppControlProxyTool("app_control_type", input, context);
+}

package/src/config/bundled-skills/computer-use/SKILL.md

@@ -6,6 +6,12 @@ metadata:
   emoji: "🖥️"
   vellum:
     display-name: "Computer Use"
+    activation-hints:
+      - "User asks the assistant to click, type, drag, or interact with the macOS GUI directly"
+      - "User wants control of a desktop app with no CLI or API alternative (games, design tools, visual workflows)"
+      - "User wants screenshots or visual inspection of what is currently on screen"
+    avoid-when:
+      - "Task can be done via a more specific skill (gmail, calendar, contacts, terminal-sessions) or a CLI / API call"
 ---
 
 This skill provides the computer_use_* action tools for controlling

package/src/config/bundled-skills/computer-use/TOOLS.json

@@ -8,13 +8,8 @@
       "risk": "low",
       "input_schema": {
         "type": "object",
-        "properties": {
-          "activity": {
-            "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
-          }
-        },
-        "required": ["activity"]
+        "properties": {},
+        "required": []
       },
       "executor": "tools/computer-use-observe.ts",
       "execution_target": "host"
@@ -29,7 +24,11 @@
         "properties": {
           "click_type": {
             "type": "string",
-            "enum": ["single", "double", "right"],
+            "enum": [
+              "single",
+              "double",
+              "right"
+            ],
             "description": "Type of click to perform (default: \"single\")"
           },
           "element_id": {
@@ -48,12 +47,14 @@
             "type": "string",
             "description": "Explanation of what you see and why you are clicking here"
           },
-          "activity": {
+          "target_client_id": {
             "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
+            "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
           }
         },
-        "required": ["reasoning"]
+        "required": [
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-click.ts",
       "execution_target": "host"
@@ -74,12 +75,15 @@
             "type": "string",
             "description": "Explanation of what you are typing and why"
           },
-          "activity": {
+          "target_client_id": {
             "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
+            "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
           }
         },
-        "required": ["text", "reasoning"]
+        "required": [
+          "text",
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-type-text.ts",
       "execution_target": "host"
@@ -100,12 +104,15 @@
             "type": "string",
             "description": "Explanation of why you are pressing this key"
           },
-          "activity": {
+          "target_client_id": {
             "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
+            "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
           }
         },
-        "required": ["key", "reasoning"]
+        "required": [
+          "key",
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-key.ts",
       "execution_target": "host"
@@ -132,7 +139,12 @@
           },
           "direction": {
             "type": "string",
-            "enum": ["up", "down", "left", "right"],
+            "enum": [
+              "up",
+              "down",
+              "left",
+              "right"
+            ],
             "description": "Scroll direction"
           },
           "amount": {
@@ -143,12 +155,16 @@
             "type": "string",
             "description": "Explanation of why you are scrolling"
           },
-          "activity": {
+          "target_client_id": {
             "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
+            "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
           }
         },
-        "required": ["direction", "amount", "reasoning"]
+        "required": [
+          "direction",
+          "amount",
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-scroll.ts",
       "execution_target": "host"
@@ -189,12 +205,14 @@
             "type": "string",
             "description": "Explanation of what you are dragging and why"
           },
-          "activity": {
+          "target_client_id": {
             "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
+            "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
           }
         },
-        "required": ["reasoning"]
+        "required": [
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-drag.ts",
       "execution_target": "host"
@@ -215,12 +233,15 @@
             "type": "string",
             "description": "Explanation of what you are waiting for"
           },
-          "activity": {
+          "target_client_id": {
             "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
+            "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
           }
         },
-        "required": ["duration_ms", "reasoning"]
+        "required": [
+          "duration_ms",
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-wait.ts",
       "execution_target": "host"
@@ -241,12 +262,15 @@
             "type": "string",
             "description": "Explanation of why you need to open or switch to this app"
           },
-          "activity": {
+          "target_client_id": {
             "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
+            "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
           }
         },
-        "required": ["app_name", "reasoning"]
+        "required": [
+          "app_name",
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-open-app.ts",
       "execution_target": "host"
@@ -267,12 +291,15 @@
             "type": "string",
             "description": "Explanation of what this script does and why AppleScript is better than UI interaction for this step"
           },
-          "activity": {
+          "target_client_id": {
             "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
+            "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
           }
         },
-        "required": ["script", "reasoning"]
+        "required": [
+          "script",
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-run-applescript.ts",
       "execution_target": "host"
@@ -288,13 +315,11 @@
           "summary": {
             "type": "string",
             "description": "Human-readable summary of what was accomplished"
-          },
-          "activity": {
-            "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
           }
         },
-        "required": ["summary"]
+        "required": [
+          "summary"
+        ]
       },
       "executor": "tools/computer-use-done.ts",
       "execution_target": "host"
@@ -314,13 +339,12 @@
           "reasoning": {
             "type": "string",
             "description": "Explanation of how you determined the answer"
-          },
-          "activity": {
-            "type": "string",
-            "description": "Brief non-technical explanation of why this tool is being called"
           }
         },
-        "required": ["answer", "reasoning"]
+        "required": [
+          "answer",
+          "reasoning"
+        ]
       },
       "executor": "tools/computer-use-respond.ts",
       "execution_target": "host"