@vellumai/assistant 0.8.3 → 0.8.5

package/src/__tests__/tool-executor.test.ts

@@ -51,6 +51,9 @@ let lastCheckArgs:
 /** Optional override for getTool — lets tests supply skill-origin tools. */
 let getToolOverride: ((name: string) => Tool | undefined) | undefined;
 
+/** Optional override for getAllTools — lets tests supply a registry snapshot. */
+let getAllToolsOverride: (() => Tool[]) | undefined;
+
 /** Override the check() result for tests that need to trigger prompting. */
 let checkResultOverride: { decision: string; reason: string } | undefined;
 
@@ -128,7 +131,7 @@ mock.module("../permissions/checker.js", () => ({
 mock.module("../memory/tool-usage-store.js", () => ({
   recordToolInvocation: () => {},
   getRecentInvocations: () => [],
-  rotateToolInvocations: () => 0,
+  rotateToolInvocations: async () => 0,
 }));
 
 mock.module("../tools/registry.js", () => ({
@@ -140,11 +143,11 @@ mock.module("../tools/registry.js", () => ({
       description: "test tool",
       category: "test",
       defaultRiskLevel: "low",
-      getDefinition: () => ({}),
+      input_schema: {},
       execute: async () => fakeToolResult,
     };
   },
-  getAllTools: () => [],
+  getAllTools: () => (getAllToolsOverride ? getAllToolsOverride() : []),
 }));
 
 mock.module("../tools/shared/filesystem/path-policy.js", () => ({
@@ -179,6 +182,7 @@ describe("ToolExecutor allowedToolNames gating", () => {
     fakeToolResult = { content: "ok", isError: false };
     lastCheckArgs = undefined;
     getToolOverride = undefined;
+    getAllToolsOverride = undefined;
     checkResultOverride = undefined;
     checkFnOverride = undefined;
     cachedAssessmentOverride = undefined;
@@ -271,6 +275,82 @@ describe("ToolExecutor allowedToolNames gating", () => {
     expect(result.content).toContain("file_read");
     expect(result.content).toContain("not currently active");
   });
+
+  test("unknown tool suggestion list is scoped to allowedToolNames", async () => {
+    // Surfacing every globally registered tool would leak tools active in
+    // other sessions and misdirect the model to tools it cannot invoke.
+    const makeTool = (name: string): Tool =>
+      ({
+        name,
+        description: "test tool",
+        category: "test",
+        defaultRiskLevel: RiskLevel.Low,
+        input_schema: { type: "object" as const, properties: {} },
+        execute: async () => fakeToolResult,
+      }) as unknown as Tool;
+    getAllToolsOverride = () => [
+      makeTool("file_read"),
+      makeTool("file_write"),
+      makeTool("secret_skill_tool"),
+    ];
+    const executor = new ToolExecutor(makePrompter());
+    const allowed = new Set(["file_read", "file_write"]);
+    const result = await executor.execute(
+      "unknown_tool",
+      { foo: "bar" },
+      makeContext({ allowedToolNames: allowed }),
+    );
+    expect(result.isError).toBe(true);
+    expect(result.content).toBe(
+      "Unknown tool: unknown_tool. Available tools: file_read, file_write",
+    );
+    expect(result.content).not.toContain("secret_skill_tool");
+  });
+
+  test("unknown tool name reports 'Unknown tool' even when allowedToolNames is set", async () => {
+    // Regression: a hallucinated tool name with allowedToolNames set used to
+    // hit the "not currently active. Load the skill that provides this tool
+    // first." gate, which sent the model chasing a nonexistent skill. The
+    // registry-lookup gate runs first now so the model sees the real list.
+    const executor = new ToolExecutor(makePrompter());
+    const allowed = new Set(["file_read"]);
+    const result = await executor.execute(
+      "unknown_tool",
+      { foo: "bar" },
+      makeContext({ allowedToolNames: allowed }),
+    );
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("Unknown tool: unknown_tool");
+    expect(result.content).not.toContain("not currently active");
+  });
+
+  test("inactive skill tool names the owning skill in the load hint", async () => {
+    const executor = new ToolExecutor(makePrompter());
+    getToolOverride = (name: string) => {
+      if (name !== "skill_tool_x") return undefined;
+      return {
+        name,
+        description: "tool from a skill",
+        category: "skill",
+        defaultRiskLevel: RiskLevel.Low,
+        executionTarget: "sandbox" as const,
+        origin: "skill" as const,
+        ownerSkillId: "my-skill",
+        input_schema: { type: "object" as const, properties: {} },
+        execute: async () => fakeToolResult,
+      };
+    };
+    const allowed = new Set(["file_read"]);
+    const result = await executor.execute(
+      "skill_tool_x",
+      {},
+      makeContext({ allowedToolNames: allowed }),
+    );
+    expect(result.isError).toBe(true);
+    expect(result.content).toBe(
+      'Tool "skill_tool_x" is not currently active. Load the "my-skill" skill that provides this tool first.',
+    );
+  });
 });
 
 describe("ToolExecutor policy context plumbing", () => {
@@ -295,11 +375,7 @@ describe("ToolExecutor policy context plumbing", () => {
         ownerSkillId: "my-skill-123",
         ownerSkillVersionHash: "abc123hash",
         executionTarget: "sandbox" as const,
-        getDefinition: () => ({
-          name,
-          description: "skill tool",
-          input_schema: { type: "object" as const, properties: {} },
-        }),
+        input_schema: { type: "object" as const, properties: {} },
         execute: async () => fakeToolResult,
       };
     };
@@ -347,12 +423,9 @@ describe("ToolExecutor policy context plumbing", () => {
         description: "core tool",
         category: "core",
         defaultRiskLevel: RiskLevel.Low,
+        executionTarget: "sandbox" as const,
         origin: "core" as const,
-        getDefinition: () => ({
-          name,
-          description: "core tool",
-          input_schema: { type: "object" as const, properties: {} },
-        }),
+        input_schema: { type: "object" as const, properties: {} },
         execute: async () => fakeToolResult,
       };
     };
@@ -384,11 +457,7 @@ describe("ToolExecutor policy context plumbing", () => {
         ownerSkillId: "host-skill",
         ownerSkillVersionHash: "host-hash",
         executionTarget: "host" as const,
-        getDefinition: () => ({
-          name,
-          description: "host skill tool",
-          input_schema: { type: "object" as const, properties: {} },
-        }),
+        input_schema: { type: "object" as const, properties: {} },
         execute: async () => fakeToolResult,
       };
     };
@@ -409,41 +478,6 @@ describe("ToolExecutor policy context plumbing", () => {
     });
   });
 
-  test("skill tool without executionTarget passes undefined executionTarget", async () => {
-    getToolOverride = (name: string) => {
-      if (name === "unknown_tool") return undefined;
-      return {
-        name,
-        description: "skill without target",
-        category: "skill",
-        defaultRiskLevel: RiskLevel.Low,
-        origin: "skill" as const,
-        ownerSkillId: "no-target-skill",
-        // executionTarget intentionally omitted
-        getDefinition: () => ({
-          name,
-          description: "skill tool",
-          input_schema: { type: "object" as const, properties: {} },
-        }),
-        execute: async () => fakeToolResult,
-      };
-    };
-
-    const executor = new ToolExecutor(makePrompter());
-    const result = await executor.execute(
-      "no_target_tool",
-      {},
-      makeContext({ requireFreshApproval: true }),
-    );
-
-    expect(result.isError).toBe(false);
-    expect(lastCheckArgs).toBeDefined();
-    expect(lastCheckArgs!.policyContext).toEqual({
-      conversationId: "conversation-1",
-      executionContext: "conversation",
-      executionTarget: undefined,
-    });
-  });
 });
 
 // ---------------------------------------------------------------------------
@@ -846,6 +880,7 @@ describe("ToolExecutor forcePromptSideEffects enforcement", () => {
 // Import the real buildSanitizedEnv (not mocked) for baseline credential tests
 const {
   buildSanitizedEnv,
+  KATA_INJECTED_ENV_VARS,
   KATA_SAFE_ENV_VARS,
   SAFE_ENV_VARS,
   ALWAYS_INJECTED_ENV_VARS,
@@ -910,6 +945,7 @@ describe("buildSanitizedEnv — baseline: credential exclusion", () => {
     const allowed: string[] = [
       ...SAFE_ENV_VARS,
       ...KATA_SAFE_ENV_VARS,
+      ...KATA_INJECTED_ENV_VARS,
       ...ALWAYS_INJECTED_ENV_VARS,
     ];
     const env = buildSanitizedEnv();

package/src/__tests__/tool-grant-request-escalation.test.ts

@@ -31,11 +31,7 @@ const fakeTool = {
   description: "Run a shell command",
   category: "shell",
   defaultRiskLevel: "high",
-  getDefinition: () => ({
-    name: "bash",
-    description: "Run a shell command",
-    input_schema: {},
-  }),
+  input_schema: {},
   execute: async () => ({ content: "ok", isError: false }),
 };
 
@@ -57,7 +53,6 @@ mock.module("../notifications/emit-signal.js", () => ({
       deliveryResults: [],
     };
   },
-  registerBroadcastFn: () => {},
 }));
 
 // Mock channel guardian service — provide a guardian binding for 'self' + 'telegram'

package/src/__tests__/tool-result-metadata-plumbing.test.ts

@@ -0,0 +1,167 @@
+/**
+ * Tests that `ToolExecutionResult.activityMetadata` plumbs through to the
+ * emitted `tool_result` server event via `handleToolResult`. The forward path
+ * is: ToolExecutionResult (set by the tool executor) → AgentEvent tool_result
+ * (emitted by the agent loop) → tool_result server message (emitted by
+ * handleToolResult to the SSE sink).
+ *
+ * Mirrors the mocked-dependency pattern used in tool-preview-lifecycle.test.ts
+ * and annotate-risk-options.test.ts.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Mock platform (must precede imports that read it) ─────────────────────────
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+mock.module("../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: () => null,
+  updateMessageContent: () => {},
+  provenanceFromTrustContext: () => ({}),
+}));
+
+mock.module("../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+// ── Imports (after mocks) ─────────────────────────────────────────────────────
+import type {
+  EventHandlerDeps,
+  EventHandlerState,
+} from "../daemon/conversation-agent-loop-handlers.js";
+import {
+  createEventHandlerState,
+  handleToolResult,
+} from "../daemon/conversation-agent-loop-handlers.js";
+import type { ServerMessage } from "../daemon/message-protocol.js";
+import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
+
+type ToolResultEvent = Extract<ServerMessage, { type: "tool_result" }>;
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function createCollectorDeps(): {
+  deps: EventHandlerDeps;
+  events: ServerMessage[];
+} {
+  const events: ServerMessage[] = [];
+  const deps = {
+    ctx: {
+      conversationId: "conv-meta",
+      provider: { name: "anthropic" },
+      traceEmitter: { emit: () => {} },
+      streamThinking: false,
+      emitActivityState: () => {},
+      markWorkspaceTopLevelDirty: () => {},
+      currentTurnSurfaces: [],
+    } as unknown as EventHandlerDeps["ctx"],
+    onEvent: (msg: ServerMessage) => events.push(msg),
+    reqId: "req-meta",
+    isFirstMessage: false,
+    shouldGenerateTitle: false,
+    rlog: new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }) as unknown as EventHandlerDeps["rlog"],
+    turnChannelContext: {
+      userMessageChannel: "vellum",
+      assistantMessageChannel: "vellum",
+    } as EventHandlerDeps["turnChannelContext"],
+    turnInterfaceContext: {
+      userMessageInterface: "macos",
+      assistantMessageInterface: "macos",
+    } as EventHandlerDeps["turnInterfaceContext"],
+  } as EventHandlerDeps;
+  return { deps, events };
+}
+
+function primeState(state: EventHandlerState, toolUseId: string): void {
+  state.toolUseIdToName.set(toolUseId, "web_search");
+  state.toolCallTimestamps.set(toolUseId, { startedAt: Date.now() });
+  state.currentTurnToolUseIds.push(toolUseId);
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe("tool_result activityMetadata plumbing", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("forwards activityMetadata to the emitted tool_result event", () => {
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "toolu_meta_present";
+    primeState(state, toolUseId);
+
+    const activityMetadata: ToolActivityMetadata = {
+      webSearch: {
+        query: "x",
+        provider: "tavily",
+        resultCount: 0,
+        durationMs: 1,
+        results: [],
+      },
+    };
+
+    handleToolResult(state, deps, {
+      type: "tool_result",
+      toolUseId,
+      content: "",
+      isError: false,
+      activityMetadata,
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent).toBeDefined();
+    expect(toolResultEvent?.activityMetadata).toEqual(activityMetadata);
+  });
+
+  test("omits activityMetadata when the executor did not populate it", () => {
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "toolu_meta_absent";
+    primeState(state, toolUseId);
+
+    handleToolResult(state, deps, {
+      type: "tool_result",
+      toolUseId,
+      content: "ok",
+      isError: false,
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent).toBeDefined();
+    expect(toolResultEvent?.activityMetadata).toBeUndefined();
+  });
+});

package/src/__tests__/trusted-contact-approval-notifier.test.ts

@@ -33,7 +33,6 @@ mock.module("../notifications/emit-signal.js", () => ({
     reason: "ok",
     deliveryResults: [],
   }),
-  registerBroadcastFn: () => {},
 }));
 
 // ── Gateway client mock ──

package/src/__tests__/trusted-contact-inline-approval-integration.test.ts

@@ -48,7 +48,6 @@ mock.module("../notifications/emit-signal.js", () => ({
       ],
     };
   },
-  registerBroadcastFn: () => {},
 }));
 
 // Mock task run rules
@@ -62,11 +61,7 @@ const fakeTool = {
   description: "Run a shell command",
   category: "shell",
   defaultRiskLevel: "high",
-  getDefinition: () => ({
-    name: "bash",
-    description: "Run a shell command",
-    input_schema: {},
-  }),
+  input_schema: {},
   execute: async () => ({ content: "ok", isError: false }),
 };
 mock.module("../tools/registry.js", () => ({

package/src/__tests__/trusted-contact-multichannel.test.ts

@@ -36,7 +36,6 @@ mock.module("../notifications/emit-signal.js", () => ({
       deliveryResults: [],
     };
   },
-  registerBroadcastFn: () => {},
 }));
 
 // Mock access-request-helper directly to capture notification calls.

package/src/__tests__/twilio-routes.test.ts

@@ -311,7 +311,7 @@ mock.module("../tools/credentials/metadata-store.js", () => ({
 }));
 
 mock.module("../providers/registry.js", () => ({
-  initializeProviders: () => {},
+  initializeProviders: async () => {},
 }));
 
 import {

package/src/__tests__/ui-file-upload-surface.test.ts

@@ -78,7 +78,7 @@ describe("file_upload interactivity", () => {
 
 describe("ui_show tool includes file_upload", () => {
   test("input_schema surface_type enum includes file_upload", () => {
-    const definition = uiShowTool.getDefinition();
+    const definition = uiShowTool;
     const surfaceTypeEnum = (
       definition.input_schema as {
         properties: { surface_type: { enum: string[] } };
@@ -89,7 +89,7 @@ describe("ui_show tool includes file_upload", () => {
   });
 
   test("description mentions file_upload", () => {
-    const definition = uiShowTool.getDefinition();
+    const definition = uiShowTool;
     expect(definition.description).toContain("file_upload");
   });
 });

package/src/__tests__/usage-routes.test.ts

@@ -59,6 +59,7 @@ function seedEvents() {
       outputTokens: 200,
       cacheCreationInputTokens: 50,
       cacheReadInputTokens: 100,
+      rawUsage: null,
     },
     { estimatedCostUsd: 0.005, pricingStatus: "priced" },
   );
@@ -82,6 +83,7 @@ function seedEvents() {
       outputTokens: 100,
       cacheCreationInputTokens: 0,
       cacheReadInputTokens: 0,
+      rawUsage: null,
     },
     { estimatedCostUsd: 0.001, pricingStatus: "priced" },
   );
@@ -104,6 +106,7 @@ function seedEvents() {
       outputTokens: 400,
       cacheCreationInputTokens: 0,
       cacheReadInputTokens: 0,
+      rawUsage: null,
     },
     { estimatedCostUsd: 0, pricingStatus: "unpriced" },
   );

package/src/__tests__/verification-control-plane-policy.test.ts

@@ -80,7 +80,7 @@ mock.module("../memory/conversation-crud.js", () => ({
 mock.module("../memory/tool-usage-store.js", () => ({
   recordToolInvocation: () => {},
   getRecentInvocations: () => [],
-  rotateToolInvocations: () => 0,
+  rotateToolInvocations: async () => 0,
 }));
 
 mock.module("../tools/registry.js", () => ({
@@ -91,7 +91,7 @@ mock.module("../tools/registry.js", () => ({
       description: "test tool",
       category: "test",
       defaultRiskLevel: "low",
-      getDefinition: () => ({}),
+      input_schema: {},
       execute: async () => fakeToolResult,
     };
   },

package/src/__tests__/web-fetch.test.ts

@@ -1622,8 +1622,8 @@ describe("web_fetch tool", () => {
     );
 
     expect(result.isError).toBe(false);
-    expect(result.content).toContain("Extracted text content is very short");
-    expect(result.content).toContain("JavaScript rendering");
+    expect(result.content).toContain("Extracted only");
+    expect(result.content).toContain("JavaScript-rendered");
   });
 
   test("does not suggest browser skill when HTML page has substantial content", async () => {