@vellumai/assistant 0.8.5 → 0.8.7-dev.202606052118.34cd356

package/src/__tests__/tool-error-hook.test.ts

@@ -0,0 +1,278 @@
+/**
+ * Tests for the default `tool-error` plugin's `post-tool-use` hook.
+ *
+ * Covers:
+ * - The hook surfaces the canonical coaching notice via `additionalContext`
+ *   (leaving the tool result's `content` untouched) when the result carries
+ *   `is_error`, and is a no-op for a successful result.
+ * - The consecutive-failure guard is derived from the conversation history per
+ *   tool name: coaching fires on the 1st through Nth back-to-back failure of a
+ *   tool and is dropped once that tool exceeds the cap. A success in between
+ *   resets the streak, and failures of a different tool don't count.
+ * - End-to-end through `runHook` + the registry: registering the default
+ *   plugin makes the hook fire, and the default runs ahead of a later
+ *   user hook so the user observes the already-set `additionalContext`.
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import { HOOKS } from "../plugin-api/constants.js";
+import type { PluginLogger, PostToolUseContext } from "../plugin-api/types.js";
+import postToolUse, {
+  TOOL_ERROR_NUDGE_TEXT,
+} from "../plugins/defaults/tool-error/hooks/post-tool-use.js";
+import { defaultToolErrorPlugin } from "../plugins/defaults/tool-error/register.js";
+import { runHook } from "../plugins/pipeline.js";
+import {
+  registerPlugin,
+  resetPluginRegistryForTests,
+} from "../plugins/registry.js";
+import type { Message, ToolResultContent } from "../providers/types.js";
+
+const noopLogger: PluginLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+};
+
+const BASE_CONTENT = "tool blew up";
+
+/** Assistant turn issuing a single `tool_use` block. */
+function toolUseTurn(id: string, name: string): Message {
+  return {
+    role: "assistant",
+    content: [{ type: "tool_use", id, name, input: {} }],
+  };
+}
+
+/** User turn carrying a single `tool_result` for a prior `tool_use`. */
+function toolResultTurn(toolUseId: string, isError: boolean): Message {
+  return {
+    role: "user",
+    content: [
+      {
+        type: "tool_result",
+        tool_use_id: toolUseId,
+        content: "result",
+        is_error: isError,
+      },
+    ],
+  };
+}
+
+function errorResponse(toolUseId: string): ToolResultContent {
+  return {
+    type: "tool_result",
+    tool_use_id: toolUseId,
+    content: BASE_CONTENT,
+    is_error: true,
+  };
+}
+
+function makeCtx(
+  toolResponse: ToolResultContent,
+  messages: Message[],
+): PostToolUseContext {
+  return {
+    conversationId: "conv-tool-error-test",
+    toolResponse,
+    messages,
+    maxInputTokens: 10_000,
+    logger: noopLogger,
+  };
+}
+
+/**
+ * Build a history of `priorErrors` back-to-back failures of `toolName`,
+ * followed by the current turn's `tool_use` (whose result is delivered via
+ * `ctx.toolResponse`, not history). Returns the history and the current
+ * tool_use id.
+ */
+function historyWithConsecutiveErrors(
+  toolName: string,
+  priorErrors: number,
+): { messages: Message[]; currentToolUseId: string } {
+  const messages: Message[] = [];
+  for (let i = 0; i < priorErrors; i++) {
+    const id = `${toolName}-${i}`;
+    messages.push(toolUseTurn(id, toolName));
+    messages.push(toolResultTurn(id, true));
+  }
+  const currentToolUseId = `${toolName}-current`;
+  messages.push(toolUseTurn(currentToolUseId, toolName));
+  return { messages, currentToolUseId };
+}
+
+describe("tool-error post-tool-use hook — direct", () => {
+  test("surfaces the canonical coaching notice via additionalContext on a first error", async () => {
+    // GIVEN an error result whose tool has not failed before.
+    const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+      "search",
+      0,
+    );
+    const ctx = makeCtx(errorResponse(currentToolUseId), messages);
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN the coaching notice is surfaced via additionalContext, and the tool
+    // result's own content is left untouched.
+    expect(ctx.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+    expect(ctx.toolResponse.content).toBe(BASE_CONTENT);
+  });
+
+  test("is a no-op for a successful result", async () => {
+    // GIVEN a successful result for a tool.
+    const messages = [toolUseTurn("search-current", "search")];
+    const ctx = makeCtx(
+      {
+        type: "tool_result",
+        tool_use_id: "search-current",
+        content: BASE_CONTENT,
+        is_error: false,
+      },
+      messages,
+    );
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN no coaching is surfaced and the content is left untouched.
+    expect(ctx.additionalContext).toBeUndefined();
+    expect(ctx.toolResponse.content).toBe(BASE_CONTENT);
+  });
+
+  test("keeps coaching up to and including the consecutive-failure cap", async () => {
+    // GIVEN a tool that has already failed 0, 1, then 2 times in a row — so the
+    // current failure is the 1st, 2nd, then 3rd (the cap) consecutive failure.
+    for (let priorErrors = 0; priorErrors <= 2; priorErrors++) {
+      const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+        "search",
+        priorErrors,
+      );
+      const ctx = makeCtx(errorResponse(currentToolUseId), messages);
+
+      // WHEN the hook runs.
+      await postToolUse(ctx);
+
+      // THEN it still coaches the retry.
+      expect(ctx.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+    }
+  });
+
+  test("drops the coaching once the tool exceeds the cap", async () => {
+    // GIVEN a tool that has already failed 3 times in a row, so the current
+    // failure is the 4th — past the cap of 3.
+    const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+      "search",
+      3,
+    );
+    const ctx = makeCtx(errorResponse(currentToolUseId), messages);
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN no coaching is surfaced (the error is likely unrecoverable) and the
+    // result is left untouched.
+    expect(ctx.additionalContext).toBeUndefined();
+    expect(ctx.toolResponse.content).toBe(BASE_CONTENT);
+  });
+
+  test("a successful result between failures resets the streak", async () => {
+    // GIVEN four failures of "search" but with a success in the middle, so the
+    // trailing run of failures is only two — the current failure is the 3rd.
+    const messages: Message[] = [
+      toolUseTurn("search-0", "search"),
+      toolResultTurn("search-0", true),
+      toolUseTurn("search-1", "search"),
+      toolResultTurn("search-1", true),
+      toolUseTurn("search-2", "search"),
+      toolResultTurn("search-2", false),
+      toolUseTurn("search-3", "search"),
+      toolResultTurn("search-3", true),
+      toolUseTurn("search-current", "search"),
+    ];
+    const ctx = makeCtx(errorResponse("search-current"), messages);
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN coaching still fires — the success reset the streak below the cap.
+    expect(ctx.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+  });
+
+  test("counts failures per tool name, not globally", async () => {
+    // GIVEN three failures of "other" and zero of "search" before the current
+    // "search" failure — a global counter would be over the cap, but the
+    // per-tool streak for "search" is just this one failure.
+    const messages: Message[] = [
+      toolUseTurn("other-0", "other"),
+      toolResultTurn("other-0", true),
+      toolUseTurn("other-1", "other"),
+      toolResultTurn("other-1", true),
+      toolUseTurn("other-2", "other"),
+      toolResultTurn("other-2", true),
+      toolUseTurn("search-current", "search"),
+    ];
+    const ctx = makeCtx(errorResponse("search-current"), messages);
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN "search" is coached on its first failure.
+    expect(ctx.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+  });
+});
+
+describe("tool-error post-tool-use hook — via runHook", () => {
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+  });
+
+  test("registering the default plugin coaches an error result", async () => {
+    // GIVEN the default tool-error plugin is registered.
+    registerPlugin(defaultToolErrorPlugin);
+    const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+      "search",
+      0,
+    );
+
+    // WHEN the post-tool-use chain runs over an error result.
+    const result = await runHook<PostToolUseContext>(
+      HOOKS.POST_TOOL_USE,
+      makeCtx(errorResponse(currentToolUseId), messages),
+    );
+
+    // THEN the coaching notice is surfaced via additionalContext.
+    expect(result.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+  });
+
+  test("default hook runs before a later-registered user hook", async () => {
+    // GIVEN the default plugin is registered first, then a user plugin whose
+    // hook records the additionalContext it observes.
+    let observed: string | undefined;
+    registerPlugin(defaultToolErrorPlugin);
+    registerPlugin({
+      manifest: { name: "observer-plugin", version: "0.0.1" },
+      hooks: {
+        "post-tool-use": async (ctx: PostToolUseContext) => {
+          observed = ctx.additionalContext;
+        },
+      },
+    });
+    const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+      "search",
+      0,
+    );
+
+    // WHEN the chain runs.
+    await runHook<PostToolUseContext>(
+      HOOKS.POST_TOOL_USE,
+      makeCtx(errorResponse(currentToolUseId), messages),
+    );
+
+    // THEN the user hook saw the already-set additionalContext.
+    expect(observed).toBe(TOOL_ERROR_NUDGE_TEXT);
+  });
+});

package/src/__tests__/tool-executor-lifecycle-events.test.ts

@@ -72,6 +72,7 @@ mock.module("../permissions/checker.js", () => ({
 
 mock.module("../memory/conversation-crud.js", () => ({
   createConversation: (title: string) => ({ id: "conversation-1", title }),
+  reserveMessage: mock(async () => ({ id: "msg-reserve" })),
 }));
 
 // Mock every export so downstream test files that dynamically import modules
@@ -85,15 +86,16 @@ mock.module("../memory/tool-usage-store.js", () => ({
 mock.module("../tools/registry.js", () => ({
   getTool: (name: string) => {
     if (name === "unknown_tool") return undefined;
-    // Skill tools carry origin and executionTarget from their manifest
+    // Skill tools carry executionTarget from their manifest. Ownership lives
+    // on the registry (mocked below via getToolOwner reading the override's
+    // owner field), so it doesn't appear on the Tool object itself.
     if (name === "skill_host_tool") {
       return {
         name,
         description: "skill host tool",
         category: "skill",
         defaultRiskLevel: "low",
-        origin: "skill" as const,
-        ownerSkillId: "test-skill",
+        owner: { kind: "skill", id: "test-skill" },
         executionTarget: "host" as const,
         input_schema: {},
         execute: async () => {
@@ -108,8 +110,7 @@ mock.module("../tools/registry.js", () => ({
         description: "skill sandbox tool",
         category: "skill",
         defaultRiskLevel: "low",
-        origin: "skill" as const,
-        ownerSkillId: "test-skill",
+        owner: { kind: "skill", id: "test-skill" },
         executionTarget: "sandbox" as const,
         input_schema: {},
         execute: async () => {
@@ -126,8 +127,7 @@ mock.module("../tools/registry.js", () => ({
         description: "skill tool with host_ prefix but sandbox target",
         category: "skill",
         defaultRiskLevel: "low",
-        origin: "skill" as const,
-        ownerSkillId: "test-skill",
+        owner: { kind: "skill", id: "test-skill" },
         executionTarget: "sandbox" as const,
         input_schema: {},
         execute: async () => {
@@ -158,6 +158,19 @@ mock.module("../tools/registry.js", () => ({
       },
     };
   },
+  // Ownership lives on the registry post-refactor. Mirror that by surfacing
+  // the optional `owner`-shaped field set inline on the override-produced
+  // tool (see the skill_* branches above).
+  getToolOwner: (name: string) => {
+    if (
+      name === "skill_host_tool" ||
+      name === "skill_sandbox_tool" ||
+      name === "host_skill_sandboxed"
+    ) {
+      return { kind: "skill" as const, id: "test-skill" };
+    }
+    return undefined;
+  },
 }));
 
 mock.module("../tools/shared/filesystem/path-policy.js", () => ({

package/src/__tests__/tool-executor.test.ts

@@ -148,6 +148,16 @@ mock.module("../tools/registry.js", () => ({
     };
   },
   getAllTools: () => (getAllToolsOverride ? getAllToolsOverride() : []),
+  // Ownership lives on the registry post-refactor; production reads it via
+  // getToolOwner(name) rather than a field on the Tool object. Mirror that by
+  // surfacing the optional `owner`-shaped field from the override-produced
+  // tool so existing tests can encode owner inline.
+  getToolOwner: (name: string) => {
+    const t = getToolOverride?.(name) as
+      | { owner?: { kind: "skill" | "mcp" | "plugin"; id: string } }
+      | undefined;
+    return t?.owner;
+  },
 }));
 
 mock.module("../tools/shared/filesystem/path-policy.js", () => ({
@@ -224,6 +234,48 @@ describe("ToolExecutor allowedToolNames gating", () => {
     expect(result.content).toBe("ok");
   });
 
+  test("canonicalizes legacy computer_use_press_key alias before active-tool gating", async () => {
+    const executor = new ToolExecutor(makePrompter());
+    const allowed = new Set(["computer_use_key"]);
+    const result = await executor.execute(
+      "computer_use_press_key",
+      {
+        key: "Space",
+        modifiers: ["Command"],
+        reasoning: "Open Spotlight",
+      },
+      makeContext({ allowedToolNames: allowed }),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toBe("ok");
+    expect(lastCheckArgs?.toolName).toBe("computer_use_key");
+    expect(lastCheckArgs?.input).toEqual({
+      key: "cmd+space",
+      reasoning: "Open Spotlight",
+    });
+  });
+
+  test("lowercases legacy computer_use_press_key key-only input", async () => {
+    const executor = new ToolExecutor(makePrompter());
+    const allowed = new Set(["computer_use_key"]);
+    const result = await executor.execute(
+      "computer_use_press_key",
+      {
+        key: "Space",
+        reasoning: "Pause media",
+      },
+      makeContext({ allowedToolNames: allowed }),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(lastCheckArgs?.toolName).toBe("computer_use_key");
+    expect(lastCheckArgs?.input).toEqual({
+      key: "space",
+      reasoning: "Pause media",
+    });
+  });
+
   test("preserves exact active create_app tool before applying compatibility aliases", async () => {
     const executor = new ToolExecutor(makePrompter());
     const allowed = new Set(["create_app", "app_create"]);
@@ -334,8 +386,7 @@ describe("ToolExecutor allowedToolNames gating", () => {
         category: "skill",
         defaultRiskLevel: RiskLevel.Low,
         executionTarget: "sandbox" as const,
-        origin: "skill" as const,
-        ownerSkillId: "my-skill",
+        owner: { kind: "skill", id: "my-skill" },
         input_schema: { type: "object" as const, properties: {} },
         execute: async () => fakeToolResult,
       };
@@ -371,9 +422,7 @@ describe("ToolExecutor policy context plumbing", () => {
         description: "skill tool",
         category: "skill",
         defaultRiskLevel: RiskLevel.Low,
-        origin: "skill" as const,
-        ownerSkillId: "my-skill-123",
-        ownerSkillVersionHash: "abc123hash",
+        owner: { kind: "skill", id: "my-skill-123" },
         executionTarget: "sandbox" as const,
         input_schema: { type: "object" as const, properties: {} },
         execute: async () => fakeToolResult,
@@ -424,7 +473,6 @@ describe("ToolExecutor policy context plumbing", () => {
         category: "core",
         defaultRiskLevel: RiskLevel.Low,
         executionTarget: "sandbox" as const,
-        origin: "core" as const,
         input_schema: { type: "object" as const, properties: {} },
         execute: async () => fakeToolResult,
       };
@@ -453,9 +501,7 @@ describe("ToolExecutor policy context plumbing", () => {
         description: "host skill tool",
         category: "skill",
         defaultRiskLevel: RiskLevel.Low,
-        origin: "skill" as const,
-        ownerSkillId: "host-skill",
-        ownerSkillVersionHash: "host-hash",
+        owner: { kind: "skill", id: "host-skill" },
         executionTarget: "host" as const,
         input_schema: { type: "object" as const, properties: {} },
         execute: async () => fakeToolResult,
@@ -477,7 +523,6 @@ describe("ToolExecutor policy context plumbing", () => {
       executionTarget: "host",
     });
   });
-
 });
 
 // ---------------------------------------------------------------------------