@vellumai/assistant 0.6.5 → 0.6.6

package/src/__tests__/tool-executor.test.ts

@@ -86,6 +86,16 @@ let checkFnOverride:
 /** Override for generateScopeOptions — when set, returns this value instead of the default. */
 let scopeOptionsOverride: ScopeOption[] | undefined;
 
+/** Override for getCachedAssessment — when set, returns this value. */
+let cachedAssessmentOverride:
+  | {
+      riskLevel: string;
+      reason: string;
+      scopeOptions: Array<{ pattern: string; label: string }>;
+      matchType: string;
+    }
+  | undefined;
+
 /** Spy on addRule to capture calls without replacing the real implementation. */
 let addRuleSpy: ReturnType<typeof spyOn> | undefined;
 
@@ -127,6 +137,7 @@ mock.module("../permissions/checker.js", () => ({
   ],
   generateScopeOptions: () =>
     scopeOptionsOverride ?? [{ label: "/tmp", scope: "/tmp" }],
+  getCachedAssessment: () => cachedAssessmentOverride,
 }));
 
 // Mock every export so downstream test files that dynamically import modules
@@ -196,6 +207,7 @@ describe("ToolExecutor allowedToolNames gating", () => {
     getToolOverride = undefined;
     checkResultOverride = undefined;
     checkFnOverride = undefined;
+    cachedAssessmentOverride = undefined;
     if (addRuleSpy) {
       addRuleSpy.mockRestore();
       addRuleSpy = undefined;
@@ -272,6 +284,7 @@ describe("ToolExecutor policy context plumbing", () => {
     getToolOverride = undefined;
     checkResultOverride = undefined;
     checkFnOverride = undefined;
+    cachedAssessmentOverride = undefined;
     if (addRuleSpy) {
       addRuleSpy.mockRestore();
       addRuleSpy = undefined;
@@ -309,6 +322,7 @@ describe("ToolExecutor policy context plumbing", () => {
     expect(result.isError).toBe(false);
     expect(lastCheckArgs).toBeDefined();
     expect(lastCheckArgs!.policyContext).toEqual({
+      conversationId: "conversation-1",
       executionContext: "conversation",
       ephemeralRules: undefined,
       executionTarget: "sandbox",
@@ -329,6 +343,7 @@ describe("ToolExecutor policy context plumbing", () => {
     expect(result.isError).toBe(false);
     expect(lastCheckArgs).toBeDefined();
     expect(lastCheckArgs!.policyContext).toEqual({
+      conversationId: "conversation-1",
       executionContext: "conversation",
       ephemeralRules: undefined,
     });
@@ -362,6 +377,7 @@ describe("ToolExecutor policy context plumbing", () => {
     expect(result.isError).toBe(false);
     expect(lastCheckArgs).toBeDefined();
     expect(lastCheckArgs!.policyContext).toEqual({
+      conversationId: "conversation-1",
       executionContext: "conversation",
       ephemeralRules: undefined,
     });
@@ -398,6 +414,7 @@ describe("ToolExecutor policy context plumbing", () => {
     expect(result.isError).toBe(false);
     expect(lastCheckArgs).toBeDefined();
     expect(lastCheckArgs!.policyContext).toEqual({
+      conversationId: "conversation-1",
       executionContext: "conversation",
       ephemeralRules: undefined,
       executionTarget: "host",
@@ -430,6 +447,7 @@ describe("ToolExecutor policy context plumbing", () => {
     expect(result.isError).toBe(false);
     expect(lastCheckArgs).toBeDefined();
     expect(lastCheckArgs!.policyContext).toEqual({
+      conversationId: "conversation-1",
       executionContext: "conversation",
       ephemeralRules: undefined,
       executionTarget: undefined,
@@ -749,6 +767,7 @@ describe("ToolExecutor strict mode + high-risk integration (PR 25)", () => {
     getToolOverride = undefined;
     checkResultOverride = undefined;
     checkFnOverride = undefined;
+    cachedAssessmentOverride = undefined;
     if (addRuleSpy) {
       addRuleSpy.mockRestore();
       addRuleSpy = undefined;
@@ -901,6 +920,7 @@ describe("ToolExecutor strict mode + high-risk integration (PR 25)", () => {
 
     expect(lastCheckArgs).toBeDefined();
     expect(lastCheckArgs!.policyContext).toEqual({
+      conversationId: "conversation-1",
       executionContext: "conversation",
       ephemeralRules: undefined,
       executionTarget: "sandbox",
@@ -1085,6 +1105,7 @@ describe("ToolExecutor strict mode + high-risk integration (PR 25)", () => {
 
     expect(lastCheckArgs).toBeDefined();
     expect(lastCheckArgs!.policyContext).toEqual({
+      conversationId: "conversation-1",
       executionContext: "conversation",
       ephemeralRules: undefined,
       executionTarget: "sandbox",
@@ -1252,6 +1273,7 @@ describe("ToolExecutor baseline: allow rule auto-allows file_edit guardian perso
     getToolOverride = undefined;
     checkResultOverride = undefined;
     checkFnOverride = undefined;
+    cachedAssessmentOverride = undefined;
     if (addRuleSpy) {
       addRuleSpy.mockRestore();
       addRuleSpy = undefined;
@@ -1369,6 +1391,7 @@ describe("ToolExecutor forcePromptSideEffects enforcement", () => {
     getToolOverride = undefined;
     checkResultOverride = undefined;
     checkFnOverride = undefined;
+    cachedAssessmentOverride = undefined;
     promptCalled = false;
     if (addRuleSpy) {
       addRuleSpy.mockRestore();
@@ -2124,6 +2147,7 @@ describe("ToolExecutor persistent-allow lifecycle", () => {
     getToolOverride = undefined;
     checkResultOverride = undefined;
     checkFnOverride = undefined;
+    cachedAssessmentOverride = undefined;
     if (addRuleSpy) {
       addRuleSpy.mockRestore();
       addRuleSpy = undefined;
@@ -2302,3 +2326,120 @@ describe("integration regressions — prompt payload (PR 11)", () => {
     expect(capturedScopes![0]).toHaveProperty("scope");
   });
 });
+
+// ---------------------------------------------------------------------------
+// Risk metadata on ToolExecutionResult (PR 5 — scope-ladder-v1)
+// ---------------------------------------------------------------------------
+
+describe("ToolExecutionResult includes risk metadata from classifier assessment", () => {
+  beforeEach(() => {
+    fakeToolResult = { content: "ok", isError: false };
+    lastCheckArgs = undefined;
+    getToolOverride = undefined;
+    checkResultOverride = undefined;
+    checkFnOverride = undefined;
+    cachedAssessmentOverride = undefined;
+    if (addRuleSpy) {
+      addRuleSpy.mockRestore();
+      addRuleSpy = undefined;
+    }
+  });
+
+  test("auto-approved tool result includes risk metadata when classifier assessment exists", async () => {
+    cachedAssessmentOverride = {
+      riskLevel: "medium",
+      reason: "Writes to a file outside the workspace",
+      scopeOptions: [
+        { pattern: "file_write:/tmp/test.txt", label: "This file only" },
+        { pattern: "file_write:/tmp/**", label: "Anything in tmp/" },
+      ],
+      matchType: "registry",
+    };
+
+    const executor = new ToolExecutor(makePrompter());
+    const result = await executor.execute(
+      "file_read",
+      { path: "README.md" },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(result.riskLevel).toBe("medium");
+    expect(result.riskReason).toBe("Writes to a file outside the workspace");
+    expect(result.riskScopeOptions).toEqual([
+      { pattern: "file_write:/tmp/test.txt", label: "This file only" },
+      { pattern: "file_write:/tmp/**", label: "Anything in tmp/" },
+    ]);
+  });
+
+  test("tool result omits risk metadata when no classifier assessment exists (e.g. MCP tools)", async () => {
+    // cachedAssessmentOverride is undefined (no classifier ran)
+    const executor = new ToolExecutor(makePrompter());
+    const result = await executor.execute(
+      "file_read",
+      { path: "README.md" },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(result.riskLevel).toBeUndefined();
+    expect(result.riskReason).toBeUndefined();
+    expect(result.riskScopeOptions).toBeUndefined();
+  });
+
+  test("denied tool result includes risk metadata", async () => {
+    checkResultOverride = {
+      decision: "deny",
+      reason: "Blocked by deny rule",
+    };
+    cachedAssessmentOverride = {
+      riskLevel: "high",
+      reason: "Recursive force delete",
+      scopeOptions: [{ pattern: "bash:rm -rf*", label: "rm -rf commands" }],
+      matchType: "registry",
+    };
+
+    const executor = new ToolExecutor(makePrompter());
+    const result = await executor.execute(
+      "bash",
+      { command: "rm -rf /" },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(true);
+    expect(result.riskLevel).toBe("high");
+    expect(result.riskReason).toBe("Recursive force delete");
+    expect(result.riskScopeOptions).toEqual([
+      { pattern: "bash:rm -rf*", label: "rm -rf commands" },
+    ]);
+  });
+
+  test("prompted-then-approved tool result includes risk metadata", async () => {
+    checkResultOverride = {
+      decision: "prompt",
+      reason: "Medium risk: requires approval",
+    };
+    cachedAssessmentOverride = {
+      riskLevel: "medium",
+      reason: "Package manager installation",
+      scopeOptions: [
+        { pattern: "bash:npm install*", label: "npm install commands" },
+        { pattern: "bash:npm*", label: "All npm commands" },
+      ],
+      matchType: "registry",
+    };
+
+    const executor = new ToolExecutor(makePrompter());
+    const result = await executor.execute(
+      "bash",
+      { command: "npm install lodash" },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toBe("ok");
+    expect(result.riskLevel).toBe("medium");
+    expect(result.riskReason).toBe("Package manager installation");
+    expect(result.riskScopeOptions).toHaveLength(2);
+  });
+});

package/src/__tests__/tool-result-truncate-pipeline.test.ts

@@ -0,0 +1,356 @@
+/**
+ * Tests for the `toolResultTruncate` plugin pipeline (PR 17).
+ *
+ * Covers:
+ * - The default terminal delegates to `truncateToolResultText`, producing
+ *   byte-for-byte identical output to calling the helper directly across
+ *   short, long, and newline-bounded inputs (property-style).
+ * - The pipeline routes through `runPipeline` with the
+ *   `DEFAULT_TIMEOUTS.toolResultTruncate` budget and returns a
+ *   `{ content, truncated }` pair whose `truncated` flag matches whether
+ *   the content actually changed.
+ * - Plugins registered alongside the default can short-circuit or decorate
+ *   the terminal's output.
+ * - A user plugin registered AFTER the default still runs — the default
+ *   plugin's middleware is a passthrough, so the onion composition cannot
+ *   shadow late-registered middleware.
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import {
+  MIN_KEEP_CHARS,
+  truncateToolResultText,
+  TRUNCATION_SUFFIX,
+} from "../context/tool-result-truncation.js";
+import type { TrustContext } from "../daemon/conversation-runtime-assembly.js";
+import {
+  defaultToolResultTruncatePlugin,
+  defaultToolResultTruncateTerminal,
+} from "../plugins/defaults/tool-result-truncate.js";
+import { DEFAULT_TIMEOUTS, runPipeline } from "../plugins/pipeline.js";
+import {
+  getMiddlewaresFor,
+  registerPlugin,
+  resetPluginRegistryForTests,
+} from "../plugins/registry.js";
+import {
+  type Middleware,
+  type ToolResultTruncateArgs,
+  type ToolResultTruncateResult,
+  type TurnContext,
+} from "../plugins/types.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+const trust: TrustContext = {
+  sourceChannel: "vellum",
+  trustClass: "guardian",
+};
+
+function makeCtx(overrides: Partial<TurnContext> = {}): TurnContext {
+  return {
+    requestId: "req-test",
+    conversationId: "conv-test",
+    turnIndex: 0,
+    trust,
+    ...overrides,
+  };
+}
+
+describe("toolResultTruncate pipeline", () => {
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+  });
+
+  // -------------------------------------------------------------------------
+  // Default terminal — isolated (no pipeline runner)
+  // -------------------------------------------------------------------------
+
+  describe("default terminal", () => {
+    test("passes short content through unchanged with truncated=false", () => {
+      const content = "hello world";
+      const result = defaultToolResultTruncateTerminal({
+        content,
+        maxChars: 100,
+      });
+      expect(result.content).toBe(content);
+      expect(result.truncated).toBe(false);
+    });
+
+    test("truncates oversize content and reports truncated=true", () => {
+      const content = "a".repeat(10_000);
+      const maxChars = 5_000;
+      const expected = truncateToolResultText(content, maxChars);
+      const result = defaultToolResultTruncateTerminal({ content, maxChars });
+      expect(result.content).toBe(expected);
+      expect(result.truncated).toBe(true);
+      expect(result.content).toContain(TRUNCATION_SUFFIX);
+    });
+
+    test("snaps to newline boundary identically to truncateToolResultText", () => {
+      const lines = Array.from(
+        { length: 1_000 },
+        (_, i) => `line ${i}: ${"x".repeat(20)}`,
+      ).join("\n");
+      const maxChars = 5_000;
+      const expected = truncateToolResultText(lines, maxChars);
+      const result = defaultToolResultTruncateTerminal({
+        content: lines,
+        maxChars,
+      });
+      expect(result.content).toBe(expected);
+      expect(result.truncated).toBe(true);
+    });
+
+    test("returns truncated=false when effectiveMax keeps the full text (maxChars < MIN_KEEP_CHARS case)", () => {
+      const textLength = MIN_KEEP_CHARS - TRUNCATION_SUFFIX.length - 10;
+      const content = "a".repeat(textLength);
+      const maxChars = 100;
+      const expected = truncateToolResultText(content, maxChars);
+      const result = defaultToolResultTruncateTerminal({ content, maxChars });
+      // Helper returns the original text unchanged in this case.
+      expect(expected).toBe(content);
+      expect(result.content).toBe(content);
+      expect(result.truncated).toBe(false);
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // End-to-end: default plugin routed through runPipeline
+  // -------------------------------------------------------------------------
+
+  describe("runPipeline with the default plugin registered", () => {
+    async function runDefault(
+      content: string,
+      maxChars: number,
+    ): Promise<ToolResultTruncateResult> {
+      registerPlugin(defaultToolResultTruncatePlugin);
+      const middlewares = getMiddlewaresFor("toolResultTruncate");
+      return runPipeline<ToolResultTruncateArgs, ToolResultTruncateResult>(
+        "toolResultTruncate",
+        middlewares,
+        async (args) => defaultToolResultTruncateTerminal(args),
+        { content, maxChars },
+        makeCtx(),
+        DEFAULT_TIMEOUTS.toolResultTruncate,
+      );
+    }
+
+    test("short content round-trip matches truncateToolResultText", async () => {
+      const content = "quick brown fox";
+      const maxChars = 200;
+      const expected = truncateToolResultText(content, maxChars);
+      const result = await runDefault(content, maxChars);
+      expect(result.content).toBe(expected);
+      expect(result.truncated).toBe(false);
+    });
+
+    test("long content round-trip matches truncateToolResultText", async () => {
+      const content = "z".repeat(50_000);
+      const maxChars = 10_000;
+      const expected = truncateToolResultText(content, maxChars);
+      const result = await runDefault(content, maxChars);
+      expect(result.content).toBe(expected);
+      expect(result.truncated).toBe(true);
+      expect(result.content).toContain(TRUNCATION_SUFFIX);
+    });
+
+    test("newline-bounded content round-trip matches truncateToolResultText", async () => {
+      const lines = Array.from(
+        { length: 500 },
+        (_, i) => `line ${i}: ${"y".repeat(40)}`,
+      ).join("\n");
+      const maxChars = 4_000;
+      const expected = truncateToolResultText(lines, maxChars);
+      const result = await runDefault(lines, maxChars);
+      expect(result.content).toBe(expected);
+      expect(result.truncated).toBe(true);
+    });
+
+    test("property test: default pipeline output equals direct truncateToolResultText across varied inputs", async () => {
+      // Deterministic pseudo-random over a fixed seed — bun's test runner
+      // doesn't ship a property-test library, so we hand-roll a tiny LCG
+      // that produces enough spread for a meaningful regression signal
+      // without introducing a dependency.
+      let seed = 0xc0ffee;
+      const rand = () => {
+        seed = (seed * 1664525 + 1013904223) & 0xffffffff;
+        return (seed >>> 0) / 0x100000000;
+      };
+      const alphabet = "abcdefghijklmnopqrstuvwxyz \n";
+
+      const cases: Array<{ content: string; maxChars: number }> = [];
+      for (let i = 0; i < 40; i++) {
+        // Lengths span short, boundary, and long relative to the maxChars
+        // budget so the property covers pass-through, newline-snap, and
+        // pure tail-drop paths.
+        const length = Math.floor(rand() * 20_000);
+        let content = "";
+        for (let j = 0; j < length; j++) {
+          content += alphabet[Math.floor(rand() * alphabet.length)];
+        }
+        const maxChars = 1_000 + Math.floor(rand() * 9_000);
+        cases.push({ content, maxChars });
+      }
+
+      // Register once outside the loop — registry is reset in `beforeEach`,
+      // so the per-case reset lives in the loop instead.
+      for (const { content, maxChars } of cases) {
+        resetPluginRegistryForTests();
+        registerPlugin(defaultToolResultTruncatePlugin);
+        const middlewares = getMiddlewaresFor("toolResultTruncate");
+        const result = await runPipeline<
+          ToolResultTruncateArgs,
+          ToolResultTruncateResult
+        >(
+          "toolResultTruncate",
+          middlewares,
+          async (args) => defaultToolResultTruncateTerminal(args),
+          { content, maxChars },
+          makeCtx(),
+          DEFAULT_TIMEOUTS.toolResultTruncate,
+        );
+        const expected = truncateToolResultText(content, maxChars);
+        expect(result.content).toBe(expected);
+        expect(result.truncated).toBe(expected !== content);
+      }
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // Middleware composition — an outer plugin can intercept / transform
+  // -------------------------------------------------------------------------
+
+  describe("middleware composition", () => {
+    test("an outer plugin can short-circuit the default with its own content", async () => {
+      const shortCircuit: Middleware<
+        ToolResultTruncateArgs,
+        ToolResultTruncateResult
+      > = async (_args, _next, _ctx) => {
+        return { content: "SUMMARY", truncated: true };
+      };
+      registerPlugin({
+        manifest: {
+          name: "short-circuit",
+          version: "1.0.0",
+          requires: {
+            pluginRuntime: "v1",
+            toolResultTruncateApi: "v1",
+          },
+        },
+        middleware: { toolResultTruncate: shortCircuit },
+      });
+      registerPlugin(defaultToolResultTruncatePlugin);
+
+      const middlewares = getMiddlewaresFor("toolResultTruncate");
+      const result = await runPipeline<
+        ToolResultTruncateArgs,
+        ToolResultTruncateResult
+      >(
+        "toolResultTruncate",
+        middlewares,
+        async (args) => defaultToolResultTruncateTerminal(args),
+        { content: "a".repeat(10_000), maxChars: 5_000 },
+        makeCtx(),
+        DEFAULT_TIMEOUTS.toolResultTruncate,
+      );
+
+      expect(result.content).toBe("SUMMARY");
+      expect(result.truncated).toBe(true);
+    });
+
+    test("an outer plugin can observe and mutate the default's output", async () => {
+      const prefixer: Middleware<
+        ToolResultTruncateArgs,
+        ToolResultTruncateResult
+      > = async (args, next, _ctx) => {
+        const inner = await next(args);
+        return { ...inner, content: `[wrapped] ${inner.content}` };
+      };
+      registerPlugin({
+        manifest: {
+          name: "prefixer",
+          version: "1.0.0",
+          requires: {
+            pluginRuntime: "v1",
+            toolResultTruncateApi: "v1",
+          },
+        },
+        middleware: { toolResultTruncate: prefixer },
+      });
+      registerPlugin(defaultToolResultTruncatePlugin);
+
+      const middlewares = getMiddlewaresFor("toolResultTruncate");
+      const content = "hello";
+      const result = await runPipeline<
+        ToolResultTruncateArgs,
+        ToolResultTruncateResult
+      >(
+        "toolResultTruncate",
+        middlewares,
+        async (args) => defaultToolResultTruncateTerminal(args),
+        { content, maxChars: 100 },
+        makeCtx(),
+        DEFAULT_TIMEOUTS.toolResultTruncate,
+      );
+
+      expect(result.content).toBe(`[wrapped] ${content}`);
+      expect(result.truncated).toBe(false);
+    });
+
+    test("user plugin registered AFTER the default still runs (no shadowing)", async () => {
+      // Production registration order: defaults load first via the
+      // side-effect imports in `defaults/index.ts`, then user plugins
+      // register on top (via `bootstrapPlugins()` or hot-reload). The
+      // user's middleware ends up at a deeper onion layer than the
+      // default. If the default's middleware were to bypass `next` and
+      // call the terminal directly, the user middleware would never run
+      // — this test guards against that regression.
+      registerPlugin(defaultToolResultTruncatePlugin);
+
+      let userMiddlewareRan = false;
+      const userMiddleware: Middleware<
+        ToolResultTruncateArgs,
+        ToolResultTruncateResult
+      > = async (args, next) => {
+        userMiddlewareRan = true;
+        return next(args);
+      };
+      registerPlugin({
+        manifest: {
+          name: "late-user-plugin",
+          version: "0.0.1",
+          requires: {
+            pluginRuntime: "v1",
+            toolResultTruncateApi: "v1",
+          },
+        },
+        middleware: { toolResultTruncate: userMiddleware },
+      });
+
+      const middlewares = getMiddlewaresFor("toolResultTruncate");
+      const content = "a".repeat(10_000);
+      const maxChars = 5_000;
+      const result = await runPipeline<
+        ToolResultTruncateArgs,
+        ToolResultTruncateResult
+      >(
+        "toolResultTruncate",
+        middlewares,
+        async (args) => defaultToolResultTruncateTerminal(args),
+        { content, maxChars },
+        makeCtx(),
+        DEFAULT_TIMEOUTS.toolResultTruncate,
+      );
+
+      expect(userMiddlewareRan).toBe(true);
+      // Terminal still runs after the user passthrough, so output matches
+      // the direct helper.
+      expect(result.content).toBe(truncateToolResultText(content, maxChars));
+      expect(result.truncated).toBe(true);
+    });
+  });
+});