@vellumai/assistant 0.8.10 → 0.8.11-staging.1

package/src/__tests__/tool-audit-listener.test.ts

@@ -1,9 +1,21 @@
-import { describe, expect, test } from "bun:test";
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// Toggle for the collectUsageData opt-out gate the listener consults when
+// populating the telemetry columns.
+let collectUsageData = true;
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({ collectUsageData }),
+}));
 
 import { createToolAuditListener } from "../events/tool-audit-listener.js";
 import type { ToolInvocationRecord } from "../memory/tool-usage-store.js";
 
 describe("tool audit listener", () => {
+  beforeEach(() => {
+    collectUsageData = true;
+  });
+
   test("records executed events with truncated output", () => {
     const records: ToolInvocationRecord[] = [];
     const listener = createToolAuditListener((record) => records.push(record));
@@ -152,5 +164,256 @@ describe("tool audit listener", () => {
     expect(records).toHaveLength(1);
     expect(records[0].result).toBe("error: boom");
     expect(records[0].decision).toBe("error");
+    expect(records[0].argBytes).toBe(
+      Buffer.byteLength(JSON.stringify({ path: "/tmp/secret" }), "utf8"),
+    );
+    expect(records[0].resultBytes).toBe(Buffer.byteLength("error: boom"));
+  });
+
+  test("records byte sizes from the full payloads, before truncation and redaction", () => {
+    const records: ToolInvocationRecord[] = [];
+    const listener = createToolAuditListener((record) => records.push(record));
+
+    listener({
+      type: "executed",
+      toolName: "file_read",
+      input: { path: "/tmp/a" },
+      workingDir: "/tmp",
+      conversationId: "conv-bytes",
+      riskLevel: "low",
+      decision: "allow",
+      durationMs: 12,
+      // 1200 chars: the stored result is capped at 1000 but the size must
+      // reflect the full payload. "é" is 1 char / 2 utf8 bytes, proving
+      // byte (not char) accounting.
+      result: { content: "é".repeat(1200), isError: false },
+    });
+
+    expect(records).toHaveLength(1);
+    expect(records[0].result).toHaveLength(1000);
+    expect(records[0].argBytes).toBe(
+      Buffer.byteLength(JSON.stringify({ path: "/tmp/a" }), "utf8"),
+    );
+    expect(records[0].resultBytes).toBe(2400);
+  });
+
+  test("prefers the executor-stamped raw inputBytes over sizing the (sanitized) event input", () => {
+    const records: ToolInvocationRecord[] = [];
+    const listener = createToolAuditListener((record) => records.push(record));
+
+    // The executor sanitizes event.input before listeners run and stamps
+    // inputBytes from the RAW input — the listener must report that size,
+    // not the redacted payload's.
+    const rawInput = { path: "/tmp/a", token: "t-1" };
+    const rawSize = Buffer.byteLength(JSON.stringify(rawInput), "utf8");
+    const sanitizedInput = { path: "/tmp/a", token: "<redacted />" };
+
+    listener({
+      type: "executed",
+      toolName: "file_read",
+      input: sanitizedInput,
+      inputBytes: rawSize,
+      workingDir: "/tmp",
+      conversationId: "conv-raw-bytes",
+      riskLevel: "low",
+      decision: "allow",
+      durationMs: 4,
+      result: { content: "ok", isError: false },
+    });
+    listener({
+      type: "error",
+      toolName: "file_read",
+      input: sanitizedInput,
+      inputBytes: rawSize,
+      workingDir: "/tmp",
+      conversationId: "conv-raw-bytes",
+      riskLevel: "low",
+      decision: "error",
+      durationMs: 4,
+      errorMessage: "boom",
+      isExpected: false,
+      errorCategory: "tool_failure",
+    });
+
+    expect(records).toHaveLength(2);
+    for (const record of records) {
+      expect(record.argBytes).toBe(rawSize);
+      expect(record.argBytes).not.toBe(
+        Buffer.byteLength(JSON.stringify(sanitizedInput), "utf8"),
+      );
+    }
+  });
+
+  test("prefers the executor-stamped raw resultBytes over sizing the (sanitized) event result", () => {
+    const records: ToolInvocationRecord[] = [];
+    const listener = createToolAuditListener((record) => records.push(record));
+
+    // For sensitive-output tools, the executor sanitizes result.content
+    // before listeners run and stamps resultBytes from the RAW content —
+    // the listener must report that size, not the placeholder-rewritten
+    // payload's.
+    const sanitizedContent = "code: VELLUM_ASSISTANT_INVITE_CODE_AB12CD34";
+    const rawSize = 4096;
+
+    listener({
+      type: "executed",
+      toolName: "create_invite",
+      input: { count: 1 },
+      workingDir: "/tmp",
+      conversationId: "conv-raw-result-bytes",
+      riskLevel: "low",
+      decision: "allow",
+      durationMs: 4,
+      result: { content: sanitizedContent, isError: false },
+      resultBytes: rawSize,
+    });
+
+    expect(records).toHaveLength(1);
+    expect(records[0].resultBytes).toBe(rawSize);
+    expect(records[0].resultBytes).not.toBe(
+      Buffer.byteLength(sanitizedContent, "utf8"),
+    );
+  });
+
+  test("persists NULL telemetry columns when usage data collection is opted out", () => {
+    collectUsageData = false;
+    const records: ToolInvocationRecord[] = [];
+    const listener = createToolAuditListener((record) => records.push(record));
+
+    const attribution = {
+      callSite: "mainAgent" as const,
+      activeProfile: "balanced",
+      overrideProfile: null,
+      callSiteProfile: null,
+      appliedProfile: "balanced",
+      profileSource: "active" as const,
+      resolvedProvider: "anthropic",
+      resolvedModel: "model-a",
+      resolvedMixArm: null,
+    };
+    listener({
+      type: "executed",
+      toolName: "file_read",
+      input: { path: "/tmp/a" },
+      inputBytes: 42,
+      resultBytes: 99,
+      workingDir: "/tmp",
+      conversationId: "conv-opt-out",
+      riskLevel: "low",
+      decision: "allow",
+      durationMs: 12,
+      result: { content: "ok", isError: false },
+      attribution,
+    });
+    listener({
+      type: "error",
+      toolName: "file_read",
+      input: { path: "/tmp/b" },
+      inputBytes: 42,
+      workingDir: "/tmp",
+      conversationId: "conv-opt-out",
+      riskLevel: "low",
+      decision: "error",
+      durationMs: 9,
+      errorMessage: "boom",
+      isExpected: false,
+      errorCategory: "tool_failure",
+      attribution,
+    });
+
+    // Telemetry columns are NULL — the tool_executed projection's
+    // arg_bytes IS NOT NULL filter makes these rows permanently
+    // unreportable, even from a zero watermark.
+    expect(records).toHaveLength(2);
+    for (const record of records) {
+      expect(record.argBytes).toBeNull();
+      expect(record.resultBytes).toBeNull();
+      expect(record.provider).toBeNull();
+      expect(record.model).toBeNull();
+      expect(record.inferenceProfile).toBeNull();
+      expect(record.inferenceProfileSource).toBeNull();
+    }
+    // The audit row itself is unaffected by the opt-out.
+    expect(records[0]).toMatchObject({
+      conversationId: "conv-opt-out",
+      toolName: "file_read",
+      input: JSON.stringify({ path: "/tmp/a" }),
+      result: "ok",
+      decision: "allow",
+      durationMs: 12,
+    });
+    expect(records[1]).toMatchObject({
+      result: "error: boom",
+      decision: "error",
+      durationMs: 9,
+    });
+  });
+
+  test("maps attribution onto executed records and leaves denied records null", () => {
+    const records: ToolInvocationRecord[] = [];
+    const listener = createToolAuditListener((record) => records.push(record));
+
+    listener({
+      type: "executed",
+      toolName: "file_read",
+      input: { path: "/tmp/a" },
+      workingDir: "/tmp",
+      conversationId: "conv-attr",
+      riskLevel: "low",
+      decision: "allow",
+      durationMs: 12,
+      result: { content: "ok", isError: false },
+      attribution: {
+        callSite: "mainAgent",
+        activeProfile: "balanced",
+        overrideProfile: null,
+        callSiteProfile: null,
+        appliedProfile: "balanced",
+        profileSource: "active",
+        resolvedProvider: "anthropic",
+        resolvedModel: "model-a",
+        resolvedMixArm: null,
+      },
+    });
+    listener({
+      type: "executed",
+      toolName: "file_read",
+      input: { path: "/tmp/b" },
+      workingDir: "/tmp",
+      conversationId: "conv-attr",
+      riskLevel: "low",
+      decision: "allow",
+      durationMs: 3,
+      result: { content: "ok", isError: false },
+      attribution: null,
+    });
+    listener({
+      type: "permission_denied",
+      toolName: "bash",
+      input: { command: "rm -rf /tmp" },
+      workingDir: "/tmp",
+      conversationId: "conv-attr",
+      riskLevel: "high",
+      decision: "deny",
+      reason: "Permission denied by user",
+      durationMs: 1,
+    });
+
+    expect(records).toHaveLength(3);
+    expect(records[0]).toMatchObject({
+      provider: "anthropic",
+      model: "model-a",
+      inferenceProfile: "balanced",
+      inferenceProfileSource: "active",
+    });
+    expect(records[1]).toMatchObject({
+      provider: null,
+      model: null,
+      inferenceProfile: null,
+      inferenceProfileSource: null,
+    });
+    expect(records[2].provider).toBeUndefined();
+    expect(records[2].argBytes).toBeUndefined();
+    expect(records[2].resultBytes).toBeUndefined();
   });
 });

package/src/__tests__/tool-error-hook.test.ts

@@ -78,6 +78,7 @@ function makeCtx(
     conversationId: "conv-tool-error-test",
     toolResponse,
     messages,
+    additionalContext: null,
     maxInputTokens: 10_000,
     logger: noopLogger,
   };
@@ -139,7 +140,7 @@ describe("tool-error post-tool-use hook — direct", () => {
     await postToolUse(ctx);
 
     // THEN no coaching is surfaced and the content is left untouched.
-    expect(ctx.additionalContext).toBeUndefined();
+    expect(ctx.additionalContext).toBeNull();
     expect(ctx.toolResponse.content).toBe(BASE_CONTENT);
   });
 
@@ -175,7 +176,7 @@ describe("tool-error post-tool-use hook — direct", () => {
 
     // THEN no coaching is surfaced (the error is likely unrecoverable) and the
     // result is left untouched.
-    expect(ctx.additionalContext).toBeUndefined();
+    expect(ctx.additionalContext).toBeNull();
     expect(ctx.toolResponse.content).toBe(BASE_CONTENT);
   });
 
@@ -251,7 +252,7 @@ describe("tool-error post-tool-use hook — via runHook", () => {
   test("default hook runs before a later-registered user hook", async () => {
     // GIVEN the default plugin is registered first, then a user plugin whose
     // hook records the additionalContext it observes.
-    let observed: string | undefined;
+    let observed: string | null | undefined;
     registerPlugin(defaultToolErrorPlugin);
     registerPlugin({
       manifest: { name: "observer-plugin", version: "0.0.1" },

package/src/__tests__/tool-execution-pipeline.benchmark.test.ts

@@ -75,6 +75,7 @@ mock.module("../util/platform.js", () => ({
 
 mock.module("../permissions/gateway-threshold-reader.js", () => ({
   getAutoApproveThreshold: async () => undefined,
+  refreshAutoApproveThreshold: async () => null,
 }));
 
 mock.module("../permissions/workspace-policy.js", () => ({

package/src/__tests__/tool-executor-lifecycle-events.test.ts

@@ -4,6 +4,7 @@ import type {
   ToolExecutionResult,
   ToolLifecycleEvent,
 } from "../tools/types.js";
+import type { UsageAttributionSnapshot } from "../usage/attribution.js";
 
 const mockConfig = {
   provider: "anthropic",
@@ -33,6 +34,9 @@ const mockConfig = {
   permissions: {
     mode: "workspace" as const,
   },
+  // The audit listener nulls the telemetry columns when this is false; the
+  // end-to-end listener tests below assert the opted-in sizing behavior.
+  collectUsageData: true,
 };
 
 let checkerDecision: "allow" | "prompt" | "deny" = "allow";
@@ -515,6 +519,275 @@ describe("ToolExecutor lifecycle events", () => {
     expect(executed.executionTarget).toBe("sandbox");
   });
 
+  // ── attribution forwarding tests ──────────────────────────
+
+  // Uses a non-main call site (voice turn) so these tests also prove the
+  // executor forwards the snapshot verbatim — non-main turns must not be
+  // rewritten to the main agent's attribution.
+  const testAttribution: UsageAttributionSnapshot = {
+    callSite: "callAgent",
+    activeProfile: "balanced",
+    overrideProfile: null,
+    callSiteProfile: "voice-profile",
+    appliedProfile: "voice-profile",
+    profileSource: "call_site",
+    resolvedProvider: "anthropic",
+    resolvedModel: "test-model",
+    resolvedMixArm: null,
+  };
+
+  test("forwards context.attribution into the executed lifecycle event", async () => {
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    await executor.execute(
+      "file_read",
+      { path: "README.md" },
+      makeContext(events, { attribution: testAttribution }),
+    );
+
+    const executed = events.find((event) => event.type === "executed");
+    if (executed?.type !== "executed")
+      throw new Error("Expected executed event");
+    expect(executed.attribution).toEqual(testAttribution);
+  });
+
+  test("forwards context.attribution into the error lifecycle event", async () => {
+    toolThrow = new Error("boom");
+
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    await executor.execute(
+      "file_read",
+      {},
+      makeContext(events, { attribution: testAttribution }),
+    );
+
+    const errorEvent = events.find((event) => event.type === "error");
+    if (errorEvent?.type !== "error") throw new Error("Expected error event");
+    expect(errorEvent.attribution).toEqual(testAttribution);
+  });
+
+  test("missing context.attribution yields null on the executed event without throwing", async () => {
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    const result = await executor.execute(
+      "file_read",
+      { path: "README.md" },
+      makeContext(events),
+    );
+
+    expect(result).toMatchObject({ content: "ok", isError: false });
+    const executed = events.find((event) => event.type === "executed");
+    if (executed?.type !== "executed")
+      throw new Error("Expected executed event");
+    expect(executed.attribution).toBeNull();
+  });
+
+  test("missing context.attribution yields null on the error event without throwing", async () => {
+    toolThrow = new Error("boom");
+
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    const result = await executor.execute("file_read", {}, makeContext(events));
+
+    expect(result.isError).toBe(true);
+    const errorEvent = events.find((event) => event.type === "error");
+    if (errorEvent?.type !== "error") throw new Error("Expected error event");
+    expect(errorEvent.attribution).toBeNull();
+  });
+
+  test("stamps attribution on pre-execution gate error events (unknown tool)", async () => {
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    const result = await executor.execute(
+      "unknown_tool",
+      { test: true },
+      makeContext(events, { attribution: testAttribution }),
+    );
+
+    expect(result.isError).toBe(true);
+    const errorEvent = events.find((event) => event.type === "error");
+    if (errorEvent?.type !== "error") throw new Error("Expected error event");
+    expect(errorEvent.errorMessage).toContain("Unknown tool: unknown_tool");
+    expect(errorEvent.attribution).toEqual(testAttribution);
+  });
+
+  test("missing attribution yields null on pre-execution gate error events", async () => {
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    const result = await executor.execute(
+      "unknown_tool",
+      { test: true },
+      makeContext(events),
+    );
+
+    expect(result.isError).toBe(true);
+    const errorEvent = events.find((event) => event.type === "error");
+    if (errorEvent?.type !== "error") throw new Error("Expected error event");
+    expect(errorEvent.attribution).toBeNull();
+  });
+
+  test("stamps attribution on the aborted pre-execution gate error event", async () => {
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+    const controller = new AbortController();
+    controller.abort();
+
+    const result = await executor.execute(
+      "file_read",
+      { path: "README.md" },
+      makeContext(events, {
+        attribution: testAttribution,
+        signal: controller.signal,
+      }),
+    );
+
+    expect(result).toEqual({ content: "Cancelled", isError: true });
+    const errorEvent = events.find((event) => event.type === "error");
+    if (errorEvent?.type !== "error") throw new Error("Expected error event");
+    expect(errorEvent.errorMessage).toBe("Cancelled");
+    expect(errorEvent.attribution).toEqual(testAttribution);
+  });
+
+  // ── raw input byte sizing tests ───────────────────────────
+
+  test("stamps inputBytes from the raw input even when sanitization redacts fields", async () => {
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    const rawInput = { path: "README.md", token: "t-1" };
+    const rawSize = Buffer.byteLength(JSON.stringify(rawInput), "utf8");
+
+    await executor.execute("file_read", rawInput, makeContext(events));
+
+    const executed = events.find((event) => event.type === "executed");
+    if (executed?.type !== "executed")
+      throw new Error("Expected executed event");
+    // The event input is sanitized, but the size reflects the raw payload.
+    expect(executed.input.token).toBe("<redacted />");
+    expect(executed.inputBytes).toBe(rawSize);
+    expect(executed.inputBytes).not.toBe(
+      Buffer.byteLength(JSON.stringify(executed.input), "utf8"),
+    );
+  });
+
+  test("stamps inputBytes from the raw input on error events", async () => {
+    toolThrow = new Error("boom");
+
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    const rawInput = { path: "README.md", api_key: "k-1" };
+
+    await executor.execute("file_read", rawInput, makeContext(events));
+
+    const errorEvent = events.find((event) => event.type === "error");
+    if (errorEvent?.type !== "error") throw new Error("Expected error event");
+    expect(errorEvent.input.api_key).toBe("<redacted />");
+    expect(errorEvent.inputBytes).toBe(
+      Buffer.byteLength(JSON.stringify(rawInput), "utf8"),
+    );
+  });
+
+  test("stamps resultBytes from the raw content before sensitive-output sanitization", async () => {
+    // Directive stripping + placeholder substitution shrink the content the
+    // lifecycle event carries; the stamped size must reflect the raw output.
+    const rawContent =
+      'Your invite: <vellum-sensitive-output kind="invite_code" value="SECRET-CODE-123" /> use SECRET-CODE-123';
+    fakeToolResult = { content: rawContent, isError: false };
+
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    await executor.execute("file_read", { path: "a" }, makeContext(events));
+
+    const executed = events.find((event) => event.type === "executed");
+    if (executed?.type !== "executed")
+      throw new Error("Expected executed event");
+    // The event carries the sanitized content...
+    expect(executed.result.content).not.toContain("SECRET-CODE-123");
+    // ...but the stamped size is the raw pre-sanitization byte length.
+    expect(executed.resultBytes).toBe(Buffer.byteLength(rawContent, "utf8"));
+    expect(executed.resultBytes).not.toBe(
+      Buffer.byteLength(executed.result.content, "utf8"),
+    );
+  });
+
+  test("stamps resultBytes for non-sensitive results too (raw equals emitted content)", async () => {
+    const events: ToolLifecycleEvent[] = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    await executor.execute("file_read", { path: "a" }, makeContext(events));
+
+    const executed = events.find((event) => event.type === "executed");
+    if (executed?.type !== "executed")
+      throw new Error("Expected executed event");
+    expect(executed.result.content).toBe("ok");
+    expect(executed.resultBytes).toBe(Buffer.byteLength("ok", "utf8"));
+  });
+
+  test("audit listener records result_bytes from the raw pre-sanitization output", async () => {
+    const { createToolAuditListener } =
+      await import("../events/tool-audit-listener.js");
+    const records: Array<{ resultBytes?: number | null; result: string }> = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    const rawContent =
+      'Code: <vellum-sensitive-output kind="invite_code" value="SECRET-CODE-456" />SECRET-CODE-456';
+    fakeToolResult = { content: rawContent, isError: false };
+
+    await executor.execute(
+      "file_read",
+      { path: "a" },
+      {
+        workingDir: "/tmp/project",
+        conversationId: "conversation-1",
+        trustClass: "guardian" as const,
+        onToolLifecycleEvent: createToolAuditListener((record) =>
+          records.push(record),
+        ),
+      },
+    );
+
+    expect(records).toHaveLength(1);
+    expect(records[0].resultBytes).toBe(Buffer.byteLength(rawContent, "utf8"));
+    // The stored result column is still the sanitized payload.
+    expect(records[0].result).not.toContain("SECRET-CODE-456");
+  });
+
+  test("audit listener records arg_bytes equal to the raw serialized input size", async () => {
+    // End-to-end: executor sanitization must not shrink the recorded
+    // arg_bytes — the audit row sizes the raw pre-redaction input.
+    const { createToolAuditListener } =
+      await import("../events/tool-audit-listener.js");
+    const records: Array<{ argBytes?: number | null; input: string }> = [];
+    const executor = new ToolExecutor(makePrompter());
+
+    const rawInput = { path: "README.md", token: "t-1" };
+
+    await executor.execute("file_read", rawInput, {
+      workingDir: "/tmp/project",
+      conversationId: "conversation-1",
+      trustClass: "guardian" as const,
+      onToolLifecycleEvent: createToolAuditListener((record) =>
+        records.push(record),
+      ),
+    });
+
+    expect(records).toHaveLength(1);
+    expect(records[0].argBytes).toBe(
+      Buffer.byteLength(JSON.stringify(rawInput), "utf8"),
+    );
+    // The stored input column is still the redacted payload.
+    expect(records[0].input).not.toContain("t-1");
+  });
+
   test("skill tool with sandbox execution_target resolves to sandbox executionTarget", async () => {
     const events: ToolLifecycleEvent[] = [];
     const executor = new ToolExecutor(makePrompter());

package/src/__tests__/tool-result-truncate-hook.test.ts

@@ -47,6 +47,7 @@ function makeCtx(content: string): PostToolUseContext {
     conversationId: "conv-test",
     toolResponse: makeToolResponse(content),
     messages: [],
+    additionalContext: null,
     maxInputTokens: MAX_INPUT_TOKENS,
     logger: noopLogger,
   };