@vellumai/assistant 0.8.12-staging.1 → 0.8.12

package/src/__tests__/tool-audit-listener.test.ts

@@ -10,6 +10,10 @@ mock.module("../config/loader.js", () => ({
 
 import { createToolAuditListener } from "../events/tool-audit-listener.js";
 import type { ToolInvocationRecord } from "../memory/tool-usage-store.js";
+import {
+  OPENAI_PROJECT_KEY_REDACTION_MARKER,
+  SYNTHETIC_OPENAI_PROJECT_KEY,
+} from "./secret-fixtures.js";
 
 describe("tool audit listener", () => {
   beforeEach(() => {
@@ -101,6 +105,88 @@ describe("tool audit listener", () => {
     expect(records[0].result).toContain("<redacted");
   });
 
+  test("redacts known-pattern secrets in tool inputs before recording", () => {
+    const records: ToolInvocationRecord[] = [];
+    const listener = createToolAuditListener((record) => records.push(record));
+
+    const input = {
+      command: `export OPENAI_API_KEY="${SYNTHETIC_OPENAI_PROJECT_KEY}"`,
+    };
+
+    listener({
+      type: "executed",
+      toolName: "bash",
+      input,
+      workingDir: "/tmp",
+      conversationId: "conv-input-redact",
+      riskLevel: "low",
+      decision: "allow",
+      durationMs: 5,
+      result: { content: "ok", isError: false },
+    });
+    listener({
+      type: "error",
+      toolName: "bash",
+      input,
+      workingDir: "/tmp",
+      conversationId: "conv-input-redact",
+      riskLevel: "low",
+      decision: "error",
+      durationMs: 5,
+      errorMessage: "boom",
+      isExpected: false,
+      errorCategory: "tool_failure",
+    });
+    listener({
+      type: "permission_denied",
+      toolName: "bash",
+      input,
+      workingDir: "/tmp",
+      conversationId: "conv-input-redact",
+      riskLevel: "high",
+      decision: "deny",
+      reason: "Permission denied by user",
+      durationMs: 5,
+    });
+
+    expect(records).toHaveLength(3);
+    for (const record of records) {
+      expect(record.input).not.toContain(SYNTHETIC_OPENAI_PROJECT_KEY);
+      // The stored input must remain parseable JSON: string leaves are
+      // redacted before stringification, so the marker lands inside a JSON
+      // string value (its quotes escaped) instead of corrupting the
+      // serialized form.
+      const parsed = JSON.parse(record.input) as { command: string };
+      expect(parsed.command).toContain(OPENAI_PROJECT_KEY_REDACTION_MARKER);
+    }
+  });
+
+  test("benign inputs round-trip byte-identical (no false-positive mangling)", () => {
+    const records: ToolInvocationRecord[] = [];
+    const listener = createToolAuditListener((record) => records.push(record));
+
+    const input = {
+      command: "git log --oneline -5 && echo done",
+      cwd: "/tmp/project",
+      env: { LANG: "en_US.UTF-8" },
+    };
+
+    listener({
+      type: "executed",
+      toolName: "bash",
+      input,
+      workingDir: "/tmp",
+      conversationId: "conv-benign-input",
+      riskLevel: "low",
+      decision: "allow",
+      durationMs: 2,
+      result: { content: "ok", isError: false },
+    });
+
+    expect(records).toHaveLength(1);
+    expect(records[0].input).toBe(JSON.stringify(input));
+  });
+
   test("does not redact non-secret content like UUIDs or hashes", () => {
     const records: ToolInvocationRecord[] = [];
     const listener = createToolAuditListener((record) => records.push(record));

package/src/__tests__/workspace-migration-100-upgrade-quality-profile-to-fable-5.test.ts

@@ -0,0 +1,174 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { upgradeQualityProfileToFable5Migration } from "../workspace/migrations/100-upgrade-quality-profile-to-fable-5.js";
+
+let workspaceDir: string;
+
+function writeConfig(data: Record<string, unknown>): void {
+  writeFileSync(
+    join(workspaceDir, "config.json"),
+    JSON.stringify(data, null, 2) + "\n",
+  );
+}
+
+function readConfig(): Record<string, unknown> {
+  return JSON.parse(readFileSync(join(workspaceDir, "config.json"), "utf-8"));
+}
+
+function readProfiles(): Record<string, Record<string, unknown>> {
+  const llm = readConfig().llm as Record<string, unknown>;
+  return llm.profiles as Record<string, Record<string, unknown>>;
+}
+
+beforeEach(() => {
+  workspaceDir = join(
+    tmpdir(),
+    `vellum-migration-100-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+  mkdirSync(workspaceDir, { recursive: true });
+});
+
+afterEach(() => {
+  if (existsSync(workspaceDir)) {
+    rmSync(workspaceDir, { recursive: true, force: true });
+  }
+});
+
+describe("100-upgrade-quality-profile-to-fable-5 migration", () => {
+  test("no-op when config.json does not exist", () => {
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    expect(existsSync(join(workspaceDir, "config.json"))).toBe(false);
+  });
+
+  test("no-op when config has no llm.profiles", () => {
+    const original = { llm: { default: { provider: "anthropic" } } };
+    writeConfig(original);
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    expect(readConfig()).toEqual(original);
+  });
+
+  test("gracefully handles invalid JSON", () => {
+    writeFileSync(join(workspaceDir, "config.json"), "not-valid-json");
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    expect(readFileSync(join(workspaceDir, "config.json"), "utf-8")).toBe(
+      "not-valid-json",
+    );
+  });
+
+  test("upgrades quality-optimized from claude-opus-4-7", () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          "quality-optimized": {
+            provider: "anthropic",
+            model: "claude-opus-4-7",
+            label: "Quality",
+          },
+        },
+      },
+    });
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    const profile = readProfiles()["quality-optimized"]!;
+    expect(profile.model).toBe("claude-fable-5");
+    expect(profile.label).toBe("Quality");
+  });
+
+  test("upgrades the managed profile but not the user-owned custom copy", () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          "quality-optimized": {
+            provider: "anthropic",
+            model: "claude-opus-4-8",
+          },
+          "custom-quality-optimized": {
+            provider: "anthropic",
+            model: "claude-opus-4-8",
+          },
+        },
+      },
+    });
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    const profiles = readProfiles();
+    expect(profiles["quality-optimized"]!.model).toBe("claude-fable-5");
+    expect(profiles["custom-quality-optimized"]!.model).toBe("claude-opus-4-8");
+  });
+
+  test("upgrades OpenRouter-prefixed model ids", () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          "quality-optimized": {
+            provider: "openrouter",
+            model: "anthropic/claude-opus-4.8",
+          },
+        },
+      },
+    });
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    expect(readProfiles()["quality-optimized"]!.model).toBe(
+      "anthropic/claude-fable-5",
+    );
+  });
+
+  test("leaves user-customized models untouched", () => {
+    const original = {
+      llm: {
+        profiles: {
+          "quality-optimized": { provider: "openai", model: "gpt-5.4" },
+        },
+      },
+    };
+    writeConfig(original);
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    expect(readConfig()).toEqual(original);
+  });
+
+  test("leaves non-quality profiles untouched", () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          balanced: { provider: "anthropic", model: "claude-opus-4-8" },
+          "quality-optimized": {
+            provider: "anthropic",
+            model: "claude-opus-4-8",
+          },
+        },
+      },
+    });
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    const profiles = readProfiles();
+    expect(profiles["balanced"]!.model).toBe("claude-opus-4-8");
+    expect(profiles["quality-optimized"]!.model).toBe("claude-fable-5");
+  });
+
+  test("idempotency: no-op on already-upgraded config (no writes)", () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          "quality-optimized": {
+            provider: "anthropic",
+            model: "claude-fable-5",
+          },
+        },
+      },
+    });
+    const beforeContent = readFileSync(
+      join(workspaceDir, "config.json"),
+      "utf-8",
+    );
+    upgradeQualityProfileToFable5Migration.run(workspaceDir);
+    expect(readFileSync(join(workspaceDir, "config.json"), "utf-8")).toBe(
+      beforeContent,
+    );
+  });
+});

package/src/__tests__/workspace-migration-101-upgrade-balanced-economy-to-minimax-m3.test.ts

@@ -0,0 +1,162 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { upgradeBalancedEconomyToMinimaxM3Migration } from "../workspace/migrations/101-upgrade-balanced-economy-to-minimax-m3.js";
+
+let workspaceDir: string;
+
+function writeConfig(data: Record<string, unknown>): void {
+  writeFileSync(
+    join(workspaceDir, "config.json"),
+    JSON.stringify(data, null, 2) + "\n",
+  );
+}
+
+function readConfig(): Record<string, unknown> {
+  return JSON.parse(readFileSync(join(workspaceDir, "config.json"), "utf-8"));
+}
+
+function readProfiles(): Record<string, Record<string, unknown>> {
+  const llm = readConfig().llm as Record<string, unknown>;
+  return llm.profiles as Record<string, Record<string, unknown>>;
+}
+
+beforeEach(() => {
+  workspaceDir = join(
+    tmpdir(),
+    `vellum-migration-101-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+  mkdirSync(workspaceDir, { recursive: true });
+});
+
+afterEach(() => {
+  if (existsSync(workspaceDir)) {
+    rmSync(workspaceDir, { recursive: true, force: true });
+  }
+});
+
+describe("101-upgrade-balanced-economy-to-minimax-m3 migration", () => {
+  test("no-op when config.json does not exist", () => {
+    upgradeBalancedEconomyToMinimaxM3Migration.run(workspaceDir);
+    expect(existsSync(join(workspaceDir, "config.json"))).toBe(false);
+  });
+
+  test("no-op when config has no llm.profiles", () => {
+    const original = { llm: { default: { provider: "fireworks" } } };
+    writeConfig(original);
+    upgradeBalancedEconomyToMinimaxM3Migration.run(workspaceDir);
+    expect(readConfig()).toEqual(original);
+  });
+
+  test("gracefully handles invalid JSON", () => {
+    writeFileSync(join(workspaceDir, "config.json"), "not-valid-json");
+    upgradeBalancedEconomyToMinimaxM3Migration.run(workspaceDir);
+    expect(readFileSync(join(workspaceDir, "config.json"), "utf-8")).toBe(
+      "not-valid-json",
+    );
+  });
+
+  test("upgrades balanced-economy from Kimi K2.6 and drops the logit-bias preset", () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          "balanced-economy": {
+            provider: "fireworks",
+            provider_connection: "fireworks-managed",
+            model: "accounts/fireworks/models/kimi-k2p6",
+            label: "Balanced Economy",
+            description: "Strong open model (Kimi K2.6) at a lower price point",
+            maxTokens: 16000,
+            effort: "high",
+            thinking: { enabled: true, streamThinking: true },
+            contextWindow: { maxInputTokens: 200000 },
+            logitBias: "suppress-cjk",
+          },
+        },
+      },
+    });
+    upgradeBalancedEconomyToMinimaxM3Migration.run(workspaceDir);
+    const profile = readProfiles()["balanced-economy"]!;
+    expect(profile.model).toBe("accounts/fireworks/models/minimax-m3");
+    expect(profile.description).toBe(
+      "Strong open model (MiniMax M3) at a lower price point",
+    );
+    expect(profile.maxTokens).toBe(32000);
+    expect(profile.contextWindow).toEqual({ maxInputTokens: 200000 });
+    expect("logitBias" in profile).toBe(false);
+    expect(profile.label).toBe("Balanced Economy");
+    expect(profile.effort).toBe("high");
+    expect(profile.thinking).toEqual({ enabled: true, streamThinking: true });
+  });
+
+  test("leaves user-customized models untouched", () => {
+    const original = {
+      llm: {
+        profiles: {
+          "balanced-economy": {
+            provider: "fireworks",
+            model: "accounts/fireworks/models/deepseek-v4-pro",
+            logitBias: "suppress-cjk",
+          },
+        },
+      },
+    };
+    writeConfig(original);
+    upgradeBalancedEconomyToMinimaxM3Migration.run(workspaceDir);
+    expect(readConfig()).toEqual(original);
+  });
+
+  test("leaves other profiles untouched", () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          "my-kimi": {
+            provider: "fireworks",
+            model: "accounts/fireworks/models/kimi-k2p6",
+          },
+          "balanced-economy": {
+            provider: "fireworks",
+            model: "accounts/fireworks/models/kimi-k2p6",
+          },
+        },
+      },
+    });
+    upgradeBalancedEconomyToMinimaxM3Migration.run(workspaceDir);
+    const profiles = readProfiles();
+    expect(profiles["my-kimi"]!.model).toBe(
+      "accounts/fireworks/models/kimi-k2p6",
+    );
+    expect(profiles["balanced-economy"]!.model).toBe(
+      "accounts/fireworks/models/minimax-m3",
+    );
+  });
+
+  test("idempotency: no-op on already-upgraded config (no writes)", () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          "balanced-economy": {
+            provider: "fireworks",
+            model: "accounts/fireworks/models/minimax-m3",
+          },
+        },
+      },
+    });
+    const beforeContent = readFileSync(
+      join(workspaceDir, "config.json"),
+      "utf-8",
+    );
+    upgradeBalancedEconomyToMinimaxM3Migration.run(workspaceDir);
+    expect(readFileSync(join(workspaceDir, "config.json"), "utf-8")).toBe(
+      beforeContent,
+    );
+  });
+});