@vellumai/assistant 0.8.11 → 0.8.12-staging.2

package/src/__tests__/tool-result-spool.test.ts

@@ -0,0 +1,337 @@
+import {
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import type { AgentEvent } from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import {
+  getToolResultFilePath,
+  postTurnTruncateToolResults,
+  THRESHOLD_CHARS,
+  TOOL_RESULT_DIR,
+  TRUNCATION_MARKER,
+} from "../context/post-turn-tool-result-truncation.js";
+import { spoolAndStubOversizedToolResults } from "../context/tool-result-spool.js";
+import { resetPluginRegistryAndRegisterDefaults } from "../plugins/defaults/index.js";
+import type {
+  ContentBlock,
+  Message,
+  ToolDefinition,
+} from "../providers/types.js";
+import {
+  createMockProvider,
+  textResponse,
+  toolUseResponse,
+} from "./helpers/mock-provider.js";
+
+function makeToolResult(
+  content: string,
+  toolUseId = "tool_use_1",
+  is_error = false,
+): ContentBlock {
+  return {
+    type: "tool_result" as const,
+    tool_use_id: toolUseId,
+    content,
+    ...(is_error ? { is_error: true } : {}),
+  };
+}
+
+const LONG = "x".repeat(THRESHOLD_CHARS + 100);
+const SHORT = "y".repeat(100);
+
+describe("spoolAndStubOversizedToolResults", () => {
+  let convDir: string;
+  const noName = () => undefined;
+
+  beforeEach(() => {
+    convDir = mkdtempSync(join(tmpdir(), "tool-result-spool-"));
+  });
+
+  afterEach(() => {
+    rmSync(convDir, { recursive: true, force: true });
+  });
+
+  test("spools an oversized result and swaps in the stub", () => {
+    const blocks = [makeToolResult(LONG, "tu_big")];
+
+    const count = spoolAndStubOversizedToolResults(blocks, {
+      conversationDir: convDir,
+      toolNameById: noName,
+    });
+
+    expect(count).toBe(1);
+    const filePath = getToolResultFilePath(convDir, "tu_big");
+    expect(existsSync(filePath)).toBe(true);
+    expect(readFileSync(filePath, "utf-8")).toBe(LONG);
+    const stubbed = blocks[0] as { content: string };
+    expect(stubbed.content).toContain(TRUNCATION_MARKER);
+    expect(stubbed.content).toContain(filePath);
+    expect(stubbed.content.length).toBeLessThan(LONG.length);
+  });
+
+  test("stub is byte-identical to the post-turn truncation stub", () => {
+    const blocks = [makeToolResult(LONG, "tu_big")];
+    spoolAndStubOversizedToolResults(blocks, {
+      conversationDir: convDir,
+      toolNameById: noName,
+    });
+
+    const { messages: postTurn } = postTurnTruncateToolResults(
+      [{ role: "user", content: [makeToolResult(LONG, "tu_big")] }],
+      { conversationDir: convDir },
+    );
+
+    expect((blocks[0] as { content: string }).content).toBe(
+      (postTurn[0].content[0] as { content: string }).content,
+    );
+  });
+
+  test("post-turn pass is a no-op on result-time stubs", () => {
+    const blocks = [makeToolResult(LONG, "tu_big")];
+    spoolAndStubOversizedToolResults(blocks, {
+      conversationDir: convDir,
+      toolNameById: noName,
+    });
+
+    const history: Message[] = [{ role: "user", content: [...blocks] }];
+    const { messages, truncatedCount } = postTurnTruncateToolResults(history, {
+      conversationDir: convDir,
+    });
+
+    expect(truncatedCount).toBe(0);
+    expect(messages).toBe(history);
+  });
+
+  test("skips below-threshold, error, exempt, file_read, marked, and ax-tree results", () => {
+    const blocks = [
+      makeToolResult(SHORT, "tu_short"),
+      makeToolResult(LONG, "tu_err", true),
+      makeToolResult(LONG, "tu_skill"),
+      makeToolResult(LONG, "tu_read"),
+      makeToolResult(`${LONG}${TRUNCATION_MARKER}`, "tu_marked"),
+      makeToolResult(`<ax-tree>${LONG}</ax-tree>`, "tu_ax"),
+      { type: "text" as const, text: LONG },
+    ];
+    const originals = blocks.map((b) => b);
+
+    const count = spoolAndStubOversizedToolResults(blocks, {
+      conversationDir: convDir,
+      toolNameById: (id) => {
+        if (id === "tu_skill") return "skill_load";
+        if (id === "tu_read") return "file_read";
+        return undefined;
+      },
+    });
+
+    expect(count).toBe(0);
+    expect(blocks).toEqual(originals);
+    expect(existsSync(join(convDir, TOOL_RESULT_DIR))).toBe(false);
+  });
+
+  test("spools each eligible block in a mixed batch", () => {
+    const blocks = [
+      makeToolResult(LONG, "tu_a"),
+      makeToolResult(SHORT, "tu_b"),
+      makeToolResult(LONG, "tu_c"),
+    ];
+
+    const count = spoolAndStubOversizedToolResults(blocks, {
+      conversationDir: convDir,
+      toolNameById: noName,
+    });
+
+    expect(count).toBe(2);
+    expect(existsSync(getToolResultFilePath(convDir, "tu_a"))).toBe(true);
+    expect(existsSync(getToolResultFilePath(convDir, "tu_b"))).toBe(false);
+    expect(existsSync(getToolResultFilePath(convDir, "tu_c"))).toBe(true);
+    expect((blocks[0] as { content: string }).content).toContain(
+      TRUNCATION_MARKER,
+    );
+    expect((blocks[1] as { content: string }).content).toBe(SHORT);
+    expect((blocks[2] as { content: string }).content).toContain(
+      TRUNCATION_MARKER,
+    );
+  });
+
+  test("keeps full content when the spool directory cannot be created", () => {
+    // A regular file at the conversation-dir path makes mkdir throw, so no
+    // stub can be written; the block must keep its full content (a stub must
+    // never exist without its file on disk).
+    const fileAsDir = join(convDir, "not-a-dir");
+    writeFileSync(fileAsDir, "occupied", "utf-8");
+    const blocks = [makeToolResult(LONG, "tu_big")];
+
+    expect(() =>
+      spoolAndStubOversizedToolResults(blocks, {
+        conversationDir: fileAsDir,
+        toolNameById: noName,
+      }),
+    ).toThrow();
+    expect((blocks[0] as { content: string }).content).toBe(LONG);
+  });
+});
+
+describe("AgentLoop result-time spooling", () => {
+  let convDir: string;
+
+  const dummyTools: ToolDefinition[] = [
+    {
+      name: "fetch_transcript",
+      description: "Fetch a transcript",
+      input_schema: { type: "object", properties: {} },
+    },
+  ];
+
+  const userMessage: Message = {
+    role: "user",
+    content: [{ type: "text", text: "Import the transcripts" }],
+  };
+
+  beforeEach(() => {
+    resetPluginRegistryAndRegisterDefaults();
+    convDir = mkdtempSync(join(tmpdir(), "tool-result-spool-loop-"));
+  });
+
+  afterEach(() => {
+    rmSync(convDir, { recursive: true, force: true });
+  });
+
+  test("oversized result enters history as the stub; full content is never sent to the provider", async () => {
+    const { provider, calls } = createMockProvider([
+      toolUseResponse("tu_1", "fetch_transcript", {}),
+      textResponse("Done."),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      tools: dummyTools,
+      toolExecutor: async () => ({ content: LONG, isError: false }),
+      resolveConversationDir: () => convDir,
+    });
+
+    const events: AgentEvent[] = [];
+    const { history } = await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    const filePath = getToolResultFilePath(convDir, "tu_1");
+    expect(readFileSync(filePath, "utf-8")).toBe(LONG);
+
+    // The second provider call carries the stub, not the full content.
+    const sentResult = calls[1].messages[2].content[0] as { content: string };
+    expect(sentResult.content).toContain(TRUNCATION_MARKER);
+    expect(sentResult.content).toContain(filePath);
+    expect(JSON.stringify(calls.map((c) => c.messages))).not.toContain(LONG);
+
+    // Durable history and the emitted tool_result event match what was sent.
+    const historyResult = history[2].content[0] as { content: string };
+    expect(historyResult.content).toBe(sentResult.content);
+    const resultEvent = events.find((e) => e.type === "tool_result") as {
+      content: string;
+    };
+    expect(resultEvent.content).toBe(sentResult.content);
+  });
+
+  test("outbound payloads stay append-only across calls (prompt-cache prefix invariant)", async () => {
+    const { provider, calls } = createMockProvider([
+      toolUseResponse("tu_1", "fetch_transcript", {}),
+      toolUseResponse("tu_2", "fetch_transcript", {}),
+      textResponse("Done."),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      tools: dummyTools,
+      toolExecutor: async () => ({ content: LONG, isError: false }),
+      resolveConversationDir: () => convDir,
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(3);
+    // Each call's payload extends the previous call's payload without
+    // rewriting it — the property that keeps the provider's prompt-cache
+    // prefix valid across the turn's iterations.
+    for (let i = 1; i < calls.length; i++) {
+      expect(calls[i].messages.slice(0, calls[i - 1].messages.length)).toEqual(
+        calls[i - 1].messages,
+      );
+    }
+  });
+
+  test("spools the raw output even when it exceeds the post-tool-use truncation budget", async () => {
+    // maxInputTokens of 10k gives the default truncate plugin a 12k-char
+    // budget (0.3 share × 4 chars/token). The spool must run before that
+    // hook, so the file holds all 20k chars of raw output rather than the
+    // hook's tail-dropped copy — otherwise the omitted tail is unrecoverable.
+    const HUGE = "z".repeat(20_000);
+    const { provider } = createMockProvider([
+      toolUseResponse("tu_1", "fetch_transcript", {}),
+      textResponse("Done."),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      config: { maxInputTokens: 10_000 },
+      tools: dummyTools,
+      toolExecutor: async () => ({ content: HUGE, isError: false }),
+      resolveConversationDir: () => convDir,
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    const filePath = getToolResultFilePath(convDir, "tu_1");
+    expect(readFileSync(filePath, "utf-8")).toBe(HUGE);
+  });
+
+  test("without resolveConversationDir the full content is sent and post-turn handles it", async () => {
+    const { provider, calls } = createMockProvider([
+      toolUseResponse("tu_1", "fetch_transcript", {}),
+      textResponse("Done."),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      tools: dummyTools,
+      toolExecutor: async () => ({ content: LONG, isError: false }),
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    const sentResult = calls[1].messages[2].content[0] as { content: string };
+    expect(sentResult.content).toBe(LONG);
+    expect(existsSync(join(convDir, TOOL_RESULT_DIR))).toBe(false);
+  });
+});

package/src/__tests__/tool-result-truncate-hook.test.ts

@@ -48,6 +48,7 @@ function makeCtx(content: string): PostToolUseContext {
     toolResponse: makeToolResponse(content),
     messages: [],
     additionalContext: null,
+    model: "claude-test-model",
     maxInputTokens: MAX_INPUT_TOKENS,
     logger: noopLogger,
   };

package/src/__tests__/validate-input.test.ts

@@ -1,6 +1,9 @@
 import { describe, expect, test } from "bun:test";
 
-import { validateInputAgainstSchema } from "../skills/validate-input.js";
+import {
+  coerceStringBooleans,
+  validateInputAgainstSchema,
+} from "../skills/validate-input.js";
 
 // ---------------------------------------------------------------------------
 // required
@@ -355,6 +358,97 @@ describe("validateInputAgainstSchema — permissive paths", () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// string-boolean coercion (pre-validation pass)
+// ---------------------------------------------------------------------------
+
+describe("coerceStringBooleans", () => {
+  const schema = {
+    type: "object",
+    properties: {
+      auto_open: { type: "boolean" },
+      name: { type: "string" },
+      flags: { type: ["boolean", "null"] },
+    },
+  };
+
+  test.each([
+    ["false", false],
+    ["true", true],
+    ["False", false],
+    ["TRUE", true],
+    [" true ", true],
+  ] as const)("coerces %p to %p for boolean-typed properties", (raw, want) => {
+    const result = coerceStringBooleans({ auto_open: raw }, schema);
+    expect(result.auto_open).toBe(want);
+  });
+
+  test("coerced input passes validation and preserves intent", () => {
+    const coerced = coerceStringBooleans(
+      { auto_open: "false", name: "x" },
+      schema,
+    );
+    expect(validateInputAgainstSchema("app_create", coerced, schema)).toEqual({
+      ok: true,
+    });
+    expect(coerced.auto_open).toBe(false);
+  });
+
+  test("leaves non-coercible strings alone (validation still rejects)", () => {
+    const input = { auto_open: "yes" };
+    const result = coerceStringBooleans(input, schema);
+    expect(result).toBe(input);
+    const validation = validateInputAgainstSchema("app_create", result, schema);
+    expect(validation.ok).toBe(false);
+  });
+
+  test("does not touch string-typed or union-typed properties", () => {
+    const input = { name: "true", flags: "false" };
+    const result = coerceStringBooleans(input, schema);
+    expect(result).toBe(input);
+    expect(result.name).toBe("true");
+    expect(result.flags).toBe("false");
+  });
+
+  test("does not touch real booleans or other types", () => {
+    const input = { auto_open: true };
+    expect(coerceStringBooleans(input, schema)).toBe(input);
+    const inputNum = { auto_open: 1 };
+    expect(coerceStringBooleans(inputNum, schema)).toBe(inputNum);
+  });
+
+  test("returns the same object when there is nothing to coerce", () => {
+    const input = { name: "x" };
+    expect(coerceStringBooleans(input, schema)).toBe(input);
+    expect(coerceStringBooleans(input, undefined)).toBe(input);
+    expect(coerceStringBooleans(input, { type: "object" })).toBe(input);
+  });
+
+  test("returns a new object and never mutates the original input", () => {
+    const input = { auto_open: "false", name: "x" };
+    const snapshot = JSON.parse(JSON.stringify(input));
+    const result = coerceStringBooleans(input, schema);
+    expect(result).not.toBe(input);
+    expect(input).toEqual(snapshot);
+    expect(result.name).toBe("x");
+  });
+});
+
+describe("validateInputAgainstSchema — boolean string error message", () => {
+  test("points at the JSON boolean form when a string is passed", () => {
+    const result = validateInputAgainstSchema(
+      "app_create",
+      { auto_open: "yes" },
+      { type: "object", properties: { auto_open: { type: "boolean" } } },
+    );
+    expect(result.ok).toBe(false);
+    if (result.ok) return;
+    expect(result.errors).toContain(
+      "auto_open must be a boolean — pass true or false as a JSON boolean, not a string",
+    );
+  });
+});
+
 // ---------------------------------------------------------------------------
 // purity
 // ---------------------------------------------------------------------------

package/src/__tests__/workspace-migration-098-remove-stale-updates-bulletin-file.test.ts

@@ -0,0 +1,65 @@
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  afterAll,
+  afterEach,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  test,
+} from "bun:test";
+
+import { releaseNotesTavilyWebSearchMigration } from "../workspace/migrations/078-release-notes-tavily-web-search.js";
+import { removeStaleUpdatesBulletinFileMigration } from "../workspace/migrations/098-remove-stale-updates-bulletin-file.js";
+
+let testRoot: string;
+let workspaceDir: string;
+
+beforeAll(() => {
+  testRoot = mkdtempSync(join(tmpdir(), "migration-098-remove-updates-"));
+});
+
+afterAll(() => {
+  rmSync(testRoot, { recursive: true, force: true });
+});
+
+beforeEach(() => {
+  workspaceDir = mkdtempSync(join(testRoot, "ws-"));
+});
+
+afterEach(() => {
+  rmSync(workspaceDir, { recursive: true, force: true });
+});
+
+function updatesPath(): string {
+  return join(workspaceDir, "UPDATES.md");
+}
+
+describe("098-remove-stale-updates-bulletin-file", () => {
+  test("deletes a leftover UPDATES.md", () => {
+    releaseNotesTavilyWebSearchMigration.run(workspaceDir);
+    expect(existsSync(updatesPath())).toBe(true);
+
+    removeStaleUpdatesBulletinFileMigration.run(workspaceDir);
+
+    expect(existsSync(updatesPath())).toBe(false);
+  });
+
+  test("no-ops when UPDATES.md is absent", () => {
+    expect(() =>
+      removeStaleUpdatesBulletinFileMigration.run(workspaceDir),
+    ).not.toThrow();
+    expect(existsSync(updatesPath())).toBe(false);
+  });
+
+  test("is idempotent", () => {
+    writeFileSync(updatesPath(), "anything\n", "utf-8");
+
+    removeStaleUpdatesBulletinFileMigration.run(workspaceDir);
+    removeStaleUpdatesBulletinFileMigration.run(workspaceDir);
+
+    expect(existsSync(updatesPath())).toBe(false);
+  });
+});

package/src/__tests__/workspace-migration-099-disable-cache-one-shot-callsites.test.ts

@@ -0,0 +1,139 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { disableCacheOneShotCallsitesMigration } from "../workspace/migrations/099-disable-cache-one-shot-callsites.js";
+
+let workspaceDir: string;
+
+function freshWorkspace(): void {
+  workspaceDir = join(
+    tmpdir(),
+    `vellum-migration-099-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+  mkdirSync(workspaceDir, { recursive: true });
+}
+
+function writeConfig(data: Record<string, unknown>): void {
+  writeFileSync(
+    join(workspaceDir, "config.json"),
+    JSON.stringify(data, null, 2) + "\n",
+  );
+}
+
+function readConfig(): Record<string, unknown> {
+  return JSON.parse(readFileSync(join(workspaceDir, "config.json"), "utf-8"));
+}
+
+beforeEach(() => {
+  freshWorkspace();
+  delete process.env.VELLUM_DEFAULT_WORKSPACE_CONFIG_PATH;
+});
+
+afterEach(() => {
+  delete process.env.VELLUM_DEFAULT_WORKSPACE_CONFIG_PATH;
+  if (existsSync(workspaceDir)) {
+    rmSync(workspaceDir, { recursive: true, force: true });
+  }
+});
+
+type CallSites = Record<string, Record<string, unknown>>;
+
+function callSites(config: Record<string, unknown>): CallSites {
+  return (config.llm as { callSites: CallSites }).callSites;
+}
+
+describe("099-disable-cache-one-shot-callsites migration", () => {
+  test("has correct migration id", () => {
+    expect(disableCacheOneShotCallsitesMigration.id).toBe(
+      "099-disable-cache-one-shot-callsites",
+    );
+  });
+
+  test("adds disableCache to existing one-shot entries, preserving other keys", () => {
+    writeConfig({
+      llm: {
+        callSites: {
+          recall: { profile: "cost-optimized", maxTokens: 4096 },
+          replySuggestion: { profile: "cost-optimized" },
+        },
+      },
+    });
+
+    disableCacheOneShotCallsitesMigration.run(workspaceDir);
+
+    const sites = callSites(readConfig());
+    expect(sites.recall).toEqual({
+      profile: "cost-optimized",
+      maxTokens: 4096,
+      disableCache: true,
+    });
+    expect(sites.replySuggestion).toEqual({
+      profile: "cost-optimized",
+      disableCache: true,
+    });
+  });
+
+  test("leaves an explicit disableCache value untouched (user opted back in)", () => {
+    writeConfig({
+      llm: {
+        callSites: {
+          recall: { profile: "cost-optimized", disableCache: false },
+        },
+      },
+    });
+
+    disableCacheOneShotCallsitesMigration.run(workspaceDir);
+
+    expect(callSites(readConfig()).recall.disableCache).toBe(false);
+  });
+
+  test("does not create entries for call sites absent from disk", () => {
+    writeConfig({
+      llm: {
+        callSites: {
+          mainAgent: { profile: "balanced" },
+        },
+      },
+    });
+
+    disableCacheOneShotCallsitesMigration.run(workspaceDir);
+
+    const sites = callSites(readConfig());
+    expect(sites.recall).toBeUndefined();
+    expect(sites.homeGreeting).toBeUndefined();
+    expect(sites.mainAgent).toEqual({ profile: "balanced" });
+  });
+
+  test("is idempotent across repeated runs", () => {
+    writeConfig({
+      llm: {
+        callSites: {
+          conversationTitle: { profile: "cost-optimized" },
+        },
+      },
+    });
+
+    disableCacheOneShotCallsitesMigration.run(workspaceDir);
+    const afterFirst = readConfig();
+    disableCacheOneShotCallsitesMigration.run(workspaceDir);
+
+    expect(readConfig()).toEqual(afterFirst);
+  });
+
+  test("no-ops when config.json is missing or has no callSites", () => {
+    disableCacheOneShotCallsitesMigration.run(workspaceDir);
+    expect(existsSync(join(workspaceDir, "config.json"))).toBe(false);
+
+    writeConfig({ llm: {} });
+    disableCacheOneShotCallsitesMigration.run(workspaceDir);
+    expect(readConfig()).toEqual({ llm: {} });
+  });
+});