@vellumai/assistant 0.8.6 → 0.8.7-dev.202606052118.34cd356

package/src/__tests__/tool-result-truncate-pipeline.test.ts

@@ -1,344 +0,0 @@
-/**
- * Tests for the `toolResultTruncate` plugin pipeline (PR 17).
- *
- * Covers:
- * - The default terminal delegates to `truncateToolResultText`, producing
- *   byte-for-byte identical output to calling the helper directly across
- *   short, long, and newline-bounded inputs (property-style).
- * - The pipeline routes through `runPipeline` with the
- *   `DEFAULT_TIMEOUTS.toolResultTruncate` budget and returns a
- *   `{ content, truncated }` pair whose `truncated` flag matches whether
- *   the content actually changed.
- * - Plugins registered alongside the default can short-circuit or decorate
- *   the terminal's output.
- * - A user plugin registered AFTER the default still runs — the default
- *   plugin's middleware is a passthrough, so the onion composition cannot
- *   shadow late-registered middleware.
- */
-
-import { beforeEach, describe, expect, test } from "bun:test";
-
-import {
-  MIN_KEEP_CHARS,
-  truncateToolResultText,
-  TRUNCATION_SUFFIX,
-} from "../context/tool-result-truncation.js";
-import type { TrustContext } from "../daemon/trust-context.js";
-import {
-  defaultToolResultTruncatePlugin,
-  defaultToolResultTruncateTerminal,
-} from "../plugins/defaults/tool-result-truncate.js";
-import { DEFAULT_TIMEOUTS, runPipeline } from "../plugins/pipeline.js";
-import {
-  getMiddlewaresFor,
-  registerPlugin,
-  resetPluginRegistryForTests,
-} from "../plugins/registry.js";
-import {
-  type Middleware,
-  type ToolResultTruncateArgs,
-  type ToolResultTruncateResult,
-  type TurnContext,
-} from "../plugins/types.js";
-
-// ---------------------------------------------------------------------------
-// Fixtures
-// ---------------------------------------------------------------------------
-
-const trust: TrustContext = {
-  sourceChannel: "vellum",
-  trustClass: "guardian",
-};
-
-function makeCtx(overrides: Partial<TurnContext> = {}): TurnContext {
-  return {
-    requestId: "req-test",
-    conversationId: "conv-test",
-    turnIndex: 0,
-    trust,
-    ...overrides,
-  };
-}
-
-describe("toolResultTruncate pipeline", () => {
-  beforeEach(() => {
-    resetPluginRegistryForTests();
-  });
-
-  // -------------------------------------------------------------------------
-  // Default terminal — isolated (no pipeline runner)
-  // -------------------------------------------------------------------------
-
-  describe("default terminal", () => {
-    test("passes short content through unchanged with truncated=false", () => {
-      const content = "hello world";
-      const result = defaultToolResultTruncateTerminal({
-        content,
-        maxChars: 100,
-      });
-      expect(result.content).toBe(content);
-      expect(result.truncated).toBe(false);
-    });
-
-    test("truncates oversize content and reports truncated=true", () => {
-      const content = "a".repeat(10_000);
-      const maxChars = 5_000;
-      const expected = truncateToolResultText(content, maxChars);
-      const result = defaultToolResultTruncateTerminal({ content, maxChars });
-      expect(result.content).toBe(expected);
-      expect(result.truncated).toBe(true);
-      expect(result.content).toContain(TRUNCATION_SUFFIX);
-    });
-
-    test("snaps to newline boundary identically to truncateToolResultText", () => {
-      const lines = Array.from(
-        { length: 1_000 },
-        (_, i) => `line ${i}: ${"x".repeat(20)}`,
-      ).join("\n");
-      const maxChars = 5_000;
-      const expected = truncateToolResultText(lines, maxChars);
-      const result = defaultToolResultTruncateTerminal({
-        content: lines,
-        maxChars,
-      });
-      expect(result.content).toBe(expected);
-      expect(result.truncated).toBe(true);
-    });
-
-    test("returns truncated=false when effectiveMax keeps the full text (maxChars < MIN_KEEP_CHARS case)", () => {
-      const textLength = MIN_KEEP_CHARS - TRUNCATION_SUFFIX.length - 10;
-      const content = "a".repeat(textLength);
-      const maxChars = 100;
-      const expected = truncateToolResultText(content, maxChars);
-      const result = defaultToolResultTruncateTerminal({ content, maxChars });
-      // Helper returns the original text unchanged in this case.
-      expect(expected).toBe(content);
-      expect(result.content).toBe(content);
-      expect(result.truncated).toBe(false);
-    });
-  });
-
-  // -------------------------------------------------------------------------
-  // End-to-end: default plugin routed through runPipeline
-  // -------------------------------------------------------------------------
-
-  describe("runPipeline with the default plugin registered", () => {
-    async function runDefault(
-      content: string,
-      maxChars: number,
-    ): Promise<ToolResultTruncateResult> {
-      registerPlugin(defaultToolResultTruncatePlugin);
-      const middlewares = getMiddlewaresFor("toolResultTruncate");
-      return runPipeline<ToolResultTruncateArgs, ToolResultTruncateResult>(
-        "toolResultTruncate",
-        middlewares,
-        async (args) => defaultToolResultTruncateTerminal(args),
-        { content, maxChars },
-        makeCtx(),
-        DEFAULT_TIMEOUTS.toolResultTruncate,
-      );
-    }
-
-    test("short content round-trip matches truncateToolResultText", async () => {
-      const content = "quick brown fox";
-      const maxChars = 200;
-      const expected = truncateToolResultText(content, maxChars);
-      const result = await runDefault(content, maxChars);
-      expect(result.content).toBe(expected);
-      expect(result.truncated).toBe(false);
-    });
-
-    test("long content round-trip matches truncateToolResultText", async () => {
-      const content = "z".repeat(50_000);
-      const maxChars = 10_000;
-      const expected = truncateToolResultText(content, maxChars);
-      const result = await runDefault(content, maxChars);
-      expect(result.content).toBe(expected);
-      expect(result.truncated).toBe(true);
-      expect(result.content).toContain(TRUNCATION_SUFFIX);
-    });
-
-    test("newline-bounded content round-trip matches truncateToolResultText", async () => {
-      const lines = Array.from(
-        { length: 500 },
-        (_, i) => `line ${i}: ${"y".repeat(40)}`,
-      ).join("\n");
-      const maxChars = 4_000;
-      const expected = truncateToolResultText(lines, maxChars);
-      const result = await runDefault(lines, maxChars);
-      expect(result.content).toBe(expected);
-      expect(result.truncated).toBe(true);
-    });
-
-    test("property test: default pipeline output equals direct truncateToolResultText across varied inputs", async () => {
-      // Deterministic pseudo-random over a fixed seed — bun's test runner
-      // doesn't ship a property-test library, so we hand-roll a tiny LCG
-      // that produces enough spread for a meaningful regression signal
-      // without introducing a dependency.
-      let seed = 0xc0ffee;
-      const rand = () => {
-        seed = (seed * 1664525 + 1013904223) & 0xffffffff;
-        return (seed >>> 0) / 0x100000000;
-      };
-      const alphabet = "abcdefghijklmnopqrstuvwxyz \n";
-
-      const cases: Array<{ content: string; maxChars: number }> = [];
-      for (let i = 0; i < 40; i++) {
-        // Lengths span short, boundary, and long relative to the maxChars
-        // budget so the property covers pass-through, newline-snap, and
-        // pure tail-drop paths.
-        const length = Math.floor(rand() * 20_000);
-        let content = "";
-        for (let j = 0; j < length; j++) {
-          content += alphabet[Math.floor(rand() * alphabet.length)];
-        }
-        const maxChars = 1_000 + Math.floor(rand() * 9_000);
-        cases.push({ content, maxChars });
-      }
-
-      // Register once outside the loop — registry is reset in `beforeEach`,
-      // so the per-case reset lives in the loop instead.
-      for (const { content, maxChars } of cases) {
-        resetPluginRegistryForTests();
-        registerPlugin(defaultToolResultTruncatePlugin);
-        const middlewares = getMiddlewaresFor("toolResultTruncate");
-        const result = await runPipeline<
-          ToolResultTruncateArgs,
-          ToolResultTruncateResult
-        >(
-          "toolResultTruncate",
-          middlewares,
-          async (args) => defaultToolResultTruncateTerminal(args),
-          { content, maxChars },
-          makeCtx(),
-          DEFAULT_TIMEOUTS.toolResultTruncate,
-        );
-        const expected = truncateToolResultText(content, maxChars);
-        expect(result.content).toBe(expected);
-        expect(result.truncated).toBe(expected !== content);
-      }
-    });
-  });
-
-  // -------------------------------------------------------------------------
-  // Middleware composition — an outer plugin can intercept / transform
-  // -------------------------------------------------------------------------
-
-  describe("middleware composition", () => {
-    test("an outer plugin can short-circuit the default with its own content", async () => {
-      const shortCircuit: Middleware<
-        ToolResultTruncateArgs,
-        ToolResultTruncateResult
-      > = async (_args, _next, _ctx) => {
-        return { content: "SUMMARY", truncated: true };
-      };
-      registerPlugin({
-        manifest: {
-          name: "short-circuit",
-          version: "1.0.0",
-        },
-        middleware: { toolResultTruncate: shortCircuit },
-      });
-      registerPlugin(defaultToolResultTruncatePlugin);
-
-      const middlewares = getMiddlewaresFor("toolResultTruncate");
-      const result = await runPipeline<
-        ToolResultTruncateArgs,
-        ToolResultTruncateResult
-      >(
-        "toolResultTruncate",
-        middlewares,
-        async (args) => defaultToolResultTruncateTerminal(args),
-        { content: "a".repeat(10_000), maxChars: 5_000 },
-        makeCtx(),
-        DEFAULT_TIMEOUTS.toolResultTruncate,
-      );
-
-      expect(result.content).toBe("SUMMARY");
-      expect(result.truncated).toBe(true);
-    });
-
-    test("an outer plugin can observe and mutate the default's output", async () => {
-      const prefixer: Middleware<
-        ToolResultTruncateArgs,
-        ToolResultTruncateResult
-      > = async (args, next, _ctx) => {
-        const inner = await next(args);
-        return { ...inner, content: `[wrapped] ${inner.content}` };
-      };
-      registerPlugin({
-        manifest: {
-          name: "prefixer",
-          version: "1.0.0",
-        },
-        middleware: { toolResultTruncate: prefixer },
-      });
-      registerPlugin(defaultToolResultTruncatePlugin);
-
-      const middlewares = getMiddlewaresFor("toolResultTruncate");
-      const content = "hello";
-      const result = await runPipeline<
-        ToolResultTruncateArgs,
-        ToolResultTruncateResult
-      >(
-        "toolResultTruncate",
-        middlewares,
-        async (args) => defaultToolResultTruncateTerminal(args),
-        { content, maxChars: 100 },
-        makeCtx(),
-        DEFAULT_TIMEOUTS.toolResultTruncate,
-      );
-
-      expect(result.content).toBe(`[wrapped] ${content}`);
-      expect(result.truncated).toBe(false);
-    });
-
-    test("user plugin registered AFTER the default still runs (no shadowing)", async () => {
-      // Production registration order: defaults load first via the
-      // side-effect imports in `defaults/index.ts`, then user plugins
-      // register on top (via `bootstrapPlugins()` or hot-reload). The
-      // user's middleware ends up at a deeper onion layer than the
-      // default. If the default's middleware were to bypass `next` and
-      // call the terminal directly, the user middleware would never run
-      // — this test guards against that regression.
-      registerPlugin(defaultToolResultTruncatePlugin);
-
-      let userMiddlewareRan = false;
-      const userMiddleware: Middleware<
-        ToolResultTruncateArgs,
-        ToolResultTruncateResult
-      > = async (args, next) => {
-        userMiddlewareRan = true;
-        return next(args);
-      };
-      registerPlugin({
-        manifest: {
-          name: "late-user-plugin",
-          version: "0.0.1",
-        },
-        middleware: { toolResultTruncate: userMiddleware },
-      });
-
-      const middlewares = getMiddlewaresFor("toolResultTruncate");
-      const content = "a".repeat(10_000);
-      const maxChars = 5_000;
-      const result = await runPipeline<
-        ToolResultTruncateArgs,
-        ToolResultTruncateResult
-      >(
-        "toolResultTruncate",
-        middlewares,
-        async (args) => defaultToolResultTruncateTerminal(args),
-        { content, maxChars },
-        makeCtx(),
-        DEFAULT_TIMEOUTS.toolResultTruncate,
-      );
-
-      expect(userMiddlewareRan).toBe(true);
-      // Terminal still runs after the user passthrough, so output matches
-      // the direct helper.
-      expect(result.content).toBe(truncateToolResultText(content, maxChars));
-      expect(result.truncated).toBe(true);
-    });
-  });
-});

package/src/cli/commands/__tests__/memory-v3-render.test.ts

@@ -1,340 +0,0 @@
-import { describe, expect, test } from "bun:test";
-
-import type {
-  LlmCallRecord,
-  MemoryV3SimulateResult,
-  MemoryV3TreeResult,
-  MemoryV3ValidateResult,
-} from "../../../runtime/routes/memory-v3-routes.js";
-import {
-  renderLlmCalls,
-  renderSimulation,
-  renderTree,
-  renderValidationReport,
-  reportHasDefects,
-} from "../memory-v3-render.js";
-
-function cleanReport(): MemoryV3ValidateResult {
-  return {
-    danglingChildRefs: [],
-    danglingChildRefCount: 0,
-    orphanPages: [],
-    orphanPageCount: 0,
-    cycles: [],
-    cycleCount: 0,
-    staleIndex: [],
-    staleIndexCount: 0,
-    unknownEdgeTargets: [],
-    unknownEdgeTargetCount: 0,
-  };
-}
-
-describe("memory v3 — renderValidationReport", () => {
-  test("renders 'none' for every empty category", () => {
-    const out = renderValidationReport(cleanReport());
-    expect(out).toContain("Memory v3 Tree Validation");
-    expect(out).toContain("Dangling child refs: none");
-    expect(out).toContain("Orphan pages: none");
-    expect(out).toContain("Cycles: none");
-    expect(out).toContain("Stale index: none");
-    expect(out).toContain("Unknown edge targets: none");
-  });
-
-  test("renders counts and offending ids for each defect category", () => {
-    const report: MemoryV3ValidateResult = {
-      danglingChildRefs: [{ node: "people", ref: "ghost", kind: "node" }],
-      danglingChildRefCount: 1,
-      orphanPages: ["stray-page"],
-      orphanPageCount: 1,
-      cycles: [{ from: "a", to: "b" }],
-      cycleCount: 1,
-      staleIndex: [
-        { node: "root", child: "people", nodeMtimeMs: 1, childMtimeMs: 2 },
-      ],
-      staleIndexCount: 1,
-      unknownEdgeTargets: [{ from: "p1", to: "missing" }],
-      unknownEdgeTargetCount: 1,
-    };
-    const out = renderValidationReport(report);
-    expect(out).toContain("Dangling child refs: 1");
-    expect(out).toContain("people → node:ghost");
-    expect(out).toContain("Orphan pages: 1");
-    expect(out).toContain("- stray-page");
-    expect(out).toContain("Cycles: 1");
-    expect(out).toContain("a → b");
-    expect(out).toContain("Stale index: 1");
-    expect(out).toContain("root (older than child people)");
-    expect(out).toContain("Unknown edge targets: 1");
-    expect(out).toContain("p1 → missing");
-  });
-});
-
-describe("memory v3 — reportHasDefects", () => {
-  test("false for a clean report", () => {
-    expect(reportHasDefects(cleanReport())).toBe(false);
-  });
-
-  test("true when any single category is non-empty", () => {
-    const report = cleanReport();
-    report.orphanPageCount = 1;
-    report.orphanPages = ["x"];
-    expect(reportHasDefects(report)).toBe(true);
-  });
-});
-
-describe("memory v3 — renderTree", () => {
-  test("renders an indented tree descending node and page children", () => {
-    const view: MemoryV3TreeResult = {
-      root: "_root",
-      nodes: [
-        {
-          id: "_root",
-          children: [
-            { kind: "node", ref: "people" },
-            { kind: "page", ref: "overview" },
-          ],
-        },
-        {
-          id: "people",
-          children: [{ kind: "page", ref: "alice" }],
-        },
-      ],
-    };
-    const out = renderTree(view);
-    expect(out).toBe(
-      ["node:_root", "  node:people", "    page:alice", "  page:overview"].join(
-        "\n",
-      ),
-    );
-  });
-
-  test("marks a shared DAG sub-node as a re-entry rather than re-expanding", () => {
-    const view: MemoryV3TreeResult = {
-      root: "_root",
-      nodes: [
-        {
-          id: "_root",
-          children: [
-            { kind: "node", ref: "a" },
-            { kind: "node", ref: "b" },
-          ],
-        },
-        { id: "a", children: [{ kind: "node", ref: "shared" }] },
-        { id: "b", children: [{ kind: "node", ref: "shared" }] },
-        { id: "shared", children: [{ kind: "page", ref: "leaf" }] },
-      ],
-    };
-    const out = renderTree(view);
-    // First reach under `a` expands; second reach under `b` is a marked re-entry.
-    expect(out).toContain("  node:a\n    node:shared\n      page:leaf");
-    expect(out).toContain("node:shared (↑ already shown)");
-    // The leaf page is expanded exactly once.
-    expect(out.match(/page:leaf/g)?.length).toBe(1);
-  });
-
-  test("bounds output on a cycle instead of looping forever", () => {
-    const view: MemoryV3TreeResult = {
-      root: "_root",
-      nodes: [
-        { id: "_root", children: [{ kind: "node", ref: "a" }] },
-        { id: "a", children: [{ kind: "node", ref: "_root" }] },
-      ],
-    };
-    const out = renderTree(view);
-    expect(out).toContain("node:_root (↑ already shown)");
-  });
-
-  test("flags a child ref whose target node is missing", () => {
-    const view: MemoryV3TreeResult = {
-      root: "_root",
-      nodes: [{ id: "_root", children: [{ kind: "node", ref: "ghost" }] }],
-    };
-    const out = renderTree(view);
-    expect(out).toContain("node:ghost (missing)");
-  });
-
-  test("lists nodes unreachable from the root", () => {
-    const view: MemoryV3TreeResult = {
-      root: "_root",
-      nodes: [
-        { id: "_root", children: [] },
-        { id: "floating", children: [] },
-      ],
-    };
-    const out = renderTree(view);
-    expect(out).toContain("Unreachable nodes (1):");
-    expect(out).toContain("- node:floating");
-  });
-});
-
-function simResult(): MemoryV3SimulateResult {
-  return {
-    query: "what should we ship next",
-    selectedSlugs: ["page-tree", "page-hot", "page-edge"],
-    sourceBySlug: {
-      "page-hot": "hot",
-      "page-tree": "tree",
-      "page-edge": "edge",
-    },
-    trace: {
-      passes: [
-        {
-          passNumber: 1,
-          scouts: [
-            { lane: "hot", slugs: ["page-hot"] },
-            { lane: "sparse", slugs: [] },
-            { lane: "dense", slugs: ["d1", "d2"] },
-          ],
-          treeLevels: [
-            {
-              node: "",
-              considered: ["people", "frames", "objects"],
-              descended: ["people", "frames"],
-              skipped: ["objects"],
-              reasoning: "query is about planning",
-            },
-          ],
-          edgeExpansions: [{ from: "page-tree", pulled: ["page-edge"] }],
-          gate: { decision: "more", questions: ["narrow to roadmap?"] },
-        },
-        {
-          passNumber: 2,
-          scouts: [{ lane: "hot", slugs: [] }],
-          gate: { decision: "ready" },
-        },
-      ],
-    },
-    cost: { ms: 1234 },
-    failureReason: null,
-    llmCalls: [],
-    effectiveConfig: {
-      passCap: 3,
-      lanes: { hot: true, sparse: true, dense: true, tree: true, edges: false },
-    },
-  };
-}
-
-describe("memory v3 — renderSimulation", () => {
-  test("renders query, effective config, per-pass trace, and grouped selection", () => {
-    const out = renderSimulation(simResult());
-    expect(out).toContain("Memory v3 Retrieval Simulation");
-    expect(out).toContain('Query: "what should we ship next"');
-    expect(out).toContain("passCap: 3");
-    // A disabled lane is surfaced in an `(off: …)` suffix.
-    expect(out).toContain("lanes: hot, sparse, dense, tree  (off: edges)");
-
-    expect(out).toContain("Passes: 2");
-    expect(out).toContain("Pass 1");
-    expect(out).toContain("scouts: hot=1  sparse=0  dense=2");
-    // The root tree level ("" node) prints as [root] with branch counts.
-    expect(out).toContain("[root]: considered 3, descended 2, skipped 1");
-    expect(out).toContain("→ people, frames");
-    expect(out).toContain("reason: query is about planning");
-    expect(out).toContain("edges: 1 seed(s) expanded, 1 pulled");
-    expect(out).toContain("gate: more");
-    expect(out).toContain("? narrow to roadmap?");
-    expect(out).toContain("gate: ready");
-
-    expect(out).toContain("Selected: 3 page(s)");
-    expect(out).toContain("Cost: 1234 ms");
-  });
-
-  test("groups selected slugs by provenance lane in fanout order", () => {
-    const out = renderSimulation(simResult());
-    const hotAt = out.indexOf("hot (1)");
-    const treeAt = out.indexOf("tree (1)");
-    const edgeAt = out.indexOf("edge (1)");
-    expect(hotAt).toBeGreaterThan(-1);
-    expect(treeAt).toBeGreaterThan(-1);
-    expect(edgeAt).toBeGreaterThan(-1);
-    // hot precedes tree precedes edge in SIMULATE_LANE_ORDER.
-    expect(hotAt).toBeLessThan(treeAt);
-    expect(treeAt).toBeLessThan(edgeAt);
-  });
-
-  test("renders all lanes inline when none are disabled", () => {
-    const result = simResult();
-    result.effectiveConfig.lanes.edges = true;
-    const out = renderSimulation(result);
-    expect(out).toContain("lanes: hot, sparse, dense, tree, edges");
-    expect(out).not.toContain("(off:");
-  });
-
-  test("surfaces a failure reason when the loop degraded", () => {
-    const result = simResult();
-    result.failureReason = "dense filter failed open";
-    const out = renderSimulation(result);
-    expect(out).toContain("Failure: dense filter failed open");
-  });
-});
-
-function llmCall(over: Partial<LlmCallRecord> = {}): LlmCallRecord {
-  return {
-    pass: 1,
-    lane: "gate",
-    callSite: "memoryV3Gate",
-    request: {
-      systemPrompt: "SYS-PROMPT",
-      messages: [
-        { role: "user", content: [{ type: "text", text: "USER-MSG" }] },
-      ],
-      tools: [
-        {
-          name: "decide_selection",
-          description: "decide",
-          input_schema: { type: "object", properties: {} },
-        },
-      ],
-    },
-    response: {
-      model: "stub-model",
-      stopReason: "tool_use",
-      usage: { inputTokens: 0, outputTokens: 0 },
-      content: [
-        {
-          type: "tool_use",
-          id: "tu-1",
-          name: "decide_selection",
-          input: { decision: "ready", selected_slugs: ["a"] },
-        },
-      ],
-    },
-    ms: 42,
-    ...over,
-  };
-}
-
-describe("memory v3 — renderLlmCalls", () => {
-  test("compact: one line per call with lane, node, and tool summary", () => {
-    const out = renderLlmCalls(
-      [
-        llmCall(),
-        llmCall({
-          lane: "descent",
-          callSite: "memoryV3Descent",
-          node: "people",
-        }),
-      ],
-      { full: false },
-    );
-    expect(out).toContain("LLM calls (2):");
-    expect(out).toContain("pass1 · gate");
-    expect(out).toContain("pass1 · descent · node=people");
-    expect(out).toContain("decide_selection");
-    expect(out).toContain("42ms");
-    // Compact mode must not dump the full system prompt.
-    expect(out).not.toContain("SYS-PROMPT");
-  });
-
-  test("full: includes system prompt, messages, tools, and tool_use input", () => {
-    const out = renderLlmCalls([llmCall()], { full: true });
-    expect(out).toContain("SYS-PROMPT");
-    expect(out).toContain("USER-MSG");
-    expect(out).toContain("decide_selection");
-    expect(out).toContain('"decision"');
-  });
-
-  test("renders 'none' when there are no calls", () => {
-    expect(renderLlmCalls([], { full: false })).toBe("LLM calls: none");
-  });
-});