@vellumai/assistant 0.8.5 → 0.8.6

package/src/memory/v3/__tests__/scouts.test.ts

@@ -358,6 +358,55 @@ describe("runScouts — dense lane", () => {
     expect(slugs.indexOf("people/x")).toBeLessThan(slugs.length - 1);
   });
 
+  test("MMR stays relevance-aware when every cosine is non-positive", async () => {
+    // All dense scores are <= 0 (weakly/negatively similar). Dividing by the
+    // pool max would collapse every relevance to a single value and degrade
+    // ranking to diversity-only, which would pull the lowest-scoring off-domain
+    // hit ahead of a higher-scoring active-domain hit. Range-normalizing keeps
+    // relevance ordering: work/b (higher score) stays ahead of people/x.
+    hybridHits = [
+      hit("work/a", { denseScore: -0.1 }),
+      hit("work/b", { denseScore: -0.5 }),
+      hit("people/x", { denseScore: -0.9 }),
+    ];
+    const { scouts } = await runScouts(
+      makeInput({
+        lanes: { hot: false, sparse: false },
+        denseQuota: { activeDomain: 30, offDomain: 8 },
+      }),
+      DEPS,
+    );
+    const slugs = scouts.find((s) => s.lane === "dense")?.slugs ?? [];
+    // Relevance-aware order: the higher-scoring work/b outranks people/x.
+    // Diversity-only (the bug) would interleave people/x ahead of work/b.
+    expect(slugs.indexOf("work/b")).toBeLessThan(slugs.indexOf("people/x"));
+  });
+
+  test("MMR positive-cosine ordering is unchanged by the range fix", async () => {
+    // A healthy positive range must behave exactly as before: relevance is
+    // score/max, so a strong active-domain run still gets diversified by the
+    // redundancy term once the subtree is over-represented.
+    hybridHits = [
+      hit("work/a", { denseScore: 0.95 }),
+      hit("work/b", { denseScore: 0.94 }),
+      hit("work/c", { denseScore: 0.93 }),
+      hit("work/d", { denseScore: 0.92 }),
+      hit("people/x", { denseScore: 0.9 }),
+    ];
+    const { scouts } = await runScouts(
+      makeInput({
+        lanes: { hot: false, sparse: false },
+        denseQuota: { activeDomain: 30, offDomain: 8 },
+      }),
+      DEPS,
+    );
+    const slugs = scouts.find((s) => s.lane === "dense")?.slugs ?? [];
+    // The strongest hit still leads; the lower-scoring off-domain hit is pulled
+    // forward (not stranded last) exactly as in the unmodified positive case.
+    expect(slugs[0]).toBe("work/a");
+    expect(slugs.indexOf("people/x")).toBeLessThan(slugs.length - 1);
+  });
+
   test("no dense hits yields no dense ScoutResult", async () => {
     hybridHits = [hit("sparse/only", { sparseScore: 2.0 })];
     const { scouts } = await runScouts(

package/src/memory/v3/__tests__/shadow-diff.test.ts

@@ -0,0 +1,225 @@
+import { describe, expect, test } from "bun:test";
+
+import type { MemoryV2ConceptRowRecord } from "../../memory-v2-activation-log-store.js";
+import { computeShadowDiff, type ShadowDiffLogRow } from "../shadow-diff.js";
+
+const ZERO = {
+  finalActivation: 0,
+  ownActivation: 0,
+  priorActivation: 0,
+  simUser: 0,
+  simAssistant: 0,
+  simNow: 0,
+  simUserRerankBoost: 0,
+  simAssistantRerankBoost: 0,
+  inRerankPool: false,
+  spreadContribution: 0,
+} as const;
+
+/** A v2 router concept row with the given selection status. */
+function v2concept(
+  slug: string,
+  status: MemoryV2ConceptRowRecord["status"],
+): MemoryV2ConceptRowRecord {
+  return { slug, status, source: "tier3:0", ...ZERO };
+}
+
+/** A v3 shadow concept row (always status 'injected') tagged with its lane. */
+function v3concept(slug: string, lane: string): MemoryV2ConceptRowRecord {
+  return { slug, status: "injected", source: "router", lane, ...ZERO };
+}
+
+function shadowRow(
+  conversationId: string,
+  createdAt: number,
+  slugLanes: Array<[string, string]>,
+): ShadowDiffLogRow {
+  return {
+    conversationId,
+    createdAt,
+    concepts: slugLanes.map(([slug, lane]) => v3concept(slug, lane)),
+  };
+}
+
+function routerRow(
+  conversationId: string,
+  createdAt: number,
+  opts: { injected?: string[]; cached?: string[]; rejected?: string[] },
+): ShadowDiffLogRow {
+  return {
+    conversationId,
+    createdAt,
+    concepts: [
+      ...(opts.injected ?? []).map((s) => v2concept(s, "injected")),
+      ...(opts.cached ?? []).map((s) => v2concept(s, "in_context")),
+      ...(opts.rejected ?? []).map((s) => v2concept(s, "not_injected")),
+    ],
+  };
+}
+
+const OPTS = { toleranceMs: 10_000, detailLimit: 50 };
+
+describe("computeShadowDiff", () => {
+  test("pairs nearest router row and diffs fresh-injected vs shadow sets", () => {
+    const shadow = [
+      shadowRow("conv1", 1000, [
+        ["a", "tree"],
+        ["b", "edge"],
+        ["c", "hot"],
+      ]),
+    ];
+    const router = [
+      routerRow("conv1", 1500, {
+        injected: ["a", "b", "x", "y"],
+        cached: ["cached1", "cached2"],
+        rejected: ["rej1"],
+      }),
+    ];
+
+    const result = computeShadowDiff(shadow, router, OPTS);
+
+    expect(result.turnsCompared).toBe(1);
+    expect(result.unpaired).toHaveLength(0);
+
+    const turn = result.turns[0]!;
+    expect(turn.deltaMs).toBe(500);
+    expect(turn.v2Count).toBe(4);
+    expect(turn.v3Count).toBe(3);
+    expect(turn.v2CachedCount).toBe(2);
+    expect(turn.overlap).toEqual(["a", "b"]);
+    expect(turn.v3Only).toEqual(["c"]);
+    expect(turn.v2Only).toEqual(["x", "y"]);
+    // cached / rejected v2 rows are NOT counted as dropped picks.
+    expect(turn.v2Only).not.toContain("cached1");
+    expect(turn.v2Only).not.toContain("rej1");
+    expect(turn.jaccard).toBeCloseTo(2 / 5, 10);
+  });
+
+  test("attributes overlap and v3-only to the v3 provenance lane", () => {
+    const shadow = [
+      shadowRow("conv1", 1000, [
+        ["a", "tree"],
+        ["b", "edge"],
+        ["c", "hot"],
+      ]),
+    ];
+    const router = [routerRow("conv1", 1100, { injected: ["a", "b"] })];
+
+    const { agg } = computeShadowDiff(shadow, router, OPTS);
+
+    expect(agg.overlapByLane).toEqual({ tree: 1, edge: 1 });
+    expect(agg.v3OnlyByLane).toEqual({ hot: 1 });
+  });
+
+  test("defaults a missing lane to 'unknown'", () => {
+    const shadow: ShadowDiffLogRow[] = [
+      {
+        conversationId: "conv1",
+        createdAt: 1000,
+        concepts: [
+          { slug: "a", status: "injected", source: "router", ...ZERO },
+        ],
+      },
+    ];
+    const router = [routerRow("conv1", 1100, { injected: [] })];
+
+    const { agg } = computeShadowDiff(shadow, router, OPTS);
+
+    expect(agg.v3OnlyByLane).toEqual({ unknown: 1 });
+  });
+
+  test("sends a shadow row with no router row within tolerance to unpaired", () => {
+    const shadow = [shadowRow("conv1", 1000, [["a", "tree"]])];
+    const router = [routerRow("conv1", 1000 + 20_000, { injected: ["a"] })];
+
+    const result = computeShadowDiff(shadow, router, OPTS);
+
+    expect(result.turnsCompared).toBe(0);
+    expect(result.unpaired).toEqual([
+      { conversationId: "conv1", shadowAt: 1000, v3Count: 1 },
+    ]);
+  });
+
+  test("greedily pairs each router row to at most one shadow row", () => {
+    const shadow = [
+      shadowRow("conv1", 1000, [["a", "tree"]]),
+      shadowRow("conv1", 5000, [["b", "edge"]]),
+    ];
+    const router = [
+      routerRow("conv1", 1200, { injected: ["a", "z"] }),
+      routerRow("conv1", 5200, { injected: ["b"] }),
+    ];
+
+    const result = computeShadowDiff(shadow, router, OPTS);
+
+    expect(result.turnsCompared).toBe(2);
+    const byShadow = new Map(result.turns.map((t) => [t.shadowAt, t]));
+    expect(byShadow.get(1000)!.routerAt).toBe(1200);
+    expect(byShadow.get(1000)!.v2Only).toEqual(["z"]);
+    expect(byShadow.get(5000)!.routerAt).toBe(5200);
+    expect(byShadow.get(5000)!.overlap).toEqual(["b"]);
+  });
+
+  test("caps per-turn detail newest-first but aggregates over all turns", () => {
+    const shadow = [
+      shadowRow("c1", 1000, [["a", "tree"]]),
+      shadowRow("c2", 2000, [["b", "tree"]]),
+      shadowRow("c3", 3000, [["c", "tree"]]),
+    ];
+    const router = [
+      routerRow("c1", 1000, { injected: ["a"] }),
+      routerRow("c2", 2000, { injected: ["b"] }),
+      routerRow("c3", 3000, { injected: ["c"] }),
+    ];
+
+    const result = computeShadowDiff(shadow, router, {
+      toleranceMs: 10_000,
+      detailLimit: 2,
+    });
+
+    expect(result.turnsCompared).toBe(3);
+    expect(result.turns.map((t) => t.shadowAt)).toEqual([3000, 2000]);
+  });
+
+  test("ranks the most-dropped and most-added slugs across turns", () => {
+    const shadow = [
+      shadowRow("c1", 1000, [["extra", "edge"]]),
+      shadowRow("c2", 2000, [["extra", "edge"]]),
+    ];
+    const router = [
+      routerRow("c1", 1000, { injected: ["dropped", "kept"] }),
+      routerRow("c2", 2000, { injected: ["dropped"] }),
+    ];
+
+    const { agg } = computeShadowDiff(shadow, router, OPTS);
+
+    expect(agg.totalV3Only).toBe(2);
+    expect(agg.totalV2Only).toBe(3);
+    expect(agg.v3OnlyTop[0]).toEqual({ slug: "extra", count: 2 });
+    expect(agg.v2OnlyTop[0]).toEqual({ slug: "dropped", count: 2 });
+    expect(agg.meanV3).toBeCloseTo(1, 10);
+    expect(agg.meanV2).toBeCloseTo(1.5, 10);
+  });
+
+  test("returns zeroed aggregates with no rows", () => {
+    const result = computeShadowDiff([], [], OPTS);
+    expect(result.turnsCompared).toBe(0);
+    expect(result.shadowRows).toBe(0);
+    expect(result.agg.meanJaccard).toBe(0);
+    expect(result.agg.v2OnlyTop).toEqual([]);
+    expect(result.turns).toEqual([]);
+  });
+
+  test("treats two empty selections as jaccard 0, not NaN", () => {
+    const shadow: ShadowDiffLogRow[] = [
+      { conversationId: "c1", createdAt: 1000, concepts: [] },
+    ];
+    const router = [routerRow("c1", 1000, { cached: ["only-cached"] })];
+
+    const result = computeShadowDiff(shadow, router, OPTS);
+
+    expect(result.turnsCompared).toBe(1);
+    expect(result.turns[0]!.jaccard).toBe(0);
+    expect(result.turns[0]!.v2CachedCount).toBe(1);
+  });
+});

package/src/memory/v3/__tests__/shadow-middleware.test.ts

@@ -35,6 +35,16 @@ mock.module("../../../util/logger.js", () => ({
   getLogger: () => makeMockLogger(),
 }));
 
+// Mirror the real `isUntrustedTrustClass` predicate (anything that is not the
+// guardian is untrusted) without pulling the resolver's contact-store graph
+// into the test module set.
+mock.module("../../../runtime/actor-trust-resolver.js", () => ({
+  isUntrustedTrustClass: (trustClass: string | undefined) =>
+    trustClass === "trusted_contact" ||
+    trustClass === "unknown" ||
+    trustClass === undefined,
+}));
+
 // ── Mutable test doubles, rewired per test ───────────────────────────────
 
 /** Drives `config.memory.v3.{enabled,shadow}` and `historical_pairs`. */
@@ -131,10 +141,13 @@ function makeCtx(): TurnContext {
   };
 }
 
-function makeArgs(signal?: AbortSignal): MemoryArgs {
+function makeArgs(
+  signal?: AbortSignal,
+  trustContext: TrustContext | undefined = trust,
+): MemoryArgs {
   return {
     conversationId: "conv-shadow",
-    trustContext: trust,
+    trustContext,
     turnIndex: 3,
     signal: signal ?? new AbortController().signal,
   };
@@ -268,6 +281,12 @@ describe("memory-v3 shadow middleware", () => {
     expect(logged.conversationId).toBe("conv-shadow");
     expect(logged.turn).toBe(3);
     expect(logged.concepts.map((c) => c.slug)).toEqual(["topic/a", "topic/b"]);
+
+    // Per-slug lane provenance is carried from sourceBySlug; a slug absent from
+    // the map (topic/b) gets no lane rather than a bogus one.
+    const bySlug = new Map(logged.concepts.map((c) => [c.slug, c]));
+    expect(bySlug.get("topic/a")?.lane).toBe("dense");
+    expect(bySlug.get("topic/b")?.lane).toBeUndefined();
   });
 
   test("v3 error → logged/swallowed, turn result unaffected, no log row", async () => {
@@ -309,4 +328,71 @@ describe("memory-v3 shadow middleware", () => {
     expect(loopCalls.length).toBe(0);
     expect(logCalls.length).toBe(0);
   });
+
+  test("untrusted actor → no v3 work, even with shadow on", async () => {
+    v3Enabled = true;
+    v3Shadow = true;
+
+    for (const trustClass of ["trusted_contact", "unknown"] as const) {
+      loopCalls.length = 0;
+      logCalls.length = 0;
+      const untrusted: TrustContext = { sourceChannel: "vellum", trustClass };
+      const result = await memoryV3ShadowMiddleware(
+        makeArgs(undefined, untrusted),
+        async () => DOWNSTREAM_RESULT,
+        makeCtx(),
+      );
+      expect(result).toBe(DOWNSTREAM_RESULT);
+
+      await flush();
+      // Untrusted turn never spends shadow retrieval LLM calls.
+      expect(loopCalls.length).toBe(0);
+      expect(logCalls.length).toBe(0);
+    }
+  });
+
+  test("absent trust context → treated as untrusted, no v3 work", async () => {
+    v3Enabled = true;
+    v3Shadow = true;
+
+    // Build args with an explicitly-undefined trustContext (the `makeArgs`
+    // default would substitute the guardian fixture, so construct directly).
+    const args: MemoryArgs = {
+      conversationId: "conv-shadow",
+      trustContext: undefined,
+      turnIndex: 3,
+      signal: new AbortController().signal,
+    };
+    const result = await memoryV3ShadowMiddleware(
+      args,
+      async () => DOWNSTREAM_RESULT,
+      makeCtx(),
+    );
+    expect(result).toBe(DOWNSTREAM_RESULT);
+
+    await flush();
+    expect(loopCalls.length).toBe(0);
+    expect(logCalls.length).toBe(0);
+  });
+
+  test("guardian actor → shadow runs (trust gate lets trusted turns through)", async () => {
+    v3Enabled = true;
+    v3Shadow = true;
+    const guardian: TrustContext = {
+      sourceChannel: "vellum",
+      trustClass: "guardian",
+    };
+
+    const result = await memoryV3ShadowMiddleware(
+      makeArgs(undefined, guardian),
+      async () => DOWNSTREAM_RESULT,
+      makeCtx(),
+    );
+    expect(result).toBe(DOWNSTREAM_RESULT);
+
+    await flush();
+    // Trusted turn fires the v3 loop and logs its selection.
+    expect(loopCalls.length).toBe(1);
+    expect(logCalls.length).toBe(1);
+  });
 });

package/src/memory/v3/__tests__/traversal.test.ts

@@ -228,6 +228,45 @@ describe("walkTree — breadthBudget", () => {
     expect([...pages].sort()).toEqual(["pa", "pb"]);
     expect(levels.map((l) => l.node).sort()).toEqual(["_root", "a", "b"]);
   });
+
+  test("spends the budget only on unvisited siblings, not already-visited picks", async () => {
+    // `_root` → {p1, p2}. `p1` descends `shared`, marking it visited. `p2`
+    // then offers `[shared, x, y]` under a budget of 2. If the budget were
+    // applied before filtering visited nodes, `shared` would consume a slot
+    // and `y` would be skipped. Filtering visited first spends the budget on
+    // `x` and `y`, so both unvisited siblings are descended.
+    const tree = makeTree("_root", {
+      _root: [node("p1"), node("p2")],
+      p1: [node("shared")],
+      p2: [node("shared"), node("x"), node("y")],
+      shared: [page("ps")],
+      x: [page("px")],
+      y: [page("py")],
+    });
+
+    const { pages, levels } = await walkTree(tree, {
+      breadthBudget: 2,
+      maxDepth: 8,
+      descend: descendAll,
+    });
+
+    const p2Level = levels.find((l) => l.node === "p2")!;
+    // `shared` is offered but already visited; the budget goes to x and y.
+    expect(p2Level.considered).toEqual(["shared", "x", "y"]);
+    expect(p2Level.descended).toEqual(["x", "y"]);
+    expect(p2Level.skipped).toEqual(["shared"]);
+
+    // The previously-skipped sibling `y` (and its page) is now reached.
+    expect([...pages].sort()).toEqual(["ps", "px", "py"]);
+    expect(levels.map((l) => l.node).sort()).toEqual([
+      "_root",
+      "p1",
+      "p2",
+      "shared",
+      "x",
+      "y",
+    ]);
+  });
 });
 
 // ---------------------------------------------------------------------------

package/src/memory/v3/__tests__/tree-walk.test.ts

@@ -279,6 +279,83 @@ describe("runTreeWalk — scripted descent", () => {
     expect(levels.map((l) => l.node).sort()).toEqual(["_root", "a"]);
   });
 
+  test("omitted keep_pages keeps every offered page at the node (recall-safe)", async () => {
+    // _root → a, a leaf bucket of two pages. The model descends but returns NO
+    // keep_pages field at all — a silent omission must keep every offered page,
+    // not drop them all.
+    const tree = makeTree("_root", {
+      _root: [node("a")],
+      a: [page("frames/example-a"), page("people/alice")],
+    });
+    const pages = makePages(["frames/example-a", "people/alice"]);
+    const calls: ProviderCall[] = [];
+    // Bespoke stub: emits tool input WITHOUT a `keep_pages` key for node "a".
+    const provider: Provider = {
+      name: "omit-keep-pages",
+      sendMessage: async (messages, tools, systemPrompt, options) => {
+        calls.push({ messages, tools, systemPrompt, options });
+        const nodeId =
+          nodeIdFromCall({ messages, tools, systemPrompt, options }) ?? "";
+        const input: Record<string, unknown> =
+          nodeId === "_root"
+            ? { descend: ["a"], keep_pages: [] }
+            : { descend: [] }; // node "a": keep_pages omitted entirely
+        return {
+          model: "stub-model",
+          stopReason: "tool_use",
+          usage: { inputTokens: 0, outputTokens: 0 },
+          content: [
+            {
+              type: "tool_use",
+              id: `tu-${nodeId}`,
+              name: "choose_branches",
+              input,
+            },
+          ],
+        };
+      },
+    };
+
+    const { pages: collected } = await runTreeWalk({
+      input: makeInput(),
+      tree,
+      pages,
+      scouts: [],
+      provider,
+    });
+
+    // Both offered pages survive the omission.
+    expect([...collected].sort()).toEqual(["frames/example-a", "people/alice"]);
+  });
+
+  test("explicit empty keep_pages keeps no pages at the node", async () => {
+    // Same tree, but node "a" returns an *explicit* empty keep_pages — honored
+    // as the model genuinely keeping nothing here.
+    const tree = makeTree("_root", {
+      _root: [node("a")],
+      a: [page("frames/example-a"), page("people/alice")],
+    });
+    const pages = makePages(["frames/example-a", "people/alice"]);
+    const calls: ProviderCall[] = [];
+    const provider = makeProvider(
+      {
+        _root: { descend: ["a"], keep: [] },
+        a: { descend: [], keep: [] },
+      },
+      calls,
+    );
+
+    const { pages: collected } = await runTreeWalk({
+      input: makeInput(),
+      tree,
+      pages,
+      scouts: [],
+      provider,
+    });
+
+    expect([...collected]).toEqual([]);
+  });
+
   test("keeps only the pages the model selects at a node", async () => {
     // _root → a, a leaf bucket of two pages. The model keeps only one.
     const tree = makeTree("_root", {

package/src/memory/v3/__tests__/validate.test.ts

@@ -178,6 +178,38 @@ describe("validateTree — cycles", () => {
 
     expect(report.cycles).toEqual([]);
   });
+
+  test("detects a cycle in a component unreachable from the root", async () => {
+    // _root is a leaf with no edges; the x ↔ y cycle lives in a disconnected
+    // component the root can never reach. The sweep over unvisited nodes must
+    // still surface it.
+    await writeNode(workspaceDir, node(ROOT_NODE_ID, []));
+    await writeNode(workspaceDir, node("x", ["node:y"]));
+    await writeNode(workspaceDir, node("y", ["node:x"]));
+    resetCaches();
+
+    const report = await validateTree(workspaceDir);
+
+    expect(report.cycleCount).toBe(1);
+    // The back-edge closes on whichever of the pair the sweep enters second.
+    const [{ from, to }] = report.cycles;
+    expect(new Set([from, to])).toEqual(new Set(["x", "y"]));
+  });
+
+  test("a page off an unreachable cyclic node stays an orphan", async () => {
+    // The disconnected-component sweep covers x/y for cycle detection but must
+    // not mark them root-reachable — `detached` therefore remains an orphan.
+    await writeNode(workspaceDir, node(ROOT_NODE_ID, []));
+    await writeNode(workspaceDir, node("x", ["node:y", "page:detached"]));
+    await writeNode(workspaceDir, node("y", ["node:x"]));
+    await writePage(workspaceDir, page("detached"));
+    resetCaches();
+
+    const report = await validateTree(workspaceDir);
+
+    expect(report.cycleCount).toBe(1);
+    expect(report.orphanPages).toEqual(["detached"]);
+  });
 });
 
 describe("validateTree — staleIndex", () => {