@vellumai/assistant 0.7.0 → 0.7.2

package/src/memory/v2/__tests__/injection.test.ts

@@ -14,12 +14,12 @@
  *
  * Hermetic by design: the embedding backend, qdrant client, and `getConfig`
  * are mocked at the module level so the suite never reaches a real backend.
- * The skill activation pipeline (`selectSkillCandidates`,
- * `computeSkillActivation`, `selectSkillInjections`) and the skill-store
- * lookup (`getSkillCapability`) are also mocked at the module level so each
- * test can stage its skill slate without touching the dedicated skills
- * Qdrant collection. The activation-store uses an in-memory SQLite database
- * so writes are real but contained.
+ * The skill activation pipeline (`computeSkillActivation`,
+ * `selectSkillInjections`) and the skill-store helpers (`getAllSkillIds`,
+ * `getSkillCapability`) are also mocked at the module level so each test can
+ * stage its skill slate without touching the dedicated skills Qdrant
+ * collection. The activation-store uses an in-memory SQLite database so
+ * writes are real but contained.
  *
  * Tests use a temp workspace (mkdtemp) and never touch `~/.vellum/`. Sample
  * page content uses generic placeholders (Alice, Bob, etc.) per the cross-
@@ -127,18 +127,19 @@ mock.module("@qdrant/js-client-rest", () => ({
 // Skill pipeline mocks
 // ---------------------------------------------------------------------------
 //
-// The skill side of the per-turn pipeline (`selectSkillCandidates`,
-// `computeSkillActivation`, `selectSkillInjections`) has its own dedicated
-// Qdrant collection and embedding round-trips. Rather than threading staged
-// hits through that whole pipeline for every test, we mock the three skill
-// helpers and the synchronous `getSkillCapability` lookup at the module level
-// and let each test stage a `topNow` ordering and the matching `SkillEntry`
-// content directly.
+// The skill side of the per-turn pipeline (`computeSkillActivation`,
+// `selectSkillInjections`) has its own dedicated Qdrant collection and
+// embedding round-trips. Rather than threading staged hits through that whole
+// pipeline for every test, we mock the two activation helpers and the two
+// skill-store helpers (`getAllSkillIds` for the candidate pool,
+// `getSkillCapability` for content lookup) at the module level and let each
+// test stage a `topNow` ordering and the matching `SkillEntry` content
+// directly.
 
 const skillState = {
   /** Ordered ids `selectSkillInjections.topNow` returns this turn. */
   topSkillIds: [] as string[],
-  /** id → SkillEntry used by `getSkillCapability`. */
+  /** id → SkillEntry used by `getSkillCapability` and `getAllSkillIds`. */
   entries: new Map<string, SkillEntry>(),
 };
 
@@ -149,17 +150,44 @@ mock.module("../activation.js", () => ({
   // activation map are inputs to `selectSkillInjections`, not anything the
   // injection logic introspects. Stub them to empty so the test stays focused
   // on the wiring, not the pipeline internals (covered in activation.test.ts).
-  selectSkillCandidates: async () => new Set<string>(),
-  computeSkillActivation: async () => new Map<string, number>(),
+  computeSkillActivation: async () => ({
+    activation: new Map<string, number>(),
+    breakdown: new Map(),
+  }),
   selectSkillInjections: ({ topK }: { topK: number }) => ({
     topNow: skillState.topSkillIds.slice(0, topK),
   }),
 }));
 
 mock.module("../skill-store.js", () => ({
+  getAllSkillIds: () => [...skillState.entries.keys()],
   getSkillCapability: (id: string) => skillState.entries.get(id) ?? null,
 }));
 
+// ---------------------------------------------------------------------------
+// Activation-log store mock
+// ---------------------------------------------------------------------------
+//
+// The real `recordMemoryV2ActivationLog` writes to the singleton
+// `getDb()` — but this test uses an isolated in-memory database, so we mock
+// the writer to capture calls in-process. `recordCalls` is the captured log
+// array; `recordShouldThrow` makes the next call throw to verify the caller
+// swallows the failure.
+
+const telemetryState = {
+  recordCalls: [] as Array<Record<string, unknown>>,
+  recordShouldThrow: false,
+};
+
+mock.module("../../memory-v2-activation-log-store.js", () => ({
+  recordMemoryV2ActivationLog: (params: Record<string, unknown>) => {
+    if (telemetryState.recordShouldThrow) {
+      throw new Error("simulated telemetry write failure");
+    }
+    telemetryState.recordCalls.push(params);
+  },
+}));
+
 // ---------------------------------------------------------------------------
 // Workspace + DB setup
 // ---------------------------------------------------------------------------
@@ -174,15 +202,9 @@ beforeAll(() => {
 
   // Seed the v2 directory layout the migration would normally create.
   mkdirSync(join(tmpWorkspace, "memory", "concepts"), { recursive: true });
-  // edges.json: one edge so spreading-activation has structure to walk.
-  writeFileSync(
-    join(tmpWorkspace, "memory", "edges.json"),
-    JSON.stringify({
-      version: 1,
-      edges: [["alice-vscode", "bob-coffee"]],
-    }),
-  );
-  // Three concept pages with generic, placeholder bodies.
+  // Three concept pages with generic, placeholder bodies. Outgoing edges
+  // live in each page's frontmatter `edges:` list — there is no separate
+  // edges-index file under the directed-edges model.
   writeFileSync(
     join(tmpWorkspace, "memory", "concepts", "alice-vscode.md"),
     `---
@@ -207,6 +229,18 @@ ref_files: []
 ---
 Carol loves jazz music — Coltrane in particular.`,
   );
+  // A page with both `edges` and `ref_files` populated so the frontmatter-
+  // injection test can assert the full canonical shape.
+  writeFileSync(
+    join(tmpWorkspace, "memory", "concepts", "frontmatter-demo.md"),
+    `---
+edges:
+  - alice-vscode
+ref_files:
+  - images/demo.jpg
+---
+Demo body content.`,
+  );
 });
 
 afterAll(() => {
@@ -298,6 +332,13 @@ function stageTurn(
   hits: Array<{ slug: string; denseScore?: number; sparseScore?: number }>,
   channels = 4,
 ): void {
+  // Clear any leftovers from a prior turn before staging this one so unused
+  // staged responses can't bleed into the next injection. The activation
+  // pipeline now skips the embedding round-trip for empty texts (turn 1's
+  // assistantMessage), so consumed-channel counts vary per turn — staging
+  // exclusively is the only way multi-turn tests stay aligned.
+  state.queryResponses.dense.length = 0;
+  state.queryResponses.sparse.length = 0;
   for (let i = 0; i < channels; i++) {
     state.queryResponses.dense.push({
       points: hits
@@ -319,6 +360,8 @@ function resetState(): void {
   state.queryResponses.sparse.length = 0;
   skillState.topSkillIds.length = 0;
   skillState.entries.clear();
+  telemetryState.recordCalls.length = 0;
+  telemetryState.recordShouldThrow = false;
   // The qdrant module caches its client; the cached client may be a
   // MockQdrantClient instance from a sibling test file. Reset to force a
   // fresh `new QdrantClient()` against this file's mock.
@@ -364,11 +407,13 @@ describe("injectMemoryV2Block", () => {
 
     expect(result.toInject).toEqual(["alice-vscode"]);
     expect(result.block).not.toBeNull();
-    expect(result.block).toContain("<memory __injected>");
-    expect(result.block).toContain("## What I Remember Right Now");
+    // `block` is the unwrapped inner content; the caller adds the
+    // `<memory>...</memory>` wrapper exactly once at injection time.
+    expect(result.block).not.toContain("<memory>");
+    expect(result.block).not.toContain("</memory>");
+    expect(result.block).not.toContain("## What I Remember Right Now");
     expect(result.block).toContain("### alice-vscode");
     expect(result.block).toContain("VS Code");
-    expect(result.block).toContain("</memory>");
 
     // State persisted: alice's activation is above epsilon and recorded;
     // everInjected captured the new slug + currentTurn.
@@ -512,6 +557,35 @@ describe("injectMemoryV2Block", () => {
     ]);
   });
 
+  test("includes the page frontmatter (edges, ref_files) in each rendered section", async () => {
+    // The frontmatter (`edges`, `ref_files`) lives on disk above the page
+    // body and is part of the page's content. Injection must reproduce both
+    // fields verbatim — bracketed by the canonical `---` delimiters — so the
+    // agent sees the page's edges and any referenced media paths alongside
+    // the prose body.
+    stageTurn([{ slug: "frontmatter-demo", denseScore: 0.9 }]);
+
+    const result = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "show me the demo",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+
+    expect(result.block).not.toBeNull();
+    // Slug header is immediately followed by the frontmatter open delimiter.
+    expect(result.block).toContain("### frontmatter-demo\n---\n");
+    // Both fields render in YAML block style with their populated values.
+    expect(result.block).toContain("edges:\n  - alice-vscode");
+    expect(result.block).toContain("ref_files:\n  - images/demo.jpg");
+    // Body still renders after the closing delimiter.
+    expect(result.block).toContain("Demo body content.");
+  });
+
   test("renders pages in activation-descending order", async () => {
     // Both slugs are fresh (no prior state). carol scores higher than alice
     // on every channel — so carol should be ranked first in topNow and
@@ -588,13 +662,24 @@ describe("injectMemoryV2Block", () => {
     expect(persisted!.everInjected).toEqual([
       { slug: "phantom-slug", turn: 1 },
     ]);
+
+    // Activation log marks the slug `page_missing` (not `injected`) so a
+    // stale Qdrant / edge-index entry pointing at a vanished page is
+    // visible in telemetry instead of masquerading as a successful inject.
+    expect(telemetryState.recordCalls.length).toBe(1);
+    const row = telemetryState.recordCalls[0] as {
+      concepts: Array<{ slug: string; status: string }>;
+    };
+    const phantom = row.concepts.find((c) => c.slug === "phantom-slug");
+    expect(phantom).toBeDefined();
+    expect(phantom!.status).toBe("page_missing");
   });
 
   // ---------------------------------------------------------------------------
   // Skill subsection rendering
   // ---------------------------------------------------------------------------
 
-  test("renders a skill-only block under the same `What I Remember Right Now` header", async () => {
+  test("renders a skill-only block alongside concept-page-only blocks", async () => {
     // No concept-page candidates this turn — the candidate query and the three
     // simBatch queries all return empty. The skill pipeline is mocked to
     // surface a single skill.
@@ -623,10 +708,11 @@ describe("injectMemoryV2Block", () => {
 
     expect(result.toInject).toEqual([]);
     expect(result.block).not.toBeNull();
-    // Same outer wrapping as concept-page-only blocks.
-    expect(result.block).toContain("<memory __injected>");
-    expect(result.block).toContain("## What I Remember Right Now");
-    expect(result.block).toContain("</memory>");
+    // `block` is the unwrapped inner content; the caller adds the
+    // `<memory>...</memory>` wrapper exactly once at injection time.
+    expect(result.block).not.toContain("<memory>");
+    expect(result.block).not.toContain("</memory>");
+    expect(result.block).not.toContain("## What I Remember Right Now");
     // No concept-page sections; skills subsection present with the right
     // bullet shape and the unconditional `→ use skill_load to activate` suffix.
     expect(result.block).not.toContain("### alice-vscode");
@@ -757,6 +843,95 @@ describe("injectMemoryV2Block", () => {
     expect(persisted!.everInjected).toEqual([]);
   });
 
+  test("context-load mode renders topNow even when every slug was previously injected", async () => {
+    // Turn 1 (per-turn): seed alice as injected.
+    stageTurn([{ slug: "alice-vscode", denseScore: 0.9 }]);
+    await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "Alice's editor",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+
+    // Subsequent context-load (post-compaction or fresh load): alice is
+    // back in the candidate pool. Per-turn would dedup against everInjected
+    // and produce a null block; context-load must re-render the full top-K
+    // because cached attachments don't exist on a fresh load.
+    stageTurn([{ slug: "alice-vscode", denseScore: 0.9 }]);
+    const result = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 2,
+      userMessage: "Reload context",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-2",
+      mode: "context-load",
+      config: makeConfig(),
+    });
+
+    expect(result.block).not.toBeNull();
+    expect(result.block).toContain("### alice-vscode");
+    // No newly-injected slug — alice was already in everInjected.
+    expect(result.toInject).toEqual([]);
+
+    // everInjected stays a single entry (alice was already there) — context-
+    // load doesn't double-stamp.
+    const persisted = await hydrate(db, "conv-1");
+    expect(persisted!.everInjected).toEqual([
+      { slug: "alice-vscode", turn: 1 },
+    ]);
+  });
+
+  test("context-load mode renders the full top-K on a fresh first turn", async () => {
+    // Turn 1 with no prior state and three candidates. Per-turn and context-
+    // load behave identically when everInjected is empty (toInject == topNow),
+    // but this asserts the contract: a fresh first user message gets the
+    // entire top-K rendered, not just a delta.
+    stageTurn([
+      { slug: "alice-vscode", denseScore: 0.9 },
+      { slug: "bob-coffee", denseScore: 0.8 },
+      { slug: "carol-jazz", denseScore: 0.7 },
+    ]);
+
+    const result = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "hi",
+      assistantMessage: "",
+      nowText: "",
+      messageId: "msg-1",
+      mode: "context-load",
+      config: makeConfig({ top_k: 3 }),
+    });
+
+    expect(result.block).not.toBeNull();
+    expect(result.block).toContain("### alice-vscode");
+    expect(result.block).toContain("### bob-coffee");
+    expect(result.block).toContain("### carol-jazz");
+    // The seeded directed edges (alice→bob, bob→alice, frontmatter-demo→alice)
+    // mean alice has two incoming predecessors and bob has one, so directed
+    // spread normalizes alice's activation more aggressively than bob's. The
+    // resulting rank order is bob > carol (no predecessors) > alice.
+    expect(new Set(result.toInject)).toEqual(
+      new Set(["alice-vscode", "bob-coffee", "carol-jazz"]),
+    );
+    expect(result.toInject).toHaveLength(3);
+
+    // All three slugs persisted to everInjected so the next per-turn doesn't
+    // re-attach the same content.
+    const persisted = await hydrate(db, "conv-1");
+    expect(new Set(persisted!.everInjected.map((e) => e.slug))).toEqual(
+      new Set(["alice-vscode", "bob-coffee", "carol-jazz"]),
+    );
+    expect(persisted!.everInjected).toHaveLength(3);
+  });
+
   test("`top_k_skills: 0` short-circuits to no skills subsection", async () => {
     // Even when the underlying mock would surface skills, the cap at 0 must
     // drop them via `selectSkillInjections.topK = 0` → empty `topNow`.
@@ -789,4 +964,220 @@ describe("injectMemoryV2Block", () => {
     expect(result.block).not.toContain("### Skills You Can Use");
     expect(result.block).not.toContain("example-skill-a");
   });
+
+  // ---------------------------------------------------------------------------
+  // Activation-log telemetry
+  // ---------------------------------------------------------------------------
+
+  test("writes one activation-log row per turn with concept rows partitioned and sorted", async () => {
+    // Turn 1: seed alice as injected so turn 2 has an `in_context` candidate.
+    stageTurn([{ slug: "alice-vscode", denseScore: 0.9 }]);
+    await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "Alice's editor",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+    expect(telemetryState.recordCalls.length).toBe(1);
+
+    // Turn 2: alice carries forward (now `in_context`); carol is freshly
+    // surfaced (`injected`); bob would be a candidate only if it carried
+    // forward, but with no prior bob entry it doesn't appear here.
+    stageTurn([
+      { slug: "alice-vscode", denseScore: 0.6 },
+      { slug: "carol-jazz", denseScore: 0.95 },
+    ]);
+    await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 2,
+      userMessage: "Carol's music",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-2",
+      config: makeConfig(),
+    });
+
+    expect(telemetryState.recordCalls.length).toBe(2);
+    const row = telemetryState.recordCalls[1] as {
+      conversationId: string;
+      turn: number;
+      mode: string;
+      concepts: Array<{
+        slug: string;
+        finalActivation: number;
+        status: string;
+        source: string;
+      }>;
+      skills: unknown[];
+      config: { top_k: number };
+    };
+    expect(row.conversationId).toBe("conv-1");
+    expect(row.turn).toBe(2);
+    expect(row.mode).toBe("per-turn");
+    expect(row.config.top_k).toBe(20);
+
+    // The candidate set is the union of fromPrior (alice) and fromAnn
+    // (alice + carol) → two concept rows.
+    expect(row.concepts.length).toBe(2);
+    const slugs = row.concepts.map((c) => c.slug);
+    expect(new Set(slugs)).toEqual(new Set(["alice-vscode", "carol-jazz"]));
+
+    // Sorted descending by finalActivation.
+    for (let i = 1; i < row.concepts.length; i++) {
+      expect(row.concepts[i - 1]!.finalActivation).toBeGreaterThanOrEqual(
+        row.concepts[i]!.finalActivation,
+      );
+    }
+
+    const byslug = new Map(row.concepts.map((c) => [c.slug, c]));
+    // Alice was attached on turn 1 → status `in_context` on turn 2.
+    expect(byslug.get("alice-vscode")!.status).toBe("in_context");
+    // Carol is freshly injected on turn 2.
+    expect(byslug.get("carol-jazz")!.status).toBe("injected");
+  });
+
+  test("context-load mode marks every rendered slug as `injected`, never `in_context`", async () => {
+    // Turn 1 (per-turn): seed alice as injected so the next turn's prior
+    // `everInjected` includes her — the same setup the per-turn telemetry
+    // test uses, so the difference between modes is unambiguous.
+    stageTurn([{ slug: "alice-vscode", denseScore: 0.9 }]);
+    await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "Alice's editor",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+    expect(telemetryState.recordCalls.length).toBe(1);
+
+    // Turn 2 in context-load mode (post-compaction or fresh load). Alice
+    // carries forward AND ranks high again; carol is a brand-new candidate.
+    // Both end up in `topNow` (and therefore in `slugsToRender` since
+    // context-load renders the full top-K). The status field must reflect
+    // that they were physically rendered into the new user message on this
+    // turn — `injected` for both — rather than reading `in_context` for
+    // alice based on stale prior `everInjected` state.
+    stageTurn([
+      { slug: "alice-vscode", denseScore: 0.6 },
+      { slug: "carol-jazz", denseScore: 0.95 },
+    ]);
+    await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 2,
+      userMessage: "Reload context",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-2",
+      mode: "context-load",
+      config: makeConfig(),
+    });
+
+    expect(telemetryState.recordCalls.length).toBe(2);
+    const row = telemetryState.recordCalls[1] as {
+      mode: string;
+      concepts: Array<{ slug: string; status: string }>;
+    };
+    expect(row.mode).toBe("context-load");
+
+    const byslug = new Map(row.concepts.map((c) => [c.slug, c]));
+    // Both rendered slugs read as `injected` — alice especially, even though
+    // she's in prior `everInjected`, because context-load actually rendered
+    // her into the fresh user message on this turn.
+    expect(byslug.get("alice-vscode")!.status).toBe("injected");
+    expect(byslug.get("carol-jazz")!.status).toBe("injected");
+
+    // No slug reads as `in_context` in context-load mode — the cache was
+    // wiped, so there is no prior cached attachment to reference.
+    for (const concept of row.concepts) {
+      expect(concept.status).not.toBe("in_context");
+    }
+  });
+
+  test("context-load mode marks candidates outside `slugsToRender` as `not_injected`", async () => {
+    // Turn 1 (per-turn): seed both alice and bob with positive activation
+    // so they survive into turn 2's prior-state candidate pool.
+    stageTurn([
+      { slug: "alice-vscode", denseScore: 0.9 },
+      { slug: "bob-coffee", denseScore: 0.8 },
+    ]);
+    await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "Alice's editor and Bob's coffee",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+
+    // Turn 2 (context-load) with `top_k: 1`: alice and bob both carry
+    // forward as candidates, but only the top-ranked slug is rendered.
+    // Whichever slug doesn't make the cut must read as `not_injected`.
+    stageTurn([
+      { slug: "alice-vscode", denseScore: 0.95 },
+      { slug: "bob-coffee", denseScore: 0.05 },
+    ]);
+    await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 2,
+      userMessage: "Reload context",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-2",
+      mode: "context-load",
+      config: makeConfig({ top_k: 1 }),
+    });
+
+    expect(telemetryState.recordCalls.length).toBe(2);
+    const row = telemetryState.recordCalls[1] as {
+      mode: string;
+      concepts: Array<{ slug: string; status: string }>;
+    };
+    expect(row.mode).toBe("context-load");
+
+    const byslug = new Map(row.concepts.map((c) => [c.slug, c]));
+    // Alice ranked first → she is in `slugsToRender` → `injected`.
+    expect(byslug.get("alice-vscode")!.status).toBe("injected");
+    // Bob was a candidate but didn't make `top_k: 1` → `not_injected`.
+    expect(byslug.get("bob-coffee")!.status).toBe("not_injected");
+  });
+
+  test("telemetry write failure is non-fatal — injection still returns a normal result", async () => {
+    telemetryState.recordShouldThrow = true;
+
+    stageTurn([{ slug: "alice-vscode", denseScore: 0.9 }]);
+    const result = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "Alice's editor",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+
+    // No row captured (the throw aborted the push), but the caller still
+    // produced a regular block + toInject result and persisted state.
+    expect(telemetryState.recordCalls.length).toBe(0);
+    expect(result.toInject).toEqual(["alice-vscode"]);
+    expect(result.block).not.toBeNull();
+    expect(result.block).toContain("### alice-vscode");
+
+    const persisted = await hydrate(db, "conv-1");
+    expect(persisted!.everInjected).toEqual([
+      { slug: "alice-vscode", turn: 1 },
+    ]);
+  });
 });