@vellumai/assistant 0.8.2 → 0.8.4

package/src/memory/v2/__tests__/injection-events.test.ts

@@ -0,0 +1,318 @@
+/**
+ * Tests for `assistant/src/memory/v2/injection-events.ts` and its sibling
+ * migration `256-memory-v2-injection-events.ts`.
+ *
+ * Coverage matrix:
+ *   - Migration creates the table + both indexes; safe to re-run.
+ *   - Backfill replays router-sourced concepts from memory_v2_activation_logs
+ *     and is idempotent on a forced re-run with cleared checkpoint.
+ *   - Backfill is a no-op when the activation-logs table doesn't exist
+ *     (pre-234 DB).
+ *   - recordInjectionEvents appends one row per slug per call; empty list
+ *     is a no-op.
+ *   - computeInjectionScore matches the closed-form decay at known deltas
+ *     (0d ≈ 1, 3d ≈ 0.5, 6d ≈ 0.25) and sums multiple events linearly.
+ *   - computeInjectionScores returns the same per-slug values in batch and
+ *     omits slugs with no events.
+ *
+ * Uses an in-memory bun:sqlite database — no real workspace DB.
+ */
+
+import { Database } from "bun:sqlite";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { makeMockLogger } from "../../../__tests__/helpers/mock-logger.js";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () => makeMockLogger(),
+}));
+
+import type { DrizzleDb } from "../../db-connection.js";
+import { getSqliteFrom } from "../../db-connection.js";
+import { migrateMemoryV2ActivationLogs } from "../../migrations/234-memory-v2-activation-logs.js";
+import {
+  downMemoryV2InjectionEvents,
+  migrateMemoryV2InjectionEvents,
+} from "../../migrations/256-memory-v2-injection-events.js";
+import * as schema from "../../schema.js";
+import {
+  computeInjectionScore,
+  computeInjectionScores,
+  INJECTION_SCORE_HALF_LIFE_MS,
+  recordInjectionEvents,
+} from "../injection-events.js";
+
+// memory_checkpoints is required by withCrashRecovery and is normally
+// created by an early core migration. Stand it up by hand so we can run
+// the v2 migrations in isolation against a fresh in-memory DB.
+const CHECKPOINTS_DDL = /*sql*/ `
+  CREATE TABLE memory_checkpoints (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL,
+    updated_at INTEGER NOT NULL
+  )
+`;
+
+let sqlite: Database;
+let database: DrizzleDb;
+
+beforeEach(() => {
+  sqlite = new Database(":memory:");
+  database = drizzle(sqlite, { schema });
+  getSqliteFrom(database).exec(CHECKPOINTS_DDL);
+});
+
+afterEach(() => {
+  sqlite.close();
+});
+
+function insertActivationLog(
+  rawDb: Database,
+  args: {
+    id: string;
+    concepts: Array<{ slug: string; source: string; status?: string }>;
+    createdAt: number;
+  },
+): void {
+  rawDb
+    .prepare(
+      `INSERT INTO memory_v2_activation_logs (
+        id, conversation_id, message_id, turn, mode,
+        concepts_json, skills_json, config_json, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      args.id,
+      "conv-1",
+      `msg-${args.id}`,
+      1,
+      "router",
+      JSON.stringify(args.concepts),
+      "[]",
+      "{}",
+      args.createdAt,
+    );
+}
+
+describe("migrateMemoryV2InjectionEvents", () => {
+  test("creates table and both indexes; safe to re-run", () => {
+    migrateMemoryV2InjectionEvents(database);
+    migrateMemoryV2InjectionEvents(database);
+
+    const raw = getSqliteFrom(database);
+    const table = raw
+      .query(
+        `SELECT name FROM sqlite_master WHERE type='table' AND name='memory_v2_injection_events'`,
+      )
+      .get();
+    expect(table).toBeTruthy();
+
+    const indexNames = new Set(
+      (
+        raw
+          .query(
+            `SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='memory_v2_injection_events'`,
+          )
+          .all() as Array<{ name: string }>
+      ).map((r) => r.name),
+    );
+    expect(indexNames.has("idx_memory_v2_injection_events_slug_time")).toBe(
+      true,
+    );
+    expect(indexNames.has("idx_memory_v2_injection_events_time")).toBe(true);
+  });
+
+  test("backfill replays router-sourced concepts and ignores carry_over", () => {
+    migrateMemoryV2ActivationLogs(database);
+    const raw = getSqliteFrom(database);
+    insertActivationLog(raw, {
+      id: "log-1",
+      concepts: [
+        { slug: "alice", source: "router", status: "injected" },
+        { slug: "bob", source: "router", status: "in_context" },
+        { slug: "ghost", source: "carry_over", status: "not_injected" },
+      ],
+      createdAt: 1_000_000,
+    });
+    insertActivationLog(raw, {
+      id: "log-2",
+      concepts: [{ slug: "alice", source: "router", status: "injected" }],
+      createdAt: 2_000_000,
+    });
+
+    migrateMemoryV2InjectionEvents(database);
+
+    const rows = raw
+      .query(
+        `SELECT slug, injected_at FROM memory_v2_injection_events ORDER BY injected_at, slug`,
+      )
+      .all() as Array<{ slug: string; injected_at: number }>;
+    expect(rows).toEqual([
+      { slug: "alice", injected_at: 1_000_000 },
+      { slug: "bob", injected_at: 1_000_000 },
+      { slug: "alice", injected_at: 2_000_000 },
+    ]);
+  });
+
+  test("backfill is a no-op when memory_v2_activation_logs is absent", () => {
+    // No activation-logs migration applied first.
+    expect(() => migrateMemoryV2InjectionEvents(database)).not.toThrow();
+    const { n } = getSqliteFrom(database)
+      .query(`SELECT COUNT(*) as n FROM memory_v2_injection_events`)
+      .get() as { n: number };
+    expect(n).toBe(0);
+  });
+
+  test("forced re-run does not double-insert existing events", () => {
+    migrateMemoryV2ActivationLogs(database);
+    const raw = getSqliteFrom(database);
+    insertActivationLog(raw, {
+      id: "log-1",
+      concepts: [{ slug: "alice", source: "router", status: "injected" }],
+      createdAt: 1_000_000,
+    });
+
+    migrateMemoryV2InjectionEvents(database);
+    // Simulate someone manually clearing the checkpoint — the in-table
+    // guard should still prevent re-backfill.
+    raw
+      .prepare(
+        `DELETE FROM memory_checkpoints WHERE key = 'migration_memory_v2_injection_events_v1'`,
+      )
+      .run();
+    migrateMemoryV2InjectionEvents(database);
+
+    const { n } = raw
+      .query(`SELECT COUNT(*) as n FROM memory_v2_injection_events`)
+      .get() as { n: number };
+    expect(n).toBe(1);
+  });
+
+  test("downMemoryV2InjectionEvents drops the table", () => {
+    migrateMemoryV2InjectionEvents(database);
+    downMemoryV2InjectionEvents(database);
+    const table = getSqliteFrom(database)
+      .query(
+        `SELECT name FROM sqlite_master WHERE type='table' AND name='memory_v2_injection_events'`,
+      )
+      .get();
+    expect(table).toBeFalsy();
+  });
+});
+
+describe("recordInjectionEvents", () => {
+  beforeEach(() => {
+    migrateMemoryV2InjectionEvents(database);
+  });
+
+  test("appends one row per slug at the same timestamp", () => {
+    const t = 1_000_000;
+    recordInjectionEvents(database, ["alice", "bob", "alice"], t);
+    const rows = getSqliteFrom(database)
+      .query(
+        `SELECT slug, injected_at FROM memory_v2_injection_events ORDER BY id`,
+      )
+      .all();
+    expect(rows).toEqual([
+      { slug: "alice", injected_at: t },
+      { slug: "bob", injected_at: t },
+      { slug: "alice", injected_at: t },
+    ]);
+  });
+
+  test("empty list is a no-op", () => {
+    recordInjectionEvents(database, [], 1_000_000);
+    const { n } = getSqliteFrom(database)
+      .query(`SELECT COUNT(*) as n FROM memory_v2_injection_events`)
+      .get() as { n: number };
+    expect(n).toBe(0);
+  });
+});
+
+describe("computeInjectionScore", () => {
+  beforeEach(() => {
+    migrateMemoryV2InjectionEvents(database);
+  });
+
+  test("returns 0 for a slug with no events", () => {
+    expect(computeInjectionScore(database, "missing", Date.now())).toBe(0);
+  });
+
+  test("single event 0 days ago → score ≈ 1", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(database, ["alice"], now);
+    expect(computeInjectionScore(database, "alice", now)).toBeCloseTo(1, 5);
+  });
+
+  test("single event 3 days (one half-life) ago → score ≈ 0.5", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - INJECTION_SCORE_HALF_LIFE_MS,
+    );
+    expect(computeInjectionScore(database, "alice", now)).toBeCloseTo(0.5, 5);
+  });
+
+  test("single event 6 days (two half-lives) ago → score ≈ 0.25", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - 2 * INJECTION_SCORE_HALF_LIFE_MS,
+    );
+    expect(computeInjectionScore(database, "alice", now)).toBeCloseTo(0.25, 5);
+  });
+
+  test("multiple events sum independently", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(database, ["alice"], now);
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - INJECTION_SCORE_HALF_LIFE_MS,
+    );
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - 2 * INJECTION_SCORE_HALF_LIFE_MS,
+    );
+    expect(computeInjectionScore(database, "alice", now)).toBeCloseTo(1.75, 5);
+  });
+});
+
+describe("computeInjectionScores", () => {
+  beforeEach(() => {
+    migrateMemoryV2InjectionEvents(database);
+  });
+
+  test("returns the same per-slug values as the single-slug helper", () => {
+    const now = 10_000_000_000;
+    recordInjectionEvents(database, ["alice", "bob"], now);
+    recordInjectionEvents(
+      database,
+      ["alice"],
+      now - INJECTION_SCORE_HALF_LIFE_MS,
+    );
+
+    const scores = computeInjectionScores(
+      database,
+      ["alice", "bob", "ghost"],
+      now,
+    );
+    expect(scores.get("alice")).toBeCloseTo(1.5, 5);
+    expect(scores.get("bob")).toBeCloseTo(1, 5);
+    // ghost has no events — omitted from the result, not present as 0.
+    expect(scores.has("ghost")).toBe(false);
+    expect(scores.get("alice")).toBeCloseTo(
+      computeInjectionScore(database, "alice", now),
+      5,
+    );
+  });
+
+  test("empty slug list returns empty map", () => {
+    expect(computeInjectionScores(database, [], Date.now()).size).toBe(0);
+  });
+});

package/src/memory/v2/__tests__/injection.test.ts

@@ -161,6 +161,38 @@ mock.module("../skill-store.js", () => ({
   listSkillEntries: () => Array.from(skillState.entries.values()),
 }));
 
+// ---------------------------------------------------------------------------
+// CLI-command-store mock
+// ---------------------------------------------------------------------------
+//
+// Mirrors the skill-store mock. CLI subcommand synthetic entries flow through
+// the unified pipeline under the `cli-commands/<name>` slug prefix and render
+// under `### CLI Commands You Can Use`. Tests stage `cliCommandState.entries`
+// and rely on `stageTurn` plumbing to land slugs in the candidate set.
+
+interface CliCommandEntryStub {
+  id: string;
+  description: string;
+  content: string;
+}
+
+const cliCommandState = {
+  entries: new Map<string, CliCommandEntryStub>(),
+};
+
+mock.module("../cli-command-store.js", () => ({
+  getCliCommandCapability: (idOrSlug: string) => {
+    const id = idOrSlug.startsWith("cli-commands/")
+      ? idOrSlug.slice("cli-commands/".length)
+      : idOrSlug;
+    return cliCommandState.entries.get(id) ?? null;
+  },
+  isCliCommandSlug: (slug: string) => slug.startsWith("cli-commands/"),
+  CLI_COMMAND_SLUG_PREFIX: "cli-commands/",
+  cliCommandSlugFor: (name: string) => `cli-commands/${name}`,
+  listCliCommandEntries: () => Array.from(cliCommandState.entries.values()),
+}));
+
 // ---------------------------------------------------------------------------
 // Activation-log store mock
 // ---------------------------------------------------------------------------
@@ -228,6 +260,8 @@ mock.module("../page-store.js", () => ({
 interface RouterResultStub {
   selectedSlugs: string[];
   failureReason: string | null;
+  /** Tier provenance per slug. Defaults to `tier3:0` for any selected slug. */
+  sourceBySlug?: Map<string, string>;
 }
 
 const routerState = {
@@ -238,12 +272,20 @@ const routerState = {
 mock.module("../router.js", () => ({
   runRouter: async () => {
     routerState.callCount++;
-    return (
-      routerState.nextResult ?? {
-        selectedSlugs: [],
-        failureReason: null,
-      }
-    );
+    const result = routerState.nextResult ?? {
+      selectedSlugs: [],
+      failureReason: null,
+    };
+    // Synthesize a default sourceBySlug for stubs that don't set one — pre-
+    // tier-provenance tests stage `selectedSlugs` only and expect every pick
+    // to flow through as a router selection. Treating them as `tier3:0` is
+    // the closest equivalent under the new model.
+    if (!result.sourceBySlug) {
+      const map = new Map<string, string>();
+      for (const slug of result.selectedSlugs) map.set(slug, "tier3:0");
+      result.sourceBySlug = map;
+    }
+    return result;
   },
 }));
 
@@ -356,8 +398,10 @@ import type { SkillEntry } from "../types.js";
 const { getSqliteFrom } = await import("../../db-connection.js");
 const { migrateActivationState } =
   await import("../../migrations/232-activation-state.js");
+const { migrateMemoryV2InjectionEvents } =
+  await import("../../migrations/256-memory-v2-injection-events.js");
 const schema = await import("../../schema.js");
-const { evictCompactedTurns, hydrate, save } =
+const { clearEverInjected, hydrate, save } =
   await import("../activation-store.js");
 const { injectMemoryV2Block } = await import("../injection.js");
 const { _resetMemoryV2QdrantForTests } = await import("../qdrant.js");
@@ -377,6 +421,7 @@ function createTestDb(): DrizzleDb {
     )
   `);
   migrateActivationState(db);
+  migrateMemoryV2InjectionEvents(db);
   return db;
 }
 
@@ -484,6 +529,7 @@ function resetState(): void {
   state.queryResponses.dense.length = 0;
   state.queryResponses.sparse.length = 0;
   skillState.entries.clear();
+  cliCommandState.entries.clear();
   telemetryState.recordCalls.length = 0;
   telemetryState.recordShouldThrow = false;
   pageStoreState.failingSlugs.clear();
@@ -503,6 +549,13 @@ function stageSkills(entries: SkillEntry[]): void {
   }
 }
 
+/** Stage cli-command-store cache entries for the upcoming render. */
+function stageCliCommands(entries: CliCommandEntryStub[]): void {
+  for (const entry of entries) {
+    cliCommandState.entries.set(entry.id, entry);
+  }
+}
+
 let db: DrizzleDb;
 beforeEach(() => {
   db = createTestDb();
@@ -653,10 +706,10 @@ describe("injectMemoryV2Block", () => {
       config: makeConfig(),
     });
 
-    // Simulate compaction: drop all everInjected entries with turn <= 1.
+    // Simulate compaction: clear the entire everInjected list.
     const beforeEvict = await hydrate(db, "conv-1");
     expect(beforeEvict).not.toBeNull();
-    const afterEvict = evictCompactedTurns(beforeEvict!, 1);
+    const afterEvict = clearEverInjected(beforeEvict!);
     expect(afterEvict.everInjected).toEqual([]);
     await save(db, "conv-1", afterEvict);
 
@@ -703,7 +756,7 @@ describe("injectMemoryV2Block", () => {
 
     expect(result.block).not.toBeNull();
     expect(result.block).toContain(
-      "**CRITICAL:** These are page summaries. Read the page file if it looks relevant.",
+      'Use `file_read("memory/concepts/path/to/file.md")` to read the full pages for any of the injected memory summaries you want more information on.',
     );
     expect(result.block).toContain(
       "# memory/concepts/summarized-page.md\nA short prose description",
@@ -736,11 +789,13 @@ describe("injectMemoryV2Block", () => {
     });
 
     expect(result.block).not.toBeNull();
-    // CRITICAL header appears exactly once.
-    const criticalCount = (
-      result.block!.match(/\*\*CRITICAL:\*\* These are page summaries\./g) ?? []
+    // Header appears exactly once.
+    const headerCount = (
+      result.block!.match(
+        /Use `file_read\("memory\/concepts\/path\/to\/file\.md"\)` to read/g,
+      ) ?? []
     ).length;
-    expect(criticalCount).toBe(1);
+    expect(headerCount).toBe(1);
     // summarized-page → short form (path + summary, no body, no frontmatter).
     expect(result.block).toContain("# memory/concepts/summarized-page.md\nA");
     expect(result.block).not.toContain("Long-form body content");
@@ -1060,6 +1115,153 @@ describe("injectMemoryV2Block", () => {
     expect(result.block).toBeNull();
   });
 
+  // ---------------------------------------------------------------------------
+  // CLI-command synthetic entries — same unified-pool plumbing as skills.
+  // ---------------------------------------------------------------------------
+
+  test("renders a retrieved cli-commands/<name> slug under CLI Commands You Can Use", async () => {
+    stageTurn([{ slug: "cli-commands/attachment", denseScore: 0.9 }]);
+    stageCliCommands([
+      {
+        id: "attachment",
+        description: "Manage file attachments for conversations",
+        content: 'The "assistant attachment" CLI command is available...',
+      },
+    ]);
+
+    const result = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "How do I register a video?",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+
+    expect(result.toInject).toEqual(["cli-commands/attachment"]);
+    expect(result.block).not.toBeNull();
+    const headerIdx = result.block!.indexOf("### CLI Commands You Can Use");
+    const lineIdx = result.block!.indexOf(
+      "- `assistant attachment`: Manage file attachments for conversations",
+    );
+    expect(headerIdx).toBeGreaterThan(-1);
+    expect(lineIdx).toBeGreaterThan(headerIdx);
+  });
+
+  test("renders concepts, skills, then cli-commands in that order in mixed blocks", async () => {
+    stageTurn([
+      { slug: "alice-vscode", denseScore: 0.95 },
+      { slug: "skills/example-skill-a", denseScore: 0.85 },
+      { slug: "cli-commands/config", denseScore: 0.75 },
+    ]);
+    stageSkills([
+      {
+        id: "example-skill-a",
+        content:
+          'The "Example Skill A" skill (example-skill-a) is available. Helps with examples.',
+      },
+    ]);
+    stageCliCommands([
+      {
+        id: "config",
+        description: "Manage configuration",
+        content: 'The "assistant config" CLI command is available...',
+      },
+    ]);
+
+    const result = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "Help me",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+
+    expect(new Set(result.toInject)).toEqual(
+      new Set([
+        "alice-vscode",
+        "skills/example-skill-a",
+        "cli-commands/config",
+      ]),
+    );
+    const conceptIdx = result.block!.indexOf(
+      "# memory/concepts/alice-vscode.md",
+    );
+    const skillsIdx = result.block!.indexOf("### Skills You Can Use");
+    const cliIdx = result.block!.indexOf("### CLI Commands You Can Use");
+    expect(conceptIdx).toBeGreaterThan(-1);
+    expect(skillsIdx).toBeGreaterThan(conceptIdx);
+    expect(cliIdx).toBeGreaterThan(skillsIdx);
+  });
+
+  test("cli-command slugs whose entry is missing from the cache are dropped silently", async () => {
+    stageTurn([{ slug: "cli-commands/missing-command", denseScore: 0.9 }]);
+
+    const result = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "anything",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+
+    expect(result.toInject).toEqual([]);
+    expect(result.block).toBeNull();
+
+    const persisted = await hydrate(db, "conv-1");
+    expect(persisted!.everInjected).toEqual([]);
+  });
+
+  test("cli-commands participate in everInjected so they dedupe across turns", async () => {
+    const entry = {
+      id: "config",
+      description: "Manage configuration",
+      content: 'The "assistant config" CLI command is available...',
+    };
+    stageTurn([{ slug: "cli-commands/config", denseScore: 0.9 }]);
+    stageCliCommands([entry]);
+    const result1 = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 1,
+      userMessage: "config",
+      assistantMessage: "",
+      nowText: "Now",
+      messageId: "msg-1",
+      config: makeConfig(),
+    });
+    expect(result1.toInject).toEqual(["cli-commands/config"]);
+    expect(result1.block).toContain("### CLI Commands You Can Use");
+
+    stageTurn([{ slug: "cli-commands/config", denseScore: 0.9 }]);
+    stageCliCommands([entry]);
+    const result2 = await injectMemoryV2Block({
+      database: db,
+      conversationId: "conv-1",
+      currentTurn: 2,
+      userMessage: "more config",
+      assistantMessage: "ok",
+      nowText: "Now",
+      messageId: "msg-2",
+      config: makeConfig(),
+    });
+    expect(result2.toInject).toEqual([]);
+    expect(result2.block).toBeNull();
+
+    const persisted = await hydrate(db, "conv-1");
+    expect(persisted!.everInjected).toEqual([
+      { slug: "cli-commands/config", turn: 1 },
+    ]);
+  });
+
   test("context-load mode renders topNow even when every slug was previously injected", async () => {
     // Turn 1 (per-turn): seed alice as injected.
     stageTurn([{ slug: "alice-vscode", denseScore: 0.9 }]);
@@ -1633,7 +1835,9 @@ describe("injectMemoryV2Block", () => {
       expect(row.mode).toBe("router");
       const aliceRow = row.concepts.find((c) => c.slug === "alice-vscode");
       expect(aliceRow).toBeDefined();
-      expect(aliceRow!.source).toBe("router");
+      // Default-stub provenance is `tier3:0` (single-batch path); see the
+      // runRouter mock for the synthesis rule.
+      expect(aliceRow!.source).toBe("tier3:0");
       expect(aliceRow!.status).toBe("injected");
       expect(aliceRow!.finalActivation).toBe(0);
       expect(aliceRow!.ownActivation).toBe(0);
@@ -1815,7 +2019,7 @@ describe("injectMemoryV2Block", () => {
       );
       expect(phantom).toBeDefined();
       expect(phantom!.status).toBe("page_missing");
-      expect(phantom!.source).toBe("router");
+      expect(phantom!.source).toBe("tier3:0");
     });
 
     test("flag-on: router re-picking a prior-everInjected slug does NOT re-render it; non-overlapping picks render and append to everInjected", async () => {
@@ -1920,7 +2124,7 @@ describe("injectMemoryV2Block", () => {
       expect(bobRow).toBeDefined();
       expect(aliceRow!.source).toBe("carry_over");
       expect(aliceRow!.status).toBe("in_context");
-      expect(bobRow!.source).toBe("router");
+      expect(bobRow!.source).toBe("tier3:0");
       expect(bobRow!.status).toBe("injected");
     });