@vellumai/assistant 0.5.7 → 0.5.8

package/src/__tests__/memory-lifecycle-e2e.test.ts

@@ -2,7 +2,7 @@
  * Memory lifecycle E2E regression test.
  *
  * Verifies the new memory pipeline end-to-end:
- * - 6-kind enum items (identity, preference, project, decision, constraint, event)
+ * - Standard-kind enum items (identity, preference, project, decision, constraint, event, journal, capability, ...)
  * - Supersession chains (supersedes/supersededBy fields)
  * - Hybrid search retrieval
  * - Two-layer XML injection format (<memory_context> with sections)
@@ -44,10 +44,31 @@ mock.module("../util/logger.js", () => ({
     }),
 }));
 
+// Stub the local embedding backend so the real ONNX model never loads
+mock.module("../memory/embedding-local.js", () => ({
+  LocalEmbeddingBackend: class {
+    readonly provider = "local" as const;
+    readonly model: string;
+    constructor(model: string) {
+      this.model = model;
+    }
+    async embed(texts: string[]): Promise<number[][]> {
+      return texts.map(() => new Array(384).fill(0));
+    }
+  },
+}));
+
+// Dynamic Qdrant mock: tests can push results to be returned by searchWithFilter/hybridSearch
+let mockQdrantResults: Array<{
+  id: string;
+  score: number;
+  payload: Record<string, unknown>;
+}> = [];
+
 mock.module("../memory/qdrant-client.js", () => ({
   getQdrantClient: () => ({
-    searchWithFilter: async () => [],
-    hybridSearch: async () => [],
+    searchWithFilter: async () => mockQdrantResults,
+    hybridSearch: async () => mockQdrantResults,
     upsertPoints: async () => {},
     deletePoints: async () => {},
   }),
@@ -60,7 +81,6 @@ const TEST_CONFIG = {
     ...DEFAULT_CONFIG.memory,
     embeddings: {
       ...DEFAULT_CONFIG.memory.embeddings,
-      provider: "openai" as const,
       required: false,
     },
     extraction: {
@@ -115,6 +135,7 @@ describe("Memory lifecycle E2E regression", () => {
     db.run("DELETE FROM conversations");
     db.run("DELETE FROM memory_jobs");
     db.run("DELETE FROM memory_checkpoints");
+    mockQdrantResults = [];
     resetCleanupScheduleThrottle();
     resetStaleSweepThrottle();
   });
@@ -128,7 +149,7 @@ describe("Memory lifecycle E2E regression", () => {
     }
   });
 
-  test("extraction produces items with 6-kind enum and supersession chains form correctly", async () => {
+  test("extraction produces items with standard-kind enum and supersession chains form correctly", async () => {
     const db = getDb();
     const now = 1_701_100_000_000;
     const conversationId = "conv-memory-lifecycle";
@@ -165,7 +186,7 @@ describe("Memory lifecycle E2E regression", () => {
       ])
       .run();
 
-    // Seed items using the 6-kind enum
+    // Seed items using the standard-kind enum
     const kinds = [
       "identity",
       "preference",
@@ -299,12 +320,8 @@ describe("Memory lifecycle E2E regression", () => {
       TEST_CONFIG,
     );
 
-    // Recency search finds segments but their finalScore (semantic*0.7 +
-    // recency*0.2 + confidence*0.1) is too low to pass tier classification
-    // (threshold > 0.6) because semantic=0 with Qdrant mocked empty.
-    // Verify recency search ran successfully.
-    expect(recall.recencyHits).toBeGreaterThan(0);
-    // Candidates exist but don't pass tier classification, so injectedText is empty
+    // Without semantic search (Qdrant mocked empty), no candidates pass
+    // tier classification (threshold > 0.6).
     expect(recall.enabled).toBe(true);
   });
 
@@ -343,15 +360,47 @@ describe("Memory lifecycle E2E regression", () => {
       })
       .run();
 
-    db.run(`
-      INSERT INTO memory_segments (
-        id, message_id, conversation_id, role, segment_index, text, token_estimate, scope_id, created_at, updated_at
-      ) VALUES (
-        'seg-injection-seed', 'msg-injection-seed', '${conversationId}', 'user', 0,
-        'My preferred timezone is America/Los_Angeles.', 10, 'default',
-        ${now + 10}, ${now + 10}
-      )
-    `);
+    // Seed a memory item so the semantic search path can find it
+    db.insert(memoryItems)
+      .values({
+        id: "item-timezone-pref",
+        kind: "preference",
+        subject: "timezone preference",
+        statement: "My preferred timezone is America/Los_Angeles.",
+        status: "active",
+        confidence: 0.9,
+        importance: 0.8,
+        fingerprint: "fp-item-timezone-pref",
+        firstSeenAt: now + 10,
+        lastSeenAt: now + 10,
+      })
+      .run();
+
+    db.insert(memoryItemSources)
+      .values({
+        memoryItemId: "item-timezone-pref",
+        messageId: "msg-injection-seed",
+        evidence: "timezone preference evidence",
+        createdAt: now + 10,
+      })
+      .run();
+
+    // Mock Qdrant to return the timezone preference item
+    mockQdrantResults = [
+      {
+        id: "emb-timezone-pref",
+        score: 0.92,
+        payload: {
+          target_type: "item",
+          target_id: "item-timezone-pref",
+          text: "My preferred timezone is America/Los_Angeles.",
+          kind: "preference",
+          status: "active",
+          created_at: now + 10,
+          last_seen_at: now + 10,
+        },
+      },
+    ];
 
     const recall = await buildMemoryRecall(
       "timezone",
@@ -359,10 +408,6 @@ describe("Memory lifecycle E2E regression", () => {
       TEST_CONFIG,
     );
 
-    // The recency-only promotion path (Step 6 in retriever) ensures the
-    // seeded segment reaches tier 2 and is injected even without semantic
-    // search. Verify structure of the two-layer XML format.
-    expect(recall.recencyHits).toBeGreaterThan(0);
     expect(recall.enabled).toBe(true);
     expect(recall.injectedText.length).toBeGreaterThan(0);
     expect(recall.injectedTokens).toBeGreaterThan(0);

package/src/__tests__/memory-recall-quality.test.ts

@@ -397,7 +397,6 @@ describe("Memory Recall Quality", () => {
         TEST_CONFIG,
       );
 
-      expect(recall.recencyHits).toBeGreaterThan(0);
       expect(recall.enabled).toBe(true);
       // With high-scoring Qdrant results, items should be injected
       expect(recall.semanticHits).toBeGreaterThan(0);
@@ -502,7 +501,6 @@ describe("Memory Recall Quality", () => {
         TEST_CONFIG,
       );
 
-      expect(recall.recencyHits).toBeGreaterThan(0);
       expect(recall.enabled).toBe(true);
       // High-importance preference should be injected
       expect(recall.injectedText).toContain("TypeScript");
@@ -566,7 +564,6 @@ describe("Memory Recall Quality", () => {
       );
 
       // Recency search finds the segment but tier classification filters it
-      expect(recall.recencyHits).toBeGreaterThan(0);
       // Superseded items should not leak into injected text
       expect(recall.injectedText).not.toContain("vim for editing code");
     });
@@ -623,7 +620,6 @@ describe("Memory Recall Quality", () => {
 
       // Recency search finds segments but tier classification filters them.
       // Key assertion: superseded MySQL item should not leak.
-      expect(recall.recencyHits).toBeGreaterThan(0);
       expect(recall.injectedText).not.toContain("MySQL");
     });
 
@@ -671,13 +667,29 @@ describe("Memory Recall Quality", () => {
         firstSeenAt: now - 50_000,
       });
 
+      // Mock Qdrant to return the active item as a semantic search result
+      mockQdrantResults = [
+        {
+          id: "emb-framework-active",
+          score: 0.92,
+          payload: {
+            target_type: "item",
+            target_id: "item-framework-active",
+            text: "Framework preference is React for this codebase",
+            kind: "preference",
+            status: "active",
+            created_at: now,
+            last_seen_at: now,
+          },
+        },
+      ];
+
       const recall = await buildMemoryRecall(
         "framework preference",
         "conv-invalid-status",
         TEST_CONFIG,
       );
-      expect(recall.recencyHits).toBeGreaterThan(0);
-      // Active segment content should be injected; invalidated item should not leak
+      // Active item should be injected via semantic search; invalidated item should not leak
       expect(recall.injectedText).toContain("React");
       expect(recall.injectedText).not.toContain("Angular");
     });
@@ -765,7 +777,6 @@ describe("Memory Recall Quality", () => {
         TEST_CONFIG,
       );
 
-      expect(recall.recencyHits).toBeGreaterThan(0);
       expect(recall.enabled).toBe(true);
       // Recent Bun item should be injected, old Node reference should not
       expect(recall.injectedText).toContain("Bun");
@@ -870,7 +881,6 @@ describe("Memory Recall Quality", () => {
         TEST_CONFIG,
       );
 
-      expect(recall.recencyHits).toBeGreaterThan(0);
       expect(recall.enabled).toBe(true);
       // Frequently accessed timezone item should be in injected text
       expect(recall.injectedText).toContain("America/Los_Angeles");
@@ -940,7 +950,6 @@ describe("Memory Recall Quality", () => {
         TEST_CONFIG,
       );
 
-      expect(recall.recencyHits).toBeGreaterThan(0);
       expect(recall.enabled).toBe(true);
       // Deployment rule should be injected
       expect(recall.injectedText).toContain("staging");
@@ -987,7 +996,7 @@ describe("Memory Recall Quality", () => {
       );
     });
 
-    test("precision@k guard verifies pipeline completes with seeded segments", async () => {
+    test("precision@k guard verifies pipeline completes with seeded items", async () => {
       const db = getDb();
       const now = 1_700_000_700_000;
       insertConversation(db, "conv-pk", now, 3);
@@ -995,17 +1004,17 @@ describe("Memory Recall Quality", () => {
       const prefs = [
         {
           msg: "msg-pk-1",
-          seg: "seg-pk-1",
+          item: "item-pk-1",
           text: "I prefer dark mode over light mode",
         },
         {
           msg: "msg-pk-2",
-          seg: "seg-pk-2",
+          item: "item-pk-2",
           text: "I like using TypeScript for all projects",
         },
         {
           msg: "msg-pk-3",
-          seg: "seg-pk-3",
+          item: "item-pk-3",
           text: "I prefer tabs over spaces for indentation",
         },
       ];
@@ -1014,18 +1023,40 @@ describe("Memory Recall Quality", () => {
         const p = prefs[i]!;
         const t = now + i * 1000;
         insertMessage(db, p.msg, "conv-pk", "user", p.text, t);
-        insertSegment(db, p.seg, p.msg, "conv-pk", "user", p.text, t);
+        insertItem(db, {
+          id: p.item,
+          kind: "preference",
+          subject: `preference-${i}`,
+          statement: p.text,
+          importance: 0.8,
+          firstSeenAt: t,
+        });
+        insertItemSource(db, p.item, p.msg, t);
       }
 
+      // Mock Qdrant to return all three preference items
+      mockQdrantResults = prefs.map((p, i) => ({
+        id: `emb-pk-${i}`,
+        score: 0.9 - i * 0.05,
+        payload: {
+          target_type: "item",
+          target_id: p.item,
+          text: p.text,
+          kind: "preference",
+          status: "active",
+          created_at: now + i * 1000,
+          last_seen_at: now + i * 1000,
+        },
+      }));
+
       const recall = await buildMemoryRecall(
         "what do I prefer",
         "conv-pk",
         TEST_CONFIG,
       );
 
-      // Recency-only candidates are promoted to tier 2 and injected.
-      // Verify the pipeline recalled the preference content.
-      expect(recall.recencyHits).toBeGreaterThan(0);
+      // Semantic search returns all three preference items which pass
+      // tier classification and are injected.
       expect(recall.enabled).toBe(true);
       assertPrecisionAtK(
         recall.injectedText,