@ghilteras/opencode-agent-memory 1.0.0 → 2.0.0

package/src/journal.test.ts

@@ -1,45 +1,48 @@
-import { afterEach, describe, expect, mock, test } from "bun:test";
+import { afterEach, beforeAll, describe, expect, mock, test } from "bun:test";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
+import * as realEmbeddings from "./embeddings";
+import NativeMiniSearch from "minisearch";
 
-import { createJournalStore, loadConfig } from "./journal";
+let embeddingGenerationCalls = 0;
+let lexicalIndexConstructions = 0;
 
-// Mock the embeddings module to avoid downloading a real model in tests
+class CountingMiniSearch extends (NativeMiniSearch as any) {
+  constructor(...args: any[]) {
+    super(...args);
+    lexicalIndexConstructions++;
+  }
+}
+
+mock.module("minisearch", () => ({ default: CountingMiniSearch }));
+
+// Mock the embeddings module to avoid downloading a real model in tests.
+// Spread the real module so every export not overridden here keeps its real
+// value regardless of test-file discovery order.
 mock.module("./embeddings", () => ({
-  generateEmbedding: async (text: string) => {
-    // Deterministic fake embedding based on text content
-    const hash = Array.from(text).reduce(
-      (acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0,
-      0,
-    );
-    return Array.from({ length: 8 }, (_, i) => Math.sin(hash + i));
+  ...realEmbeddings,
+  generateEmbedding: async () => {
+    embeddingGenerationCalls++;
+    return Array.from({ length: 8 }, (_, i) => i / 8);
   },
   EMBEDDING_DIMENSION: 8,
   embeddingModelName: () => "test-model",
-  cosineSimilarity: (a: number[], b: number[]) => {
-    if (a.length !== b.length)
-      throw new Error(
-        `Embedding dimension mismatch: ${a.length} vs ${b.length}`,
-      );
-    let dot = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < a.length; i++) {
-      const ai = a[i]!;
-      const bi = b[i]!;
-      dot += ai * bi;
-      normA += ai * ai;
-      normB += bi * bi;
-    }
-    const denom = Math.sqrt(normA) * Math.sqrt(normB);
-    return denom === 0 ? 0 : dot / denom;
-  },
 }));
 
 async function mkTmpDir(): Promise<string> {
   return fs.mkdtemp(path.join("/tmp/", "opencode-journal-"));
 }
 
+// Bind the journal module AFTER the embeddings mock registration (above) via
+// dynamic import, so the store uses the fake embeddings regardless of bun's
+// module-mock rewiring semantics; a static import would be hoisted above it.
+let createJournalStore!: (typeof import("./journal"))["createJournalStore"];
+let loadConfig!: (typeof import("./journal"))["loadConfig"];
+
+beforeAll(async () => {
+  ({ createJournalStore, loadConfig } = await import("./journal"));
+});
+
 describe("loadConfig", () => {
   test("classifies a missing file and returns an empty config", async () => {
     const dir = await mkTmpDir();
@@ -131,6 +134,8 @@ describe("journal store", () => {
   let tmpDir: string;
 
   afterEach(async () => {
+    embeddingGenerationCalls = 0;
+    lexicalIndexConstructions = 0;
     if (tmpDir) {
       await fs.rm(tmpDir, { recursive: true, force: true });
     }
@@ -403,7 +408,7 @@ describe("journal store", () => {
     expect(result.allTags).toEqual(["debugging", "perf", "rust"]);
   });
 
-  test("search by text uses semantic matching", async () => {
+  test("search by text uses lexical OR matching", async () => {
     tmpDir = await mkTmpDir();
     const store = createJournalStore(tmpDir);
 
@@ -417,13 +422,11 @@ describe("journal store", () => {
       body: "Set up pytest with coverage reporting.",
     });
 
-    // With mock embeddings, text search falls through to semantic matching
-    const result = await store.search({ text: "Rust performance" });
-    // Should find at least the matching entry
-    expect(result.total).toBeGreaterThan(0);
+    const result = await store.search({ text: "Rust absentterm" });
+    expect(result.entries.map((entry) => entry.title)).toEqual(["Rust performance"]);
   });
 
-  test("title query surfaces long-body entry despite diluted embedding", async () => {
+  test("title query matches terms from the title", async () => {
     tmpDir = await mkTmpDir();
     const store = createJournalStore(tmpDir);
     const entry = await store.write({
@@ -431,191 +434,205 @@ describe("journal store", () => {
       body: "Unrelated topics include coastal erosion, orbital mechanics, ceramic glazing, " +
         "database indexing, ancient trade routes, orchard irrigation, keyboard switches, " +
         "bird migration, public transit planning, and archival paper chemistry. These notes " +
-        "continue across varied subjects so the long body dilutes its mean-pooled embedding. " +
+        "continue across varied subjects in this deliberately long journal entry. " +
         "Further unrelated details cover weather instruments, typography, fermentation, " +
         "geology, woodworking, astronomy, logistics, and museum conservation. The entry is " +
-        "deliberately verbose and semantically unrelated to a fresh exact-title query, while " +
+        "deliberately verbose and does not repeat the title terms in its body, while " +
         "its title remains the searchable anchor for this regression case.",
     });
 
-    // The deterministic mock embeddings produce an unrelated query vector here;
-    // the title-anchor floor guarantees presence regardless of cosine score.
     const result = await store.search({ text: "Agent-memory release delivery channel" });
     expect(result.entries.some((candidate) => candidate.id === entry.id)).toBe(true);
   });
 
-  test("search skips stale-dimension embedding and falls back to text match", async () => {
+  test("lexical matching finds a rare exact token", async () => {
     tmpDir = await mkTmpDir();
     const store = createJournalStore(tmpDir);
+    const entry = await store.write({ title: "Unrelated note", body: "The rarelexicaltoken appears here." });
+    const query = "rarelexicaltoken";
 
-    const entry = await store.write({
-      title: "Stale dims entry",
-      body: "unique-stale-dim-content",
-    });
-
-    // First search: caches the entry with the correct mock embedding.
-    await store.search({ text: "unique-stale-dim-content" });
-
-    // Simulate a v2 embedding with a WRONG dimension (e.g. from a 768d model).
-    // Search must not throw: it should skip the embedding and fall back to
-    // text matching for this entry.
-    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
-    await fs.writeFile(
-      embeddingFile,
-      JSON.stringify({
-        v: 2,
-        model: "some-768d-model",
-        dimension: 768,
-        vector: Array.from({ length: 768 }, () => 0.1),
-      }),
-      "utf-8",
-    );
-
-    // Query whose embedding would collide with the stale one if cosine were
-    // attempted (dimension mismatch would throw inside cosineSimilarity).
-    const result = await store.search({ text: "unique-stale-dim-content" });
-    const titles = result.entries.map((e) => e.title);
-    expect(titles).toContain("Stale dims entry");
-    // No throw above is the assertion: mismatched dimension degrades to text.
+    const result = await store.search({ text: query });
+    expect(result.total).toBe(1);
+    expect(result.entries.map((e) => e.id)).toEqual([entry.id]);
   });
 
-  test("sidecar rewrite after cache warm is picked up by second search", async () => {
+  test("OR matching returns unique matches and stable pagination across identical calls", async () => {
     tmpDir = await mkTmpDir();
     const store = createJournalStore(tmpDir);
+    for (let i = 0; i < 4; i++) {
+      await store.write({ title: `Stable ${i}`, body: "sharedtoken searchable" });
+      await new Promise((r) => setTimeout(r, 5));
+    }
 
-    const entry = await store.write({
-      title: "Sidecar invalidation",
-      body: "unique-sidecar-invalidation-content",
-    });
-
-    // First search: caches the entry + correct sidecar fingerprint.
-    const r1 = await store.search({ text: "unique-sidecar-invalidation-content" });
-    expect(r1.entries.some((e) => e.title === "Sidecar invalidation")).toBe(true);
-
-    // Rewrite the sidecar with a wrong-dimension embedding (simulating
-    // sidecar regeneration without touching the .md).
-    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
-    await fs.writeFile(
-      embeddingFile,
-      JSON.stringify({
-        v: 2,
-        model: "some-768d-model",
-        dimension: 768,
-        vector: Array.from({ length: 768 }, () => 0.1),
-      }),
-      "utf-8",
-    );
-
-    // Second search must invalidate the cache (sidecar mtime changed) and
-    // pick up the stale-dimension embedding, falling back to text match.
-    const r2 = await store.search({ text: "unique-sidecar-invalidation-content" });
-    expect(r2.entries.some((e) => e.title === "Sidecar invalidation")).toBe(true);
+    const first = await store.search({ text: "sharedtoken", limit: 2 });
+    const second = await store.search({ text: "sharedtoken", limit: 2 });
+    expect(first.total).toBe(4);
+    expect(second.total).toBe(4);
+    expect(first.entries.map((e) => e.id)).toEqual(second.entries.map((e) => e.id));
+    const next = await store.search({ text: "sharedtoken", limit: 2, offset: 2 });
+    expect([...first.entries, ...next.entries].map((e) => e.id)).toHaveLength(4);
+    expect(new Set([...first.entries, ...next.entries].map((e) => e.id)).size).toBe(4);
   });
 
-  test("sidecar deleted between searches falls back to text match", async () => {
+  test("identical text searches reuse the lexical index snapshot", async () => {
     tmpDir = await mkTmpDir();
     const store = createJournalStore(tmpDir);
-
-    const entry = await store.write({
-      title: "Sidecar deleted entry",
-      body: "unique-sidecar-deleted-content",
-    });
-
-    // First search: caches the entry + sidecar fingerprint.
-    const r1 = await store.search({ text: "unique-sidecar-deleted-content" });
-    expect(r1.entries.some((e) => e.title === "Sidecar deleted entry")).toBe(true);
-
-    // Delete the sidecar file.
-    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
-    await fs.unlink(embeddingFile);
-
-    // Second search: cache miss (sidecar gone), falls back to text match
-    // (score 0.5) rather than using the stale cached embedding.
-    const r2 = await store.search({ text: "unique-sidecar-deleted-content" });
-    const found = r2.entries.find((e) => e.title === "Sidecar deleted entry");
-    expect(found).toBeDefined();
-    // Score is 0.5 (text fallback), not a cosine score
+    await store.write({ title: "Cache token", body: "repeated searchable material" });
+    const before = lexicalIndexConstructions;
+    await store.search({ text: "cache" });
+    expect(lexicalIndexConstructions).toBe(before + 1);
+    await store.search({ text: "cache" });
+    expect(lexicalIndexConstructions).toBe(before + 1);
   });
 
-  test("search skips corrupt v2 embedding (empty vector) without throwing", async () => {
+  test("adding an entry rebuilds the index and makes it searchable", async () => {
     tmpDir = await mkTmpDir();
     const store = createJournalStore(tmpDir);
+    await store.write({ title: "Before", body: "ordinary content" });
+    const before = lexicalIndexConstructions;
+    await store.search({ text: "newlyaddedtoken" });
+    await store.write({ title: "New entry", body: "newlyaddedtoken appears here" });
+    const result = await store.search({ text: "newlyaddedtoken" });
+    expect(lexicalIndexConstructions).toBe(before + 2);
+    expect(result.entries.map((entry) => entry.title)).toEqual(["New entry"]);
+  });
 
-    const entry = await store.write({
-      title: "Corrupt embedding entry",
-      body: "unique-corrupt-embedding-content",
-    });
-
-    // Simulate a corrupt-but-well-formed v2 embedding: dimension claims 384
-    // but the vector is empty. Search must NOT throw — it should skip the
-    // embedding and fall back to text matching for this entry.
-    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
-    await fs.writeFile(
-      embeddingFile,
-      JSON.stringify({ v: 2, model: "some-model", dimension: 8, vector: [] }),
-      "utf-8",
-    );
-
-    const result = await store.search({ text: "unique-corrupt-embedding-content" });
-    const titles = result.entries.map((e) => e.title);
-    expect(titles).toContain("Corrupt embedding entry");
-    // No throw above is the assertion: corrupt v2 embedding degrades to text.
+  test("editing entry bytes invalidates the index through the entry stat cache", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+    const entry = await store.write({ title: "Editable", body: "old unique phrase" });
+    await store.search({ text: "old" });
+    const before = lexicalIndexConstructions;
+    const original = await fs.readFile(entry.filePath, "utf-8");
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    await fs.writeFile(entry.filePath, original.replace("old unique phrase", "replacementtoken with more bytes"));
+    const oldResult = await store.search({ text: "old" });
+    const newResult = await store.search({ text: "replacementtoken" });
+    expect(lexicalIndexConstructions).toBe(before + 1);
+    expect(oldResult.total).toBe(0);
+    expect(newResult.entries.map((candidate) => candidate.id)).toEqual([entry.id]);
+  });
+
+  test("a metadata filter changing membership rebuilds the lexical index", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+    await store.write({ title: "Alpha", body: "sharedfiltertoken", project: "/proj/a" });
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    await store.write({ title: "Beta", body: "sharedfiltertoken", project: "/proj/b" });
+    const before = lexicalIndexConstructions;
+    const a = await store.search({ text: "sharedfiltertoken", project: "/proj/a" });
+    const b = await store.search({ text: "sharedfiltertoken", project: "/proj/b" });
+    expect(lexicalIndexConstructions).toBe(before + 2);
+    expect(a.entries.map((entry) => entry.title)).toEqual(["Alpha"]);
+    expect(b.entries.map((entry) => entry.title)).toEqual(["Beta"]);
+  });
+
+  test("interleaved searches with different filters retain their own index", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+    const a = await store.write({ title: "Filtered A", body: "shared interleaved token", project: "/proj/a" });
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    const b = await store.write({ title: "Filtered B", body: "shared interleaved token", project: "/proj/b" });
+    const [resultA, resultB] = await Promise.all([
+      store.search({ text: "interleaved", project: "/proj/a" }),
+      store.search({ text: "interleaved", project: "/proj/b" }),
+    ]);
+    expect(resultA.entries.map((entry) => entry.id)).toEqual([a.id]);
+    expect(resultB.entries.map((entry) => entry.id)).toEqual([b.id]);
+    expect(resultA.total).toBe(1);
+    expect(resultB.total).toBe(1);
   });
 
-  test("search skips corrupt v2 embedding (missing vector) without throwing", async () => {
+  test("OR rankings and totals match an uncached MiniSearch fixture", async () => {
     tmpDir = await mkTmpDir();
     const store = createJournalStore(tmpDir);
+    const docs = [
+      ["Alpha systems", "blue green systems reliability"],
+      ["Green garden", "green plants bloom in spring"],
+      ["Blue ocean", "blue water and marine life"],
+      ["Reliable storage", "systems store durable records"],
+      ["Spring systems", "green systems launch in spring"],
+      ["Marine records", "ocean blue records and data"],
+      ["Durable garden", "plants grow with reliable soil"],
+      ["Data storage", "durable systems for data"],
+      ["Blooming ocean", "spring marine bloom"],
+      ["Record reliability", "reliable records stay durable"],
+    ];
+    const entries = [] as Awaited<ReturnType<typeof store.write>>[];
+    for (const [title, body] of docs) {
+      entries.push(await store.write({ title: title!, body: body! }));
+      await new Promise((resolve) => setTimeout(resolve, 2));
+    }
+    const queries = ["green systems", "blue records", "durable reliable", "spring ocean", "plants data"];
+    for (const text of queries) {
+      const uncached = new NativeMiniSearch({ fields: ["title", "body"], storeFields: ["id"] });
+      uncached.addAll(entries.map((entry) => ({ id: entry.id, title: entry.title, body: entry.body })));
+      const expected = uncached.search(text, { combineWith: "OR" })
+        .map((match) => ({ entry: entries.find((entry) => entry.id === String(match.id))!, score: match.score }))
+        .sort((a, b) => b.score - a.score || b.entry.created.getTime() - a.entry.created.getTime() || (a.entry.id < b.entry.id ? 1 : a.entry.id > b.entry.id ? -1 : 0));
+      const actual = await store.search({ text });
+      expect({ ids: actual.entries.map((entry) => entry.id), total: actual.total }).toEqual({
+        ids: expected.map(({ entry }) => entry.id),
+        total: expected.length,
+      });
+    }
+  });
 
-    const entry = await store.write({
-      title: "Missing vector entry",
-      body: "unique-missing-vector-content",
-    });
+  test("punctuation-only query falls back to case-insensitive title/body substring matches", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+    const titleMatch = await store.write({ title: "!!!", body: "older" });
+    await new Promise((r) => setTimeout(r, 5));
+    const bodyMatch = await store.write({ title: "Other", body: "A marker !!! appears here." });
+    await new Promise((r) => setTimeout(r, 5));
+    await store.write({ title: "No match", body: "ordinary text" });
+    const result = await store.search({ text: "!!!" });
+    expect(result.total).toBe(2);
+    expect(result.entries.map((entry) => entry.id)).toEqual([bodyMatch.id, titleMatch.id]);
+  });
 
-    // v2 file that omits the vector field entirely. Search must not throw.
-    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
-    await fs.writeFile(
-      embeddingFile,
-      JSON.stringify({ v: 2, model: "some-model", dimension: 8 }),
-      "utf-8",
-    );
+  test("text search does not generate embeddings", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+    const entry = await store.write({ title: "Keyword note", body: "the lexicalneedle is here" });
+    const callsBeforeSearch = embeddingGenerationCalls;
 
-    const result = await store.search({ text: "unique-missing-vector-content" });
-    const titles = result.entries.map((e) => e.title);
-    expect(titles).toContain("Missing vector entry");
+    const result = await store.search({ text: "lexicalneedle" });
+    expect(result.entries.map((candidate) => candidate.id)).toEqual([entry.id]);
+    expect(embeddingGenerationCalls - callsBeforeSearch).toBe(0);
   });
 
-  test("legacy bare-array embedding is still usable for semantic search", async () => {
+  test("text search matches query terms with OR and orders score, creation time, then id descending", async () => {
     tmpDir = await mkTmpDir();
     const store = createJournalStore(tmpDir);
+    const low = await store.write({ title: "Tie", body: "alpha" });
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    const highId = await store.write({ title: "Tie", body: "alpha" });
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    const newest = await store.write({ title: "Tie", body: "beta alpha" });
+    const tiedCreated = "2026-01-01T00:00:00.000Z";
+    for (const entry of [low, highId]) {
+      const original = await fs.readFile(entry.filePath, "utf-8");
+      await fs.writeFile(entry.filePath, original.replace(/^created: .+$/m, `created: '${tiedCreated}'`));
+    }
 
-    const entry = await store.write({
-      title: "Legacy embedding",
-      body: "legacy-vector-body",
-    });
+    const result = await store.search({ text: "alpha beta" });
+    expect(result.entries.map((entry) => entry.id)).toEqual([newest.id, highId.id, low.id]);
+    expect(result.total).toBe(3);
+  });
 
-    // Overwrite with a legacy v1 bare-array embedding (from all-MiniLM-L6-v2,
-    // which also produces 8-dim vectors under the test mock). Use the same
-    // mock formula as generateEmbedding so the cosine is positive and the
-    // entry is actually matched via semantic search.
-    const hash = Array.from("legacy-vector-body").reduce(
-      (acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0,
-      0,
-    );
-    const legacyVector = Array.from(
-      { length: 8 },
-      (_, i) => Math.sin(hash + i),
-    );
-    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
-    await fs.writeFile(
-      embeddingFile,
-      JSON.stringify(legacyVector),
-      "utf-8",
-    );
+  test("search without text remains chronological and does not generate embeddings", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+    await store.write({ title: "Older", body: "first" });
+    await new Promise((r) => setTimeout(r, 5));
+    await store.write({ title: "Newer", body: "second" });
+    const callsBeforeSearch = embeddingGenerationCalls;
+    const indexesBeforeSearch = lexicalIndexConstructions;
 
-    // Legacy array must be treated as { vector, model: "unknown-legacy",
-    // dimension: length }; with a matching dimension it stays usable.
-    const result = await store.search({ text: "legacy-vector-body" });
-    const titles = result.entries.map((e) => e.title);
-    expect(titles).toContain("Legacy embedding");
+    const result = await store.search({});
+    expect(result.entries.map((entry) => entry.title)).toEqual(["Newer", "Older"]);
+    expect(embeddingGenerationCalls).toBe(callsBeforeSearch);
+    expect(lexicalIndexConstructions).toBe(indexesBeforeSearch);
   });
 });