@ghilteras/opencode-agent-memory 0.3.1 → 0.4.1

package/README.md

@@ -1,12 +1,13 @@
 # opencode-agent-memory
 
-> **Fork notice**: This is the maintained fork `@ghilteras/opencode-agent-memory` v0.3.0 of
+> **Fork notice**: This is the maintained fork `@ghilteras/opencode-agent-memory` v0.4.0 of
 > [opencode-agent-memory](https://github.com/joshuadavidthomas/opencode-agent-memory)
 > (joshuadavidthomas, MIT). This release adopts the "proper timestamps without cache
 > busting" design (per-block `modified_at` frontmatter, remove the volatile
 > `memory_metadata` block, append memory XML at the end of the system prompt) derived
 > from the Annakan/draxxris fork plan. Upstream PR #20 (freeze metadata timestamps) is
-> superseded by the stronger per-block design and remains open upstream.
+> superseded by the stronger per-block design and remains open upstream. v0.4.0 swaps
+> the embedding model to a multilingual one and adds a versioned embedding format.
 
 [Letta](https://letta.com)-style editable [memory blocks](https://docs.letta.com/guides/agents/memory-blocks/) for [OpenCode](https://opencode.ai).
 
@@ -86,7 +87,22 @@ When the journal is enabled, the agent gets 3 additional tools:
 | `journal_search` | Search entries semantically, filter by project or tags, with pagination |
 | `journal_read` | Read a specific journal entry by ID |
 
-Journal entries are append-only markdown files with YAML frontmatter, stored in `~/.config/opencode/journal/`. Each entry records which project, model, provider, agent, and session it was written from. Semantic search uses local embeddings ([all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2)) - no data leaves your machine.
+Journal entries are append-only markdown files with YAML frontmatter, stored in `~/.config/opencode/journal/`. Each entry records which project, model, provider, agent, and session it was written from. Semantic search uses local embeddings ([paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2), 384d, multilingual EN/IT) - no data leaves your machine.
+
+Embedding files (`.embedding`, written alongside each entry) use a versioned format (`{ v: 2, model, dimension, vector }`) so that stale or mismatched embeddings are detected: if the stored dimension doesn't match the current model, the entry falls back to text matching instead of failing the search. Legacy bare-array embeddings from v0.3.x remain readable.
+
+### cacheDir Configuration
+
+By default the transformers.js model cache lives in the default location (`~/.cache/huggingface`). You can relocate it by setting `cacheDir` at the top level of `~/.config/opencode/agent-memory.json`:
+
+```json
+{
+  "cacheDir": "/home/angelo/.cache/opencode/memory-model",
+  "journal": {
+    "enabled": true
+  }
+}
+```
 
 ### Default Blocks
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ghilteras/opencode-agent-memory",
-  "version": "0.3.1",
+  "version": "0.4.1",
   "description": "Letta-style editable memory blocks + journal for OpenCode (maintained fork of opencode-agent-memory).",
   "author": "Angelo Pantano <ghilteras@protonmail.com>",
   "license": "MIT",

package/src/embeddings.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  EMBEDDING_DIMENSION,
+  cosineSimilarity,
+  embeddingModelName,
+  getEmbeddingDimension,
+} from "./embeddings";
+
+describe("embeddings", () => {
+  test("getEmbeddingDimension returns 384", () => {
+    expect(getEmbeddingDimension()).toBe(384);
+    expect(EMBEDDING_DIMENSION).toBe(384);
+  });
+
+  test("embeddingModelName returns the multilingual model", () => {
+    expect(embeddingModelName()).toBe(
+      "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
+    );
+  });
+
+  test("cosineSimilarity throws on dimension mismatch", () => {
+    expect(() => cosineSimilarity([1, 2, 3], [1, 2, 3, 4])).toThrow(
+      /dimension mismatch/i,
+    );
+  });
+
+  test("versioned embedding round-trip serializes and parses back", () => {
+    const vector = Array.from(
+      { length: EMBEDDING_DIMENSION },
+      (_, i) => Math.sin(i),
+    );
+    const serialized = JSON.stringify({
+      v: 2,
+      model: embeddingModelName(),
+      dimension: EMBEDDING_DIMENSION,
+      vector,
+    });
+
+    const parsed = JSON.parse(serialized);
+    expect(parsed.v).toBe(2);
+    expect(parsed.model).toBe(embeddingModelName());
+    expect(parsed.dimension).toBe(EMBEDDING_DIMENSION);
+    expect(parsed.vector).toEqual(vector);
+    expect(parsed.vector.length).toBe(EMBEDDING_DIMENSION);
+  });
+});

package/src/embeddings.ts

@@ -1,24 +1,47 @@
-const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
+export const MODEL_NAME = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
+export const EMBEDDING_DIMENSION = 384;
 const MODEL_DTYPE = "q8";
 
+export function getEmbeddingDimension(): number {
+  return EMBEDDING_DIMENSION;
+}
+
+export function embeddingModelName(): string {
+  return MODEL_NAME;
+}
+
 let pipelinePromise: Promise<any> | undefined;
 
-async function getPipeline() {
+async function getPipeline(cacheDir?: string) {
   if (!pipelinePromise) {
     pipelinePromise = (async () => {
       const { pipeline } = await import("@huggingface/transformers");
       return pipeline("feature-extraction", MODEL_NAME, {
         dtype: MODEL_DTYPE,
+        cache_dir: cacheDir,
       });
     })();
   }
   return pipelinePromise;
 }
 
-export async function generateEmbedding(text: string): Promise<number[]> {
-  const pipe = await getPipeline();
+export async function warmupEmbedder(cacheDir?: string): Promise<void> {
+  await getPipeline(cacheDir);
+}
+
+export async function generateEmbedding(
+  text: string,
+  cacheDir?: string,
+): Promise<number[]> {
+  const pipe = await getPipeline(cacheDir);
   const output = await pipe(text, { pooling: "mean", normalize: true });
-  return Array.from(output.data as Float32Array) as number[];
+  const out = Array.from(output.data as Float32Array) as number[];
+  if (out.length !== EMBEDDING_DIMENSION) {
+    throw new Error(
+      `Embedding dimension mismatch: expected ${EMBEDDING_DIMENSION}, got ${out.length}`,
+    );
+  }
+  return out;
 }
 
 export function cosineSimilarity(a: number[], b: number[]): number {

package/src/frontmatter.ts

@@ -1,7 +1,7 @@
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 
-import yaml from "js-yaml";
+import * as yaml from "js-yaml";
 
 export function splitFrontmatter(text: string): {
   frontmatterText: string | undefined;

package/src/journal.test.ts

@@ -14,7 +14,13 @@ mock.module("./embeddings", () => ({
     );
     return Array.from({ length: 8 }, (_, i) => Math.sin(hash + i));
   },
+  EMBEDDING_DIMENSION: 8,
+  embeddingModelName: () => "test-model",
   cosineSimilarity: (a: number[], b: number[]) => {
+    if (a.length !== b.length)
+      throw new Error(
+        `Embedding dimension mismatch: ${a.length} vs ${b.length}`,
+      );
     let dot = 0;
     let normA = 0;
     let normB = 0;
@@ -149,8 +155,12 @@ describe("journal store", () => {
     const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
     const raw = await fs.readFile(embeddingFile, "utf-8");
     const embedding = JSON.parse(raw);
-    expect(Array.isArray(embedding)).toBe(true);
-    expect(embedding.length).toBeGreaterThan(0);
+    // v0.4.0: versioned embedding object, not a bare array
+    expect(embedding.v).toBe(2);
+    expect(embedding.model).toBe("test-model");
+    expect(embedding.dimension).toBe(8);
+    expect(Array.isArray(embedding.vector)).toBe(true);
+    expect(embedding.vector.length).toBeGreaterThan(0);
   });
 
   test("read returns full entry by id", async () => {
@@ -381,4 +391,179 @@ describe("journal store", () => {
     // Should find at least the matching entry
     expect(result.total).toBeGreaterThan(0);
   });
+
+  test("search skips stale-dimension embedding and falls back to text match", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+
+    const entry = await store.write({
+      title: "Stale dims entry",
+      body: "unique-stale-dim-content",
+    });
+
+    // First search: caches the entry with the correct mock embedding.
+    await store.search({ text: "unique-stale-dim-content" });
+
+    // Simulate a v2 embedding with a WRONG dimension (e.g. from a 768d model).
+    // Search must not throw: it should skip the embedding and fall back to
+    // text matching for this entry.
+    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
+    await fs.writeFile(
+      embeddingFile,
+      JSON.stringify({
+        v: 2,
+        model: "some-768d-model",
+        dimension: 768,
+        vector: Array.from({ length: 768 }, () => 0.1),
+      }),
+      "utf-8",
+    );
+
+    // Query whose embedding would collide with the stale one if cosine were
+    // attempted (dimension mismatch would throw inside cosineSimilarity).
+    const result = await store.search({ text: "unique-stale-dim-content" });
+    const titles = result.entries.map((e) => e.title);
+    expect(titles).toContain("Stale dims entry");
+    // No throw above is the assertion: mismatched dimension degrades to text.
+  });
+
+  test("sidecar rewrite after cache warm is picked up by second search", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+
+    const entry = await store.write({
+      title: "Sidecar invalidation",
+      body: "unique-sidecar-invalidation-content",
+    });
+
+    // First search: caches the entry + correct sidecar fingerprint.
+    const r1 = await store.search({ text: "unique-sidecar-invalidation-content" });
+    expect(r1.entries.some((e) => e.title === "Sidecar invalidation")).toBe(true);
+
+    // Rewrite the sidecar with a wrong-dimension embedding (simulating
+    // sidecar regeneration without touching the .md).
+    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
+    await fs.writeFile(
+      embeddingFile,
+      JSON.stringify({
+        v: 2,
+        model: "some-768d-model",
+        dimension: 768,
+        vector: Array.from({ length: 768 }, () => 0.1),
+      }),
+      "utf-8",
+    );
+
+    // Second search must invalidate the cache (sidecar mtime changed) and
+    // pick up the stale-dimension embedding, falling back to text match.
+    const r2 = await store.search({ text: "unique-sidecar-invalidation-content" });
+    expect(r2.entries.some((e) => e.title === "Sidecar invalidation")).toBe(true);
+  });
+
+  test("sidecar deleted between searches falls back to text match", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+
+    const entry = await store.write({
+      title: "Sidecar deleted entry",
+      body: "unique-sidecar-deleted-content",
+    });
+
+    // First search: caches the entry + sidecar fingerprint.
+    const r1 = await store.search({ text: "unique-sidecar-deleted-content" });
+    expect(r1.entries.some((e) => e.title === "Sidecar deleted entry")).toBe(true);
+
+    // Delete the sidecar file.
+    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
+    await fs.unlink(embeddingFile);
+
+    // Second search: cache miss (sidecar gone), falls back to text match
+    // (score 0.5) rather than using the stale cached embedding.
+    const r2 = await store.search({ text: "unique-sidecar-deleted-content" });
+    const found = r2.entries.find((e) => e.title === "Sidecar deleted entry");
+    expect(found).toBeDefined();
+    // Score is 0.5 (text fallback), not a cosine score
+  });
+
+  test("search skips corrupt v2 embedding (empty vector) without throwing", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+
+    const entry = await store.write({
+      title: "Corrupt embedding entry",
+      body: "unique-corrupt-embedding-content",
+    });
+
+    // Simulate a corrupt-but-well-formed v2 embedding: dimension claims 384
+    // but the vector is empty. Search must NOT throw — it should skip the
+    // embedding and fall back to text matching for this entry.
+    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
+    await fs.writeFile(
+      embeddingFile,
+      JSON.stringify({ v: 2, model: "some-model", dimension: 8, vector: [] }),
+      "utf-8",
+    );
+
+    const result = await store.search({ text: "unique-corrupt-embedding-content" });
+    const titles = result.entries.map((e) => e.title);
+    expect(titles).toContain("Corrupt embedding entry");
+    // No throw above is the assertion: corrupt v2 embedding degrades to text.
+  });
+
+  test("search skips corrupt v2 embedding (missing vector) without throwing", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+
+    const entry = await store.write({
+      title: "Missing vector entry",
+      body: "unique-missing-vector-content",
+    });
+
+    // v2 file that omits the vector field entirely. Search must not throw.
+    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
+    await fs.writeFile(
+      embeddingFile,
+      JSON.stringify({ v: 2, model: "some-model", dimension: 8 }),
+      "utf-8",
+    );
+
+    const result = await store.search({ text: "unique-missing-vector-content" });
+    const titles = result.entries.map((e) => e.title);
+    expect(titles).toContain("Missing vector entry");
+  });
+
+  test("legacy bare-array embedding is still usable for semantic search", async () => {
+    tmpDir = await mkTmpDir();
+    const store = createJournalStore(tmpDir);
+
+    const entry = await store.write({
+      title: "Legacy embedding",
+      body: "legacy-vector-body",
+    });
+
+    // Overwrite with a legacy v1 bare-array embedding (from all-MiniLM-L6-v2,
+    // which also produces 8-dim vectors under the test mock). Use the same
+    // mock formula as generateEmbedding so the cosine is positive and the
+    // entry is actually matched via semantic search.
+    const hash = Array.from("legacy-vector-body").reduce(
+      (acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0,
+      0,
+    );
+    const legacyVector = Array.from(
+      { length: 8 },
+      (_, i) => Math.sin(hash + i),
+    );
+    const embeddingFile = entry.filePath.replace(/\.md$/, ".embedding");
+    await fs.writeFile(
+      embeddingFile,
+      JSON.stringify(legacyVector),
+      "utf-8",
+    );
+
+    // Legacy array must be treated as { vector, model: "unknown-legacy",
+    // dimension: length }; with a matching dimension it stays usable.
+    const result = await store.search({ text: "legacy-vector-body" });
+    const titles = result.entries.map((e) => e.title);
+    expect(titles).toContain("Legacy embedding");
+  });
 });

package/src/journal.ts

@@ -2,10 +2,10 @@ import * as fs from "node:fs/promises";
 import * as os from "node:os";
 import * as path from "node:path";
 
-import yaml from "js-yaml";
+import * as yaml from "js-yaml";
 import { z } from "zod";
 
-import { cosineSimilarity, generateEmbedding } from "./embeddings";
+import { cosineSimilarity, EMBEDDING_DIMENSION, embeddingModelName, generateEmbedding } from "./embeddings";
 import { atomicWriteFile, buildFrontmatterDocument, splitFrontmatter } from "./frontmatter";
 
 const TagSchema = z.looseObject({
@@ -14,6 +14,7 @@ const TagSchema = z.looseObject({
 });
 
 const ConfigSchema = z.looseObject({
+  cacheDir: z.string().optional(),
   journal: z
     .looseObject({
       enabled: z.boolean().optional(),
@@ -117,11 +118,50 @@ async function readEntryFile(filePath: string): Promise<JournalEntry> {
   };
 }
 
-async function loadEmbedding(entryPath: string): Promise<number[] | undefined> {
+type ParsedEmbedding =
+  | { vector: number[]; model: string; dimension: number }
+  | undefined;
+
+type CachedEntry = {
+  entryMtimeMs: number;
+  entrySize: number;
+  embMtimeMs: number;
+  embSize: number;
+  entry: JournalEntry;
+  embedding: ParsedEmbedding;
+};
+
+async function loadEmbedding(entryPath: string): Promise<ParsedEmbedding> {
   const ePath = embeddingPath(entryPath);
   try {
     const raw = await fs.readFile(ePath, "utf-8");
-    return JSON.parse(raw) as number[];
+    const parsed = JSON.parse(raw);
+    if (Array.isArray(parsed)) {
+      // Legacy v1 (bare array from all-MiniLM-L6-v2)
+      return {
+        vector: parsed as number[],
+        model: "unknown-legacy",
+        dimension: parsed.length,
+      };
+    }
+    if (parsed && typeof parsed === "object" && parsed.v === 2) {
+      const vector = Array.isArray(parsed.vector)
+        ? parsed.vector
+        : undefined;
+      if (
+        vector &&
+        vector.length === (parsed.dimension as number) &&
+        vector.length > 0
+      ) {
+        return {
+          vector,
+          model: parsed.model as string,
+          dimension: parsed.dimension as number,
+        };
+      }
+      return undefined;
+    }
+    return undefined;
   } catch {
     return undefined;
   }
@@ -160,12 +200,17 @@ export type JournalStore = {
   }): Promise<{ entries: JournalEntry[]; total: number; allTags: string[] }>;
 };
 
-export function createJournalStore(configDir?: string): JournalStore {
+export function createJournalStore(configDir?: string, cacheDir?: string): JournalStore {
   const journalDir = path.join(
     configDir ?? path.join(os.homedir(), ".config", "opencode"),
     "journal",
   );
 
+  // Per-store cache: each store instance owns its entries, so stores with
+  // different (including nested or ephemeral) journal directories never
+  // interfere and no foreign keys can accumulate.
+  const entryCache = new Map<string, CachedEntry>();
+
   return {
     async write(entry) {
       await fs.mkdir(journalDir, { recursive: true });
@@ -192,10 +237,15 @@ export function createJournalStore(configDir?: string): JournalStore {
       // Generate and save embedding for semantic search
       const searchableText = `${entry.title}\n${entry.body}`;
       try {
-        const embedding = await generateEmbedding(searchableText);
+        const embedding = await generateEmbedding(searchableText, cacheDir);
         await fs.writeFile(
           embeddingPath(filePath),
-          JSON.stringify(embedding),
+          JSON.stringify({
+            v: 2, // schema version for the embedding file
+            model: embeddingModelName(), // the embedding model that produced it
+            dimension: EMBEDDING_DIMENSION,
+            vector: embedding, // the actual float array
+          }),
           "utf-8",
         );
       } catch {
@@ -252,11 +302,17 @@ export function createJournalStore(configDir?: string): JournalStore {
         return { entries: [], total: 0, allTags: [] };
       }
 
+      // Prune cache entries for files that no longer exist
+      const alive = new Set(files.map((f) => path.join(journalDir, f)));
+      for (const key of entryCache.keys()) {
+        if (!alive.has(key)) entryCache.delete(key);
+      }
+
       // If a text query is provided, try semantic search first
       let queryEmbedding: number[] | undefined;
       if (query.text) {
         try {
-          queryEmbedding = await generateEmbedding(query.text);
+          queryEmbedding = await generateEmbedding(query.text, cacheDir);
         } catch {
           // Fall back to text search if embedding fails
         }
@@ -265,14 +321,46 @@ export function createJournalStore(configDir?: string): JournalStore {
       // Collect all tags across every entry (before filtering)
       const tagSet = new Set<string>();
 
-      for (const file of files) {
+      const indexed = await Promise.all(files.map(async (file) => {
         const filePath = path.join(journalDir, file);
-        let entry: JournalEntry;
         try {
-          entry = await readEntryFile(filePath);
+          const ePath = embeddingPath(filePath);
+          const [entryStat, embStat] = await Promise.all([
+            fs.stat(filePath),
+            fs.stat(ePath).catch(() => ({ mtimeMs: -1, size: -1 } as const)),
+          ]);
+          const cached = entryCache.get(filePath);
+          if (
+            cached &&
+            cached.entryMtimeMs === entryStat.mtimeMs &&
+            cached.entrySize === entryStat.size &&
+            cached.embMtimeMs === embStat.mtimeMs &&
+            cached.embSize === embStat.size
+          ) {
+            return cached;
+          }
+          const [entry, embedding] = await Promise.all([
+            readEntryFile(filePath),
+            loadEmbedding(filePath),
+          ]);
+          const value: CachedEntry = {
+            entryMtimeMs: entryStat.mtimeMs,
+            entrySize: entryStat.size,
+            embMtimeMs: embStat.mtimeMs,
+            embSize: embStat.size,
+            entry,
+            embedding,
+          };
+          entryCache.set(filePath, value);
+          return value;
         } catch {
-          continue;
+          return undefined;
         }
+      }));
+
+      for (const cached of indexed) {
+        if (!cached) continue;
+        const { entry } = cached;
 
         // Collect tags before applying filters
         for (const tag of entry.tags) {
@@ -297,9 +385,21 @@ export function createJournalStore(configDir?: string): JournalStore {
         if (query.text) {
           if (queryEmbedding) {
             // Semantic search
-            const entryEmbedding = await loadEmbedding(filePath);
-            if (entryEmbedding) {
-              score = cosineSimilarity(queryEmbedding, entryEmbedding);
+            const storedEmbedding = cached.embedding;
+            if (storedEmbedding) {
+              // Hard gate: dimension must match, or the embedding is stale
+              // (different model) and cannot be compared with cosine.
+              // A mismatched/stale embedding degrades gracefully to text match.
+              if (storedEmbedding.dimension !== EMBEDDING_DIMENSION) {
+                const haystack =
+                  `${entry.title}\n${entry.body}`.toLowerCase();
+                score = haystack.includes(query.text.toLowerCase()) ? 0.5 : 0;
+              } else {
+                score = cosineSimilarity(
+                  queryEmbedding,
+                  storedEmbedding.vector,
+                );
+              }
             } else {
               // No embedding stored; fall back to text match
               const haystack =

package/src/memory.ts

@@ -2,7 +2,7 @@ import * as fs from "node:fs/promises";
 import * as os from "node:os";
 import * as path from "node:path";
 
-import yaml from "js-yaml";
+import * as yaml from "js-yaml";
 import { z } from "zod";
 
 import { atomicWriteFile, buildFrontmatterDocument, splitFrontmatter } from "./frontmatter";

package/src/plugin.ts

@@ -16,6 +16,7 @@ import {
   MemorySet,
 } from "./tools";
 import type { JournalContext } from "./tools";
+import { warmupEmbedder } from "./embeddings";
 
 export const MemoryPlugin: Plugin = async ({ directory }) => {
   const store = createMemoryStore(directory);
@@ -36,7 +37,10 @@ export const MemoryPlugin: Plugin = async ({ directory }) => {
   let journalSystemNote = "";
 
   if (journalEnabled) {
-    const journalStore = createJournalStore();
+    const journalStore = createJournalStore(undefined, config.cacheDir);
+    // Warmup the embedder in the background; pass configured cacheDir so the
+    // first-init-wins singleton caches to the correct directory.
+    void warmupEmbedder(config.cacheDir).catch(() => {});
     journalTools = {
       journal_write: JournalWrite(journalStore, journalCtx),
       journal_read: JournalRead(journalStore),