claude-memory-layer 1.0.19 → 1.0.21

package/dist/services/memory-service.js

@@ -1598,6 +1598,33 @@ var SQLiteEventStore = class {
       ids
     );
   }
+  /**
+   * Clear embedding outbox (used for embedding model migration)
+   */
+  async clearEmbeddingOutbox() {
+    await this.initialize();
+    sqliteRun(this.db, `DELETE FROM embedding_outbox`);
+  }
+  /**
+   * Count total events
+   */
+  async countEvents() {
+    await this.initialize();
+    const row = sqliteGet(this.db, `SELECT COUNT(*) as count FROM events`);
+    return row?.count || 0;
+  }
+  /**
+   * Get events page in timestamp ascending order (stable migration/reindex scans)
+   */
+  async getEventsPage(limit = 1e3, offset = 0) {
+    await this.initialize();
+    const rows = sqliteAll(
+      this.db,
+      `SELECT * FROM events ORDER BY timestamp ASC LIMIT ? OFFSET ?`,
+      [limit, offset]
+    );
+    return rows.map(this.rowToEvent);
+  }
   /**
    * Mark outbox items as failed
    */
@@ -2563,6 +2590,23 @@ var VectorStore = class {
     const result = await this.table.countRows();
     return result;
   }
+  /**
+   * Clear all vectors (used for embedding model migration)
+   */
+  async clearAll() {
+    await this.initialize();
+    if (!this.db)
+      return;
+    try {
+      if (typeof this.db.dropTable === "function") {
+        await this.db.dropTable(this.tableName);
+      } else if (typeof this.db.drop_table === "function") {
+        await this.db.drop_table(this.tableName);
+      }
+    } catch {
+    }
+    this.table = null;
+  }
   /**
    * Check if vector exists for event
    */
@@ -2575,13 +2619,15 @@ var VectorStore = class {
 };
 
 // src/core/embedder.ts
-import { pipeline } from "@xenova/transformers";
+import { pipeline } from "@huggingface/transformers";
 var Embedder = class {
   pipeline = null;
   modelName;
+  activeModelName;
   initialized = false;
-  constructor(modelName = "Xenova/all-MiniLM-L6-v2") {
+  constructor(modelName = "jinaai/jina-embeddings-v5-text-nano-text-matching") {
     this.modelName = modelName;
+    this.activeModelName = modelName;
   }
   /**
    * Initialize the embedding pipeline
@@ -2589,8 +2635,21 @@ var Embedder = class {
   async initialize() {
     if (this.initialized)
       return;
-    this.pipeline = await pipeline("feature-extraction", this.modelName);
-    this.initialized = true;
+    try {
+      this.pipeline = await pipeline("feature-extraction", this.modelName);
+      this.activeModelName = this.modelName;
+      this.initialized = true;
+      return;
+    } catch (primaryError) {
+      const fallbackModel = process.env.CLAUDE_MEMORY_EMBEDDING_FALLBACK_MODEL || "onnx-community/embeddinggemma-300m-ONNX";
+      if (fallbackModel === this.modelName) {
+        throw primaryError;
+      }
+      console.warn(`[Embedder] Primary model failed (${this.modelName}). Falling back to ${fallbackModel}`);
+      this.pipeline = await pipeline("feature-extraction", fallbackModel);
+      this.activeModelName = fallbackModel;
+      this.initialized = true;
+    }
   }
   /**
    * Generate embedding for a single text
@@ -2607,7 +2666,7 @@ var Embedder = class {
     const vector = Array.from(output.data);
     return {
       vector,
-      model: this.modelName,
+      model: this.activeModelName,
       dimensions: vector.length
     };
   }
@@ -2631,7 +2690,7 @@ var Embedder = class {
         const vector = Array.from(output.data);
         results.push({
           vector,
-          model: this.modelName,
+          model: this.activeModelName,
           dimensions: vector.length
         });
       }
@@ -2655,13 +2714,14 @@ var Embedder = class {
    * Get model name
    */
   getModelName() {
-    return this.modelName;
+    return this.activeModelName;
   }
 };
 var defaultEmbedder = null;
 function getDefaultEmbedder() {
+  const envModel = process.env.CLAUDE_MEMORY_EMBEDDING_MODEL;
   if (!defaultEmbedder) {
-    defaultEmbedder = new Embedder();
+    defaultEmbedder = new Embedder(envModel || void 0);
   }
   return defaultEmbedder;
 }
@@ -5921,8 +5981,10 @@ var MemoryService = class {
   readOnly;
   lightweightMode;
   mdMirror;
+  storagePath;
   constructor(config) {
     const storagePath = this.expandPath(config.storagePath);
+    this.storagePath = storagePath;
     this.readOnly = config.readOnly ?? false;
     this.lightweightMode = config.lightweightMode ?? false;
     this.mdMirror = new MarkdownMirror2(process.cwd());
@@ -5958,7 +6020,8 @@ var MemoryService = class {
       );
     }
     this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
-    this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
+    const embeddingModel = config.embeddingModel || process.env.CLAUDE_MEMORY_EMBEDDING_MODEL;
+    this.embedder = embeddingModel ? new Embedder(embeddingModel) : getDefaultEmbedder();
     this.matcher = getDefaultMatcher();
     this.retriever = createRetriever(
       this.sqliteStore,
@@ -6902,6 +6965,89 @@ var MemoryService = class {
   recordMemoryAccess(eventId, sessionId, confidence = 1) {
     this.graduation.recordAccess(eventId, sessionId, confidence);
   }
+  getEmbeddingModelName() {
+    return this.embedder.getModelName();
+  }
+  /**
+   * Ensure embedding model metadata is in sync and optionally migrate vectors.
+   * Migration strategy: clear vector index + clear embedding outbox + re-enqueue all events.
+   */
+  async ensureEmbeddingModelForImport(options) {
+    await this.initialize();
+    const currentModel = this.getEmbeddingModelName();
+    const metaPath = path3.join(this.storagePath, "embedding-meta.json");
+    let previousModel = null;
+    try {
+      if (fs4.existsSync(metaPath)) {
+        const parsed = JSON.parse(fs4.readFileSync(metaPath, "utf-8"));
+        previousModel = parsed?.model || null;
+      }
+    } catch {
+      previousModel = null;
+    }
+    const stats = await this.getStats();
+    const hasExistingVectors = (stats.vectorCount || 0) > 0;
+    if (!previousModel && !hasExistingVectors) {
+      fs4.writeFileSync(metaPath, JSON.stringify({ model: currentModel, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2));
+      return { changed: false, previousModel: null, currentModel, enqueued: 0, reason: "initialized-meta" };
+    }
+    const modelChanged = previousModel !== currentModel;
+    const legacyUnknownButVectorsExist = !previousModel && hasExistingVectors;
+    if (!modelChanged && !legacyUnknownButVectorsExist) {
+      return { changed: false, previousModel, currentModel, enqueued: 0 };
+    }
+    if (options?.autoMigrate === false) {
+      return {
+        changed: true,
+        previousModel,
+        currentModel,
+        enqueued: 0,
+        reason: legacyUnknownButVectorsExist ? "legacy-vectors-without-meta" : "model-mismatch"
+      };
+    }
+    const wasRunning = this.vectorWorker?.isRunning() || false;
+    if (wasRunning)
+      this.vectorWorker?.stop();
+    await this.vectorStore.clearAll();
+    await this.sqliteStore.clearEmbeddingOutbox();
+    const pageSize = 1e3;
+    let offset = 0;
+    let enqueued = 0;
+    while (true) {
+      const page = await this.sqliteStore.getEventsPage(pageSize, offset);
+      if (page.length === 0)
+        break;
+      for (const event of page) {
+        await this.sqliteStore.enqueueForEmbedding(event.id, event.content);
+        enqueued += 1;
+      }
+      offset += page.length;
+      if (page.length < pageSize)
+        break;
+    }
+    fs4.writeFileSync(
+      metaPath,
+      JSON.stringify(
+        {
+          model: currentModel,
+          previousModel,
+          migratedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          enqueued
+        },
+        null,
+        2
+      )
+    );
+    if (wasRunning)
+      this.vectorWorker?.start();
+    return {
+      changed: true,
+      previousModel,
+      currentModel,
+      enqueued,
+      reason: legacyUnknownButVectorsExist ? "legacy-vectors-without-meta" : "model-mismatch"
+    };
+  }
   /**
    * Backward-compatible alias used by some hooks
    */