claude-memory-layer 1.0.19 → 1.0.21

package/dist/cli/index.js

@@ -1613,6 +1613,33 @@ var SQLiteEventStore = class {
       ids
     );
   }
+  /**
+   * Clear embedding outbox (used for embedding model migration)
+   */
+  async clearEmbeddingOutbox() {
+    await this.initialize();
+    sqliteRun(this.db, `DELETE FROM embedding_outbox`);
+  }
+  /**
+   * Count total events
+   */
+  async countEvents() {
+    await this.initialize();
+    const row = sqliteGet(this.db, `SELECT COUNT(*) as count FROM events`);
+    return row?.count || 0;
+  }
+  /**
+   * Get events page in timestamp ascending order (stable migration/reindex scans)
+   */
+  async getEventsPage(limit = 1e3, offset = 0) {
+    await this.initialize();
+    const rows = sqliteAll(
+      this.db,
+      `SELECT * FROM events ORDER BY timestamp ASC LIMIT ? OFFSET ?`,
+      [limit, offset]
+    );
+    return rows.map(this.rowToEvent);
+  }
   /**
    * Mark outbox items as failed
    */
@@ -2578,6 +2605,23 @@ var VectorStore = class {
     const result = await this.table.countRows();
     return result;
   }
+  /**
+   * Clear all vectors (used for embedding model migration)
+   */
+  async clearAll() {
+    await this.initialize();
+    if (!this.db)
+      return;
+    try {
+      if (typeof this.db.dropTable === "function") {
+        await this.db.dropTable(this.tableName);
+      } else if (typeof this.db.drop_table === "function") {
+        await this.db.drop_table(this.tableName);
+      }
+    } catch {
+    }
+    this.table = null;
+  }
   /**
    * Check if vector exists for event
    */
@@ -2590,13 +2634,15 @@ var VectorStore = class {
 };
 
 // src/core/embedder.ts
-import { pipeline } from "@xenova/transformers";
+import { pipeline } from "@huggingface/transformers";
 var Embedder = class {
   pipeline = null;
   modelName;
+  activeModelName;
   initialized = false;
-  constructor(modelName = "Xenova/all-MiniLM-L6-v2") {
+  constructor(modelName = "jinaai/jina-embeddings-v5-text-nano-text-matching") {
     this.modelName = modelName;
+    this.activeModelName = modelName;
   }
   /**
    * Initialize the embedding pipeline
@@ -2604,8 +2650,21 @@ var Embedder = class {
   async initialize() {
     if (this.initialized)
       return;
-    this.pipeline = await pipeline("feature-extraction", this.modelName);
-    this.initialized = true;
+    try {
+      this.pipeline = await pipeline("feature-extraction", this.modelName);
+      this.activeModelName = this.modelName;
+      this.initialized = true;
+      return;
+    } catch (primaryError) {
+      const fallbackModel = process.env.CLAUDE_MEMORY_EMBEDDING_FALLBACK_MODEL || "onnx-community/embeddinggemma-300m-ONNX";
+      if (fallbackModel === this.modelName) {
+        throw primaryError;
+      }
+      console.warn(`[Embedder] Primary model failed (${this.modelName}). Falling back to ${fallbackModel}`);
+      this.pipeline = await pipeline("feature-extraction", fallbackModel);
+      this.activeModelName = fallbackModel;
+      this.initialized = true;
+    }
   }
   /**
    * Generate embedding for a single text
@@ -2622,7 +2681,7 @@ var Embedder = class {
     const vector = Array.from(output.data);
     return {
       vector,
-      model: this.modelName,
+      model: this.activeModelName,
       dimensions: vector.length
     };
   }
@@ -2646,7 +2705,7 @@ var Embedder = class {
         const vector = Array.from(output.data);
         results.push({
           vector,
-          model: this.modelName,
+          model: this.activeModelName,
           dimensions: vector.length
         });
       }
@@ -2670,13 +2729,14 @@ var Embedder = class {
    * Get model name
    */
   getModelName() {
-    return this.modelName;
+    return this.activeModelName;
   }
 };
 var defaultEmbedder = null;
 function getDefaultEmbedder() {
+  const envModel = process.env.CLAUDE_MEMORY_EMBEDDING_MODEL;
   if (!defaultEmbedder) {
-    defaultEmbedder = new Embedder();
+    defaultEmbedder = new Embedder(envModel || void 0);
   }
   return defaultEmbedder;
 }
@@ -5932,8 +5992,10 @@ var MemoryService = class {
   readOnly;
   lightweightMode;
   mdMirror;
+  storagePath;
   constructor(config) {
     const storagePath = this.expandPath(config.storagePath);
+    this.storagePath = storagePath;
     this.readOnly = config.readOnly ?? false;
     this.lightweightMode = config.lightweightMode ?? false;
     this.mdMirror = new MarkdownMirror2(process.cwd());
@@ -5969,7 +6031,8 @@ var MemoryService = class {
       );
     }
     this.vectorStore = new VectorStore(path3.join(storagePath, "vectors"));
-    this.embedder = config.embeddingModel ? new Embedder(config.embeddingModel) : getDefaultEmbedder();
+    const embeddingModel = config.embeddingModel || process.env.CLAUDE_MEMORY_EMBEDDING_MODEL;
+    this.embedder = embeddingModel ? new Embedder(embeddingModel) : getDefaultEmbedder();
     this.matcher = getDefaultMatcher();
     this.retriever = createRetriever(
       this.sqliteStore,
@@ -6913,6 +6976,89 @@ var MemoryService = class {
   recordMemoryAccess(eventId, sessionId, confidence = 1) {
     this.graduation.recordAccess(eventId, sessionId, confidence);
   }
+  getEmbeddingModelName() {
+    return this.embedder.getModelName();
+  }
+  /**
+   * Ensure embedding model metadata is in sync and optionally migrate vectors.
+   * Migration strategy: clear vector index + clear embedding outbox + re-enqueue all events.
+   */
+  async ensureEmbeddingModelForImport(options) {
+    await this.initialize();
+    const currentModel = this.getEmbeddingModelName();
+    const metaPath = path3.join(this.storagePath, "embedding-meta.json");
+    let previousModel = null;
+    try {
+      if (fs4.existsSync(metaPath)) {
+        const parsed = JSON.parse(fs4.readFileSync(metaPath, "utf-8"));
+        previousModel = parsed?.model || null;
+      }
+    } catch {
+      previousModel = null;
+    }
+    const stats = await this.getStats();
+    const hasExistingVectors = (stats.vectorCount || 0) > 0;
+    if (!previousModel && !hasExistingVectors) {
+      fs4.writeFileSync(metaPath, JSON.stringify({ model: currentModel, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2));
+      return { changed: false, previousModel: null, currentModel, enqueued: 0, reason: "initialized-meta" };
+    }
+    const modelChanged = previousModel !== currentModel;
+    const legacyUnknownButVectorsExist = !previousModel && hasExistingVectors;
+    if (!modelChanged && !legacyUnknownButVectorsExist) {
+      return { changed: false, previousModel, currentModel, enqueued: 0 };
+    }
+    if (options?.autoMigrate === false) {
+      return {
+        changed: true,
+        previousModel,
+        currentModel,
+        enqueued: 0,
+        reason: legacyUnknownButVectorsExist ? "legacy-vectors-without-meta" : "model-mismatch"
+      };
+    }
+    const wasRunning = this.vectorWorker?.isRunning() || false;
+    if (wasRunning)
+      this.vectorWorker?.stop();
+    await this.vectorStore.clearAll();
+    await this.sqliteStore.clearEmbeddingOutbox();
+    const pageSize = 1e3;
+    let offset = 0;
+    let enqueued = 0;
+    while (true) {
+      const page = await this.sqliteStore.getEventsPage(pageSize, offset);
+      if (page.length === 0)
+        break;
+      for (const event of page) {
+        await this.sqliteStore.enqueueForEmbedding(event.id, event.content);
+        enqueued += 1;
+      }
+      offset += page.length;
+      if (page.length < pageSize)
+        break;
+    }
+    fs4.writeFileSync(
+      metaPath,
+      JSON.stringify(
+        {
+          model: currentModel,
+          previousModel,
+          migratedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          enqueued
+        },
+        null,
+        2
+      )
+    );
+    if (wasRunning)
+      this.vectorWorker?.start();
+    return {
+      changed: true,
+      previousModel,
+      currentModel,
+      enqueued,
+      reason: legacyUnknownButVectorsExist ? "legacy-vectors-without-meta" : "model-mismatch"
+    };
+  }
   /**
    * Backward-compatible alias used by some hooks
    */
@@ -7899,6 +8045,7 @@ eventsRouter.get("/", async (c) => {
   const eventType = c.req.query("type");
   const level = c.req.query("level");
   const sort = c.req.query("sort") || "recent";
+  const q = (c.req.query("q") || "").trim().toLowerCase();
   const limit = parseInt(c.req.query("limit") || "100", 10);
   const offset = parseInt(c.req.query("offset") || "0", 10);
   const memoryService = getServiceFromQuery(c);
@@ -7916,6 +8063,9 @@ eventsRouter.get("/", async (c) => {
     if (eventType) {
       events = events.filter((e) => e.eventType === eventType);
     }
+    if (q) {
+      events = events.filter((e) => (e.content || "").toLowerCase().includes(q));
+    }
     if (sort === "accessed") {
       events.sort((a, b) => {
         const aTime = a.last_accessed_at || "";
@@ -9559,7 +9709,7 @@ function getHooksConfig(pluginPath) {
   };
 }
 var program = new Command();
-program.name("claude-memory-layer").description("Claude Code Memory Plugin CLI").version("1.0.19");
+program.name("claude-memory-layer").description("Claude Code Memory Plugin CLI").version("1.0.21");
 program.command("install").description("Install hooks into Claude Code settings").option("--path <path>", "Custom plugin path (defaults to auto-detect)").action(async (options) => {
   try {
     const pluginPath = options.path || getPluginPath();
@@ -10051,9 +10201,12 @@ program.command("organize-import [sourceDir]").description("Import existing mark
     process.exit(1);
   }
 });
-program.command("import").description("Import existing Claude Code conversation history").option("-p, --project <path>", "Import from specific project path").option("-s, --session <file>", "Import specific session file (JSONL)").option("-a, --all", "Import all sessions from all projects").option("-l, --limit <number>", "Limit messages per session").option("-f, --force", "Force reimport: delete existing events and reimport with turn_id grouping").option("-v, --verbose", "Show detailed progress").action(async (options) => {
+program.command("import").description("Import existing Claude Code conversation history").option("-p, --project <path>", "Import from specific project path").option("-s, --session <file>", "Import specific session file (JSONL)").option("-a, --all", "Import all sessions from all projects").option("-l, --limit <number>", "Limit messages per session").option("-f, --force", "Force reimport: delete existing events and reimport with turn_id grouping").option("--embedding-model <name>", "Embedding model override (default: jinaai/jina-embeddings-v5-text-nano-text-matching, or env CLAUDE_MEMORY_EMBEDDING_MODEL; fallback env: CLAUDE_MEMORY_EMBEDDING_FALLBACK_MODEL)").option("-v, --verbose", "Show detailed progress").action(async (options) => {
   const startTime = Date.now();
   const targetProjectPath = options.project || process.cwd();
+  if (options.embeddingModel) {
+    process.env.CLAUDE_MEMORY_EMBEDDING_MODEL = options.embeddingModel;
+  }
   const service = getMemoryServiceForProject(targetProjectPath);
   const importer = createSessionHistoryImporter(service);
   const importOpts = {
@@ -10065,7 +10218,16 @@ program.command("import").description("Import existing Claude Code conversation
   try {
     console.log("\n\u23F3 Initializing memory service...");
     await service.initialize();
-    console.log("  \u2705 Ready\n");
+    console.log(`  \u2705 Ready (embedder: ${service.getEmbeddingModelName()})
+`);
+    const migration = await service.ensureEmbeddingModelForImport({ autoMigrate: true });
+    if (migration.changed) {
+      console.log("\u{1F501} Embedding model migration detected/required");
+      console.log(`   Previous: ${migration.previousModel || "legacy-unknown"}`);
+      console.log(`   Current:  ${migration.currentModel}`);
+      console.log(`   Re-queued embeddings: ${migration.enqueued}`);
+      console.log("   (Import will continue and process embeddings with the new model)\n");
+    }
     if (options.force) {
       console.log("\u{1F504} Force mode: existing events will be deleted and reimported with turn_id grouping\n");
     }
@@ -10088,6 +10250,14 @@ program.command("import").description("Import existing Claude Code conversation
       const globalService = getDefaultMemoryService();
       const globalImporter = createSessionHistoryImporter(globalService);
       await globalService.initialize();
+      console.log(`  \u2705 Global service ready (embedder: ${globalService.getEmbeddingModelName()})`);
+      const globalMigration = await globalService.ensureEmbeddingModelForImport({ autoMigrate: true });
+      if (globalMigration.changed) {
+        console.log("\u{1F501} Global embedding migration detected");
+        console.log(`   Previous: ${globalMigration.previousModel || "legacy-unknown"}`);
+        console.log(`   Current:  ${globalMigration.currentModel}`);
+        console.log(`   Re-queued embeddings: ${globalMigration.enqueued}`);
+      }
       result = await globalImporter.importAll(importOpts);
       console.log("\n\u{1F9E0} Processing embeddings...");
       const embedCount2 = await globalService.processPendingEmbeddings();