@icex-labs/openclaw-memory-engine 3.4.0 → 3.5.0

package/index.js

@@ -20,7 +20,7 @@ import { existsSync } from "node:fs";
 import { resolveWorkspace, getCoreSizeLimit, DEFAULT_TOP_K, MAX_TOP_K } from "./lib/paths.js";
 import { readCore, writeCore, dotGet, dotSet, autoParse } from "./lib/core.js";
 import { loadArchival, appendRecord, rewriteArchival, archivalPath } from "./lib/archival.js";
-import { indexEmbedding, loadEmbeddingCache, saveEmbeddingCache } from "./lib/embedding.js";
+import { indexEmbedding, loadEmbeddingCache, saveEmbeddingCache, backfillEmbeddings } from "./lib/embedding.js";
 import { hybridSearch } from "./lib/search.js";
 import { consolidateText } from "./lib/consolidate.js";
 import { findDuplicates, applyDedup } from "./lib/dedup.js";
@@ -95,6 +95,26 @@ export default definePluginEntry({
     // Factory ctx has: { sessionKey, workspaceDir, agentId, ... }
     // ═══════════════════════════════════════════════════════════════════
 
+    // Background: auto-backfill missing embeddings on startup
+    const defaultWs = resolveWorkspace(null);
+    setTimeout(() => {
+      try {
+        const records = loadArchival(defaultWs);
+        const cache = loadEmbeddingCache(defaultWs);
+        const missing = records.filter((r) => r.id && !cache[r.id]).length;
+        if (missing > 0) {
+          console.error(`[memory-engine] Backfilling ${missing} missing embeddings...`);
+          backfillEmbeddings(defaultWs, records, {
+            onProgress: (done, total) => {
+              if (done % 500 === 0) console.error(`[memory-engine] Embedding backfill: ${done}/${total}`);
+            },
+          }).then((result) => {
+            console.error(`[memory-engine] Backfill complete: ${result.processed} embedded, ${result.errors} errors`);
+          }).catch(() => {});
+        }
+      } catch { /* ignore startup errors */ }
+    }, 10000); // delay 10s after gateway start to avoid blocking
+
     // ─── core_memory_read ───
     api.registerTool(withAgent((agentId) => ({
       name: "core_memory_read",

package/lib/embedding.js

@@ -68,3 +68,67 @@ export async function indexEmbedding(ws, record) {
     saveEmbeddingCache(ws);
   }
 }
+
+/**
+ * Batch-embed records that are missing from cache.
+ * Runs in background with batching (100 per API call) and rate limiting.
+ * @param {string} ws - workspace path
+ * @param {object[]} records - all archival records
+ * @param {object} [options]
+ * @param {number} [options.batchSize=100]
+ * @param {number} [options.delayMs=200]
+ * @param {function} [options.onProgress] - callback(done, total)
+ */
+export async function backfillEmbeddings(ws, records, options = {}) {
+  const apiKey = resolveApiKey();
+  if (!apiKey) return { processed: 0, errors: 0, skipped: 0 };
+
+  const batchSize = options.batchSize || 100;
+  const delayMs = options.delayMs || 200;
+  const cache = loadEmbeddingCache(ws);
+
+  const missing = records.filter((r) => r.id && !cache[r.id]);
+  if (missing.length === 0) return { processed: 0, errors: 0, skipped: 0 };
+
+  let processed = 0;
+  let errors = 0;
+
+  for (let i = 0; i < missing.length; i += batchSize) {
+    const batch = missing.slice(i, i + batchSize);
+    const texts = batch.map((r) =>
+      [r.content, r.entity, ...(r.tags || [])].filter(Boolean).join(" "),
+    );
+
+    try {
+      const res = await fetch("https://api.openai.com/v1/embeddings", {
+        method: "POST",
+        headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+        body: JSON.stringify({ input: texts, model: EMBEDDING_MODEL, dimensions: EMBEDDING_DIM }),
+      });
+
+      if (!res.ok) { errors++; continue; }
+
+      const data = await res.json();
+      for (let j = 0; j < batch.length; j++) {
+        if (data.data?.[j]?.embedding) {
+          cache[batch[j].id] = data.data[j].embedding;
+          processed++;
+        }
+      }
+
+      // Save after each batch
+      saveEmbeddingCache(ws);
+
+      if (options.onProgress) options.onProgress(processed, missing.length);
+
+      // Rate limit
+      if (i + batchSize < missing.length) {
+        await new Promise((r) => setTimeout(r, delayMs));
+      }
+    } catch {
+      errors++;
+    }
+  }
+
+  return { processed, errors, skipped: missing.length - processed - errors };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icex-labs/openclaw-memory-engine",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "description": "MemGPT-style hierarchical memory plugin for OpenClaw — core memory block + archival storage with semantic search",
   "type": "module",
   "main": "index.js",