@icex-labs/openclaw-memory-engine 3.4.0 → 3.5.1

package/index.js

@@ -16,11 +16,12 @@
 
 import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
 import { existsSync } from "node:fs";
+import { join } from "node:path";
 
 import { resolveWorkspace, getCoreSizeLimit, DEFAULT_TOP_K, MAX_TOP_K } from "./lib/paths.js";
 import { readCore, writeCore, dotGet, dotSet, autoParse } from "./lib/core.js";
 import { loadArchival, appendRecord, rewriteArchival, archivalPath } from "./lib/archival.js";
-import { indexEmbedding, loadEmbeddingCache, saveEmbeddingCache } from "./lib/embedding.js";
+import { indexEmbedding, loadEmbeddingCache, saveEmbeddingCache, backfillEmbeddings } from "./lib/embedding.js";
 import { hybridSearch } from "./lib/search.js";
 import { consolidateText } from "./lib/consolidate.js";
 import { findDuplicates, applyDedup } from "./lib/dedup.js";
@@ -95,6 +96,40 @@ export default definePluginEntry({
     // Factory ctx has: { sessionKey, workspaceDir, agentId, ... }
     // ═══════════════════════════════════════════════════════════════════
 
+    // Background: auto-backfill missing embeddings on startup (all workspaces)
+    setTimeout(async () => {
+      try {
+        // Discover all unique workspaces from openclaw.json agent configs
+        const workspaces = new Set();
+        workspaces.add(resolveWorkspace(null)); // default
+        try {
+          const cfgPath = join(process.env.HOME || "/tmp", ".openclaw", "openclaw.json");
+          const cfg = JSON.parse(readFileSync(cfgPath, "utf-8"));
+          for (const agent of cfg?.agents?.list || []) {
+            if (agent.workspace) workspaces.add(agent.workspace);
+          }
+        } catch { /* ignore */ }
+
+        for (const wsDir of workspaces) {
+          try {
+            const records = loadArchival(wsDir);
+            if (records.length === 0) continue;
+            const cache = loadEmbeddingCache(wsDir);
+            const missing = records.filter((r) => r.id && !cache[r.id]).length;
+            if (missing > 0) {
+              console.error(`[memory-engine] Backfilling ${missing} embeddings in ${wsDir}...`);
+              const result = await backfillEmbeddings(wsDir, records, {
+                onProgress: (done, total) => {
+                  if (done % 500 === 0) console.error(`[memory-engine] ${wsDir}: ${done}/${total}`);
+                },
+              });
+              console.error(`[memory-engine] ${wsDir}: done — ${result.processed} embedded, ${result.errors} errors`);
+            }
+          } catch { /* skip workspace errors */ }
+        }
+      } catch { /* ignore startup errors */ }
+    }, 10000); // delay 10s after gateway start
+
     // ─── core_memory_read ───
     api.registerTool(withAgent((agentId) => ({
       name: "core_memory_read",

package/lib/embedding.js

@@ -68,3 +68,67 @@ export async function indexEmbedding(ws, record) {
     saveEmbeddingCache(ws);
   }
 }
+
+/**
+ * Batch-embed records that are missing from cache.
+ * Runs in background with batching (100 per API call) and rate limiting.
+ * @param {string} ws - workspace path
+ * @param {object[]} records - all archival records
+ * @param {object} [options]
+ * @param {number} [options.batchSize=100]
+ * @param {number} [options.delayMs=200]
+ * @param {function} [options.onProgress] - callback(done, total)
+ */
+export async function backfillEmbeddings(ws, records, options = {}) {
+  const apiKey = resolveApiKey();
+  if (!apiKey) return { processed: 0, errors: 0, skipped: 0 };
+
+  const batchSize = options.batchSize || 100;
+  const delayMs = options.delayMs || 200;
+  const cache = loadEmbeddingCache(ws);
+
+  const missing = records.filter((r) => r.id && !cache[r.id]);
+  if (missing.length === 0) return { processed: 0, errors: 0, skipped: 0 };
+
+  let processed = 0;
+  let errors = 0;
+
+  for (let i = 0; i < missing.length; i += batchSize) {
+    const batch = missing.slice(i, i + batchSize);
+    const texts = batch.map((r) =>
+      [r.content, r.entity, ...(r.tags || [])].filter(Boolean).join(" "),
+    );
+
+    try {
+      const res = await fetch("https://api.openai.com/v1/embeddings", {
+        method: "POST",
+        headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+        body: JSON.stringify({ input: texts, model: EMBEDDING_MODEL, dimensions: EMBEDDING_DIM }),
+      });
+
+      if (!res.ok) { errors++; continue; }
+
+      const data = await res.json();
+      for (let j = 0; j < batch.length; j++) {
+        if (data.data?.[j]?.embedding) {
+          cache[batch[j].id] = data.data[j].embedding;
+          processed++;
+        }
+      }
+
+      // Save after each batch
+      saveEmbeddingCache(ws);
+
+      if (options.onProgress) options.onProgress(processed, missing.length);
+
+      // Rate limit
+      if (i + batchSize < missing.length) {
+        await new Promise((r) => setTimeout(r, delayMs));
+      }
+    } catch {
+      errors++;
+    }
+  }
+
+  return { processed, errors, skipped: missing.length - processed - errors };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icex-labs/openclaw-memory-engine",
-  "version": "3.4.0",
+  "version": "3.5.1",
   "description": "MemGPT-style hierarchical memory plugin for OpenClaw — core memory block + archival storage with semantic search",
   "type": "module",
   "main": "index.js",