@icex-labs/openclaw-memory-engine 3.3.2 → 3.5.0

package/extras/migrate-legacy.mjs

@@ -0,0 +1,165 @@
+#!/usr/bin/env node
+/**
+ * migrate-legacy.mjs — Import existing file-based memory into archival.jsonl
+ *
+ * Scans workspace for: MEMORY.md, memory/*.md, memory/weekly/*.md, memory/topics/*.md
+ * Extracts facts, deduplicates, and appends to memory/archival.jsonl.
+ *
+ * Usage: node migrate-legacy.mjs [workspace_path]
+ */
+
+import { readFileSync, appendFileSync, existsSync, readdirSync } from "node:fs";
+import { join, basename } from "node:path";
+
+const WS = process.argv[2] || process.env.OPENCLAW_WORKSPACE || join(process.env.HOME || "/tmp", ".openclaw", "workspace");
+const ARCHIVAL = join(WS, "memory", "archival.jsonl");
+
+console.log(`🧠 Legacy memory migration`);
+console.log(`   Workspace: ${WS}`);
+console.log(`   Archival:  ${ARCHIVAL}`);
+console.log(``);
+
+// Load existing archival for dedup
+const existingContent = new Set();
+if (existsSync(ARCHIVAL)) {
+  for (const line of readFileSync(ARCHIVAL, "utf-8").trim().split("\n").filter(Boolean)) {
+    try { existingContent.add(JSON.parse(line).content?.toLowerCase()); } catch {}
+  }
+}
+console.log(`Existing archival: ${existingContent.size} records`);
+
+// Generic entity inference (no personal data)
+const ENTITY_PATTERNS = [
+  [/\b(IBKR|Interactive Brokers|NAV|portfolio|投资|HELOC|mortgage|finance)/i, "finance"],
+  [/\b(immigration|PR|IRCC|CBSA|visa|律师|lawyer|petition)/i, "immigration"],
+  [/\b(quant|trading|backtest|signal|portfolio|Sharpe)/i, "trading"],
+  [/\b(doctor|医生|hospital|health|medication|药|体检|clinic)/i, "health"],
+  [/\b(car|vehicle|SUV|sedan|truck)\b/i, "vehicles"],
+  [/\b(k3d|ArgoCD|Helm|kubectl|GitOps|cluster|deploy|CI|CD)/i, "infrastructure"],
+  [/\b(OpenClaw|gateway|plugin|session|agent|memory|compaction)/i, "openclaw"],
+  [/\b(Discord|Telegram|Slack|bot|channel)/i, "messaging"],
+  [/\b(school|university|college|学校|education)/i, "education"],
+  [/\b(house|home|property|rent|房)/i, "property"],
+  [/\b(lawyer|legal|court|lawsuit|案|诉)/i, "legal"],
+];
+
+function inferEntity(text) {
+  for (const [pat, name] of ENTITY_PATTERNS) {
+    if (pat.test(text)) return name;
+  }
+  return "general";
+}
+
+function extractFacts(text) {
+  const facts = [];
+  for (const line of text.split(/\n/).map((l) => l.trim()).filter(Boolean)) {
+    if (line.startsWith("#") || line.length < 15) continue;
+    if (/^(##|===|---|\*\*\*|```|>|\|)/.test(line)) continue;
+    const sentences = line.split(/(?<=[。.！!？?；;])\s*/).filter(Boolean);
+    for (const s of sentences) {
+      const clean = s.replace(/^[-*•]\s*/, "").replace(/^\d+\.\s*/, "").trim();
+      if (clean.length >= 15 && clean.length <= 500) facts.push(clean);
+    }
+  }
+  return facts;
+}
+
+// Collect all legacy files
+const files = [];
+
+// MEMORY.md
+const memoryMd = join(WS, "MEMORY.md");
+if (existsSync(memoryMd)) files.push({ path: memoryMd, tag: "long-term" });
+
+// memory/*.md (daily logs)
+const memDir = join(WS, "memory");
+if (existsSync(memDir)) {
+  for (const f of readdirSync(memDir).filter((f) => /\.md$/.test(f) && f !== ".abstract")) {
+    files.push({ path: join(memDir, f), tag: "daily" });
+  }
+}
+
+// memory/weekly/*.md
+const weeklyDir = join(WS, "memory", "weekly");
+if (existsSync(weeklyDir)) {
+  for (const f of readdirSync(weeklyDir).filter((f) => f.endsWith(".md"))) {
+    files.push({ path: join(weeklyDir, f), tag: "weekly" });
+  }
+}
+
+// memory/topics/*.md
+const topicDir = join(WS, "memory", "topics");
+if (existsSync(topicDir)) {
+  for (const f of readdirSync(topicDir).filter((f) => f.endsWith(".md"))) {
+    files.push({ path: join(topicDir, f), tag: "topic" });
+  }
+}
+
+if (files.length === 0) {
+  console.log("\nNo legacy memory files found. Nothing to migrate.");
+  process.exit(0);
+}
+
+console.log(`Found ${files.length} files to scan\n`);
+
+let inserted = 0;
+let skipped = 0;
+
+for (const { path, tag } of files) {
+  const content = readFileSync(path, "utf-8");
+  const facts = extractFacts(content);
+  let fileInserted = 0;
+
+  for (const fact of facts) {
+    const factLower = fact.toLowerCase();
+
+    // Exact dedup
+    if (existingContent.has(factLower)) {
+      skipped++;
+      continue;
+    }
+
+    // Keyword overlap dedup (>75% overlap = skip)
+    let isDupe = false;
+    const factWords = new Set(factLower.split(/\s+/).filter((w) => w.length > 2));
+    if (factWords.size > 0) {
+      for (const ex of existingContent) {
+        const exWords = new Set(ex.split(/\s+/).filter((w) => w.length > 2));
+        let overlap = 0;
+        for (const w of factWords) {
+          if (exWords.has(w)) overlap++;
+        }
+        if (overlap / factWords.size > 0.75) {
+          isDupe = true;
+          break;
+        }
+      }
+    }
+    if (isDupe) {
+      skipped++;
+      continue;
+    }
+
+    const record = {
+      id: `arch-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+      ts: new Date().toISOString(),
+      last_accessed: null,
+      access_count: 0,
+      importance: 5,
+      content: fact,
+      entity: inferEntity(fact),
+      tags: [tag],
+      source: "migration",
+    };
+
+    appendFileSync(ARCHIVAL, JSON.stringify(record) + "\n", "utf-8");
+    existingContent.add(factLower);
+    inserted++;
+    fileInserted++;
+  }
+
+  if (fileInserted > 0) console.log(`  ${basename(path)}: +${fileInserted} facts`);
+}
+
+console.log(`\n✅ Migration complete: ${inserted} facts imported, ${skipped} skipped (duplicates)`);
+console.log(`Total archival: ${existingContent.size} records`);

package/index.js

@@ -20,7 +20,7 @@ import { existsSync } from "node:fs";
 import { resolveWorkspace, getCoreSizeLimit, DEFAULT_TOP_K, MAX_TOP_K } from "./lib/paths.js";
 import { readCore, writeCore, dotGet, dotSet, autoParse } from "./lib/core.js";
 import { loadArchival, appendRecord, rewriteArchival, archivalPath } from "./lib/archival.js";
-import { indexEmbedding, loadEmbeddingCache, saveEmbeddingCache } from "./lib/embedding.js";
+import { indexEmbedding, loadEmbeddingCache, saveEmbeddingCache, backfillEmbeddings } from "./lib/embedding.js";
 import { hybridSearch } from "./lib/search.js";
 import { consolidateText } from "./lib/consolidate.js";
 import { findDuplicates, applyDedup } from "./lib/dedup.js";
@@ -95,6 +95,26 @@ export default definePluginEntry({
     // Factory ctx has: { sessionKey, workspaceDir, agentId, ... }
     // ═══════════════════════════════════════════════════════════════════
 
+    // Background: auto-backfill missing embeddings on startup
+    const defaultWs = resolveWorkspace(null);
+    setTimeout(() => {
+      try {
+        const records = loadArchival(defaultWs);
+        const cache = loadEmbeddingCache(defaultWs);
+        const missing = records.filter((r) => r.id && !cache[r.id]).length;
+        if (missing > 0) {
+          console.error(`[memory-engine] Backfilling ${missing} missing embeddings...`);
+          backfillEmbeddings(defaultWs, records, {
+            onProgress: (done, total) => {
+              if (done % 500 === 0) console.error(`[memory-engine] Embedding backfill: ${done}/${total}`);
+            },
+          }).then((result) => {
+            console.error(`[memory-engine] Backfill complete: ${result.processed} embedded, ${result.errors} errors`);
+          }).catch(() => {});
+        }
+      } catch { /* ignore startup errors */ }
+    }, 10000); // delay 10s after gateway start to avoid blocking
+
     // ─── core_memory_read ───
     api.registerTool(withAgent((agentId) => ({
       name: "core_memory_read",

package/lib/embedding.js

@@ -68,3 +68,67 @@ export async function indexEmbedding(ws, record) {
     saveEmbeddingCache(ws);
   }
 }
+
+/**
+ * Batch-embed records that are missing from cache.
+ * Runs in background with batching (100 per API call) and rate limiting.
+ * @param {string} ws - workspace path
+ * @param {object[]} records - all archival records
+ * @param {object} [options]
+ * @param {number} [options.batchSize=100]
+ * @param {number} [options.delayMs=200]
+ * @param {function} [options.onProgress] - callback(done, total)
+ */
+export async function backfillEmbeddings(ws, records, options = {}) {
+  const apiKey = resolveApiKey();
+  if (!apiKey) return { processed: 0, errors: 0, skipped: 0 };
+
+  const batchSize = options.batchSize || 100;
+  const delayMs = options.delayMs || 200;
+  const cache = loadEmbeddingCache(ws);
+
+  const missing = records.filter((r) => r.id && !cache[r.id]);
+  if (missing.length === 0) return { processed: 0, errors: 0, skipped: 0 };
+
+  let processed = 0;
+  let errors = 0;
+
+  for (let i = 0; i < missing.length; i += batchSize) {
+    const batch = missing.slice(i, i + batchSize);
+    const texts = batch.map((r) =>
+      [r.content, r.entity, ...(r.tags || [])].filter(Boolean).join(" "),
+    );
+
+    try {
+      const res = await fetch("https://api.openai.com/v1/embeddings", {
+        method: "POST",
+        headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+        body: JSON.stringify({ input: texts, model: EMBEDDING_MODEL, dimensions: EMBEDDING_DIM }),
+      });
+
+      if (!res.ok) { errors++; continue; }
+
+      const data = await res.json();
+      for (let j = 0; j < batch.length; j++) {
+        if (data.data?.[j]?.embedding) {
+          cache[batch[j].id] = data.data[j].embedding;
+          processed++;
+        }
+      }
+
+      // Save after each batch
+      saveEmbeddingCache(ws);
+
+      if (options.onProgress) options.onProgress(processed, missing.length);
+
+      // Rate limit
+      if (i + batchSize < missing.length) {
+        await new Promise((r) => setTimeout(r, delayMs));
+      }
+    } catch {
+      errors++;
+    }
+  }
+
+  return { processed, errors, skipped: missing.length - processed - errors };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@icex-labs/openclaw-memory-engine",
-  "version": "3.3.2",
+  "version": "3.5.0",
   "description": "MemGPT-style hierarchical memory plugin for OpenClaw — core memory block + archival storage with semantic search",
   "type": "module",
   "main": "index.js",

package/setup.sh

@@ -106,6 +106,40 @@ else
   echo "⏭️  archival.jsonl already exists ($lines records)"
 fi
 
+# --- 3b. Migrate legacy memory files into archival ---
+if command -v node &>/dev/null && [ -f "$PLUGIN_DIR/extras/migrate-legacy.mjs" ]; then
+  # Check if there are legacy files to migrate
+  legacy_count=0
+  [ -f "$WORKSPACE/MEMORY.md" ] && legacy_count=$((legacy_count + 1))
+  legacy_count=$((legacy_count + $(ls "$MEMORY_DIR"/*.md 2>/dev/null | wc -l | tr -d ' ')))
+  legacy_count=$((legacy_count + $(ls "$MEMORY_DIR"/weekly/*.md 2>/dev/null | wc -l | tr -d ' ')))
+  legacy_count=$((legacy_count + $(ls "$MEMORY_DIR"/topics/*.md 2>/dev/null | wc -l | tr -d ' ')))
+
+  archival_count=$(wc -l < "$MEMORY_DIR/archival.jsonl" 2>/dev/null | tr -d ' ' || echo "0")
+
+  if [ "$legacy_count" -gt 0 ] && [ "$archival_count" -lt 10 ]; then
+    echo ""
+    echo "📦 Found $legacy_count legacy memory files (MEMORY.md, daily logs, weekly summaries, topics)."
+    if $NON_INTERACTIVE; then
+      echo "   Migrating automatically..."
+      node "$PLUGIN_DIR/extras/migrate-legacy.mjs" "$WORKSPACE" 2>&1 | tail -3
+    else
+      printf "   Migrate into archival memory? [Y/n]: "
+      read -r migrate_answer
+      if [ "${migrate_answer:-Y}" != "n" ] && [ "${migrate_answer:-Y}" != "N" ]; then
+        node "$PLUGIN_DIR/extras/migrate-legacy.mjs" "$WORKSPACE" 2>&1 | tail -5
+      else
+        echo "⏭️  Skipping migration. Run manually later: node $PLUGIN_DIR/extras/migrate-legacy.mjs $WORKSPACE"
+      fi
+    fi
+    echo ""
+  else
+    if [ "$archival_count" -gt 10 ]; then
+      echo "⏭️  Archival already has $archival_count records, skipping migration"
+    fi
+  fi
+fi
+
 # --- 4. Install memory-maintenance.sh ---
 SCRIPTS_DIR="$WORKSPACE/scripts"
 mkdir -p "$SCRIPTS_DIR"