@icex-labs/openclaw-memory-engine 3.3.1

package/lib/backup.js

@@ -0,0 +1,99 @@
+/**
+ * Export/import for backup and migration.
+ */
+
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { readCore, writeCore } from "./core.js";
+import { loadArchival, appendRecord, rewriteArchival } from "./archival.js";
+import { loadEmbeddingCache, saveEmbeddingCache } from "./embedding.js";
+
+const FORMAT_TAG = "openclaw-memory-engine";
+const FORMAT_VERSION = "1.2.0";
+
+/**
+ * Export core + archival + embeddings to a JSON file.
+ * @returns {{ path: string, stats: object }}
+ */
+export function exportMemory(ws, outputPath) {
+  const date = new Date().toISOString().slice(0, 10);
+  const outPath = outputPath || join(ws, "memory", `export-${date}.json`);
+
+  const core = readCore(ws);
+  const records = loadArchival(ws);
+  const embeddings = loadEmbeddingCache(ws);
+
+  const data = {
+    _meta: {
+      format: FORMAT_TAG,
+      version: FORMAT_VERSION,
+      exported_at: new Date().toISOString(),
+      workspace: ws,
+    },
+    core,
+    archival: records,
+    embeddings,
+    stats: {
+      core_size: JSON.stringify(core).length,
+      archival_count: records.length,
+      embedding_count: Object.keys(embeddings).length,
+    },
+  };
+
+  writeFileSync(outPath, JSON.stringify(data, null, 2), "utf-8");
+  return { path: outPath, stats: data.stats };
+}
+
+/**
+ * Import from an export file.
+ * @param {string} mode - "replace" or "merge"
+ * @returns {string} result description
+ */
+export function importMemory(ws, inputPath, mode = "merge") {
+  if (!existsSync(inputPath)) throw new Error(`File not found: ${inputPath}`);
+
+  const raw = readFileSync(inputPath, "utf-8");
+  let data;
+  try { data = JSON.parse(raw); } catch (e) { throw new Error(`Invalid JSON: ${e.message}`); }
+
+  if (data._meta?.format !== FORMAT_TAG) {
+    throw new Error("Not a memory-engine export file.");
+  }
+
+  if (mode === "replace") {
+    if (data.core) writeCore(ws, data.core);
+    if (data.archival) rewriteArchival(ws, data.archival);
+    if (data.embeddings) {
+      // Direct write to cache + disk
+      const cache = loadEmbeddingCache(ws);
+      Object.assign(cache, data.embeddings);
+      saveEmbeddingCache(ws);
+    }
+    return `REPLACED: core + ${data.archival?.length || 0} archival + ${Object.keys(data.embeddings || {}).length} embeddings`;
+  }
+
+  // Merge mode
+  const existing = loadArchival(ws);
+  const existingContents = new Set(existing.map((r) => r.content));
+  const importRecords = data.archival || [];
+
+  let added = 0;
+  for (const r of importRecords) {
+    if (!existingContents.has(r.content)) {
+      appendRecord(ws, { content: r.content, entity: r.entity, tags: r.tags, source: "import" });
+      existingContents.add(r.content);
+      added++;
+    }
+  }
+
+  let embAdded = 0;
+  if (data.embeddings) {
+    const embCache = loadEmbeddingCache(ws);
+    for (const [id, emb] of Object.entries(data.embeddings)) {
+      if (!embCache[id]) { embCache[id] = emb; embAdded++; }
+    }
+    saveEmbeddingCache(ws);
+  }
+
+  return `MERGED: ${added} new records (${importRecords.length - added} skipped), ${embAdded} new embeddings`;
+}

package/lib/consolidate.js

@@ -0,0 +1,102 @@
+/**
+ * Auto-extract structured facts from text blocks.
+ * Splits by sentence boundaries (Chinese + English), infers entity, deduplicates.
+ */
+
+import { loadArchival, appendRecord } from "./archival.js";
+import { indexEmbedding } from "./embedding.js";
+
+/** Generic entity inference patterns (no personal data). */
+const ENTITY_PATTERNS = [
+  [/\b(IBKR|Interactive Brokers)\b/i, "IBKR"],
+  [/\b(immigration|PR|IRCC|CBSA|visa)\b/i, "immigration"],
+  [/\b(quant|trading|backtest|portfolio)\b/i, "trading"],
+  [/\b(doctor|医生|hospital|医院|clinic)\b/i, "health"],
+  [/\b(car|vehicle|SUV|sedan|truck|Tesla|Toyota|Lexus|BMW)\b/i, "vehicles"],
+  [/\b(house|home|mortgage|rent|property)\b/i, "property"],
+  [/\b(school|university|college|学校)\b/i, "education"],
+  [/\b(insurance|保险)\b/i, "insurance"],
+  [/\b(lawyer|律师|attorney|legal)\b/i, "legal"],
+];
+
+function inferEntity(text, fallback) {
+  for (const [pat, name] of ENTITY_PATTERNS) {
+    if (pat.test(text)) return name;
+  }
+  return fallback;
+}
+
+/** Split text into sentence-level fact candidates. */
+function extractCandidates(text) {
+  const rawLines = text.split(/\n/).map((l) => l.trim()).filter(Boolean);
+  const segments = [];
+
+  for (const line of rawLines) {
+    const sentences = line
+      .split(/(?<=[。.！!？?；;])\s*/)
+      .map((s) => s.trim())
+      .filter(Boolean);
+    segments.push(...(sentences.length > 1 ? sentences : [line]));
+  }
+
+  return segments
+    .filter((seg) => {
+      if (seg.startsWith("#") || seg.length < 10) return false;
+      if (/^(##|===|---|\*\*\*)/.test(seg)) return false;
+      return true;
+    })
+    .map((seg) => seg.replace(/^[-*•]\s*/, "").replace(/^\d+\.\s*/, "").trim())
+    .filter((s) => s.length >= 10);
+}
+
+/** Check if a fact is a near-duplicate of existing content (keyword overlap >70%). */
+function isDuplicate(factLower, existingTexts) {
+  const factWords = new Set(factLower.split(/\s+/).filter((w) => w.length > 2));
+  if (factWords.size === 0) return false;
+
+  for (const ex of existingTexts) {
+    const exWords = new Set(ex.split(/\s+/).filter((w) => w.length > 2));
+    let overlap = 0;
+    for (const w of factWords) {
+      if (exWords.has(w)) overlap++;
+    }
+    if (overlap / factWords.size > 0.7) return true;
+  }
+  return false;
+}
+
+/**
+ * Extract facts from text, deduplicate, and insert into archival.
+ * @returns {{ inserted: string[], skipped: string[], total: number }}
+ */
+export async function consolidateText(ws, text, defaultEntity = "", defaultTags = []) {
+  const candidates = extractCandidates(text);
+  if (candidates.length === 0) return { inserted: [], skipped: [], total: 0 };
+
+  const existing = loadArchival(ws);
+  const existingTexts = existing.map((r) => (r.content || "").toLowerCase());
+  const inserted = [];
+  const skipped = [];
+
+  for (const fact of candidates) {
+    const factLower = fact.toLowerCase();
+
+    if (isDuplicate(factLower, existingTexts)) {
+      skipped.push(fact.slice(0, 60));
+      continue;
+    }
+
+    const entity = inferEntity(fact, defaultEntity);
+    const record = appendRecord(ws, {
+      content: fact,
+      entity,
+      tags: defaultTags,
+      source: "consolidate",
+    });
+    indexEmbedding(ws, record).catch(() => {});
+    inserted.push(record.id);
+    existingTexts.push(factLower);
+  }
+
+  return { inserted, skipped, total: candidates.length };
+}

package/lib/core.js

@@ -0,0 +1,76 @@
+/**
+ * Core memory: small structured identity block (~500 tokens).
+ * Stored as memory/core.json. Agent reads at session start, updates atomically.
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { corePath } from "./paths.js";
+
+const DEFAULT_CORE = {
+  _meta: { version: 1, updated_at: "", description: "Core memory block." },
+  user: {},
+  relationship: {},
+  preferences: {},
+  current_focus: [],
+};
+
+export function readCore(ws) {
+  const p = corePath(ws);
+  if (!existsSync(p)) {
+    mkdirSync(join(ws, "memory"), { recursive: true });
+    const init = {
+      ...DEFAULT_CORE,
+      _meta: { ...DEFAULT_CORE._meta, updated_at: new Date().toISOString() },
+    };
+    writeFileSync(p, JSON.stringify(init, null, 2), "utf-8");
+    return init;
+  }
+  return JSON.parse(readFileSync(p, "utf-8"));
+}
+
+export function writeCore(ws, data) {
+  data._meta = data._meta || {};
+  data._meta.updated_at = new Date().toISOString();
+  writeFileSync(corePath(ws), JSON.stringify(data, null, 2), "utf-8");
+}
+
+/** Navigate a dot-path to read a value. */
+export function dotGet(obj, path) {
+  const parts = path.split(".");
+  let cur = obj;
+  for (const p of parts) {
+    if (cur == null || typeof cur !== "object") return undefined;
+    cur = cur[p];
+  }
+  return cur;
+}
+
+/** Navigate a dot-path to set a value. Returns old value. */
+export function dotSet(obj, path, value) {
+  const parts = path.split(".");
+  let cur = obj;
+  for (let i = 0; i < parts.length - 1; i++) {
+    if (cur[parts[i]] === undefined) cur[parts[i]] = {};
+    cur = cur[parts[i]];
+  }
+  const old = cur[parts[parts.length - 1]];
+  cur[parts[parts.length - 1]] = value;
+  return old;
+}
+
+/**
+ * Auto-parse: if value is a JSON string that looks like an array/object, parse it.
+ * Fixes LLMs passing '["a","b"]' as a string instead of an actual array.
+ */
+export function autoParse(value) {
+  if (typeof value !== "string") return value;
+  const trimmed = value.trim();
+  if (
+    (trimmed.startsWith("[") && trimmed.endsWith("]")) ||
+    (trimmed.startsWith("{") && trimmed.endsWith("}"))
+  ) {
+    try { return JSON.parse(trimmed); } catch { /* keep as string */ }
+  }
+  return value;
+}

package/lib/dashboard.js

@@ -0,0 +1,235 @@
+/**
+ * Dashboard: generates a self-contained HTML file for browsing memory.
+ * Timeline view, graph visualization, search explorer, stats.
+ */
+
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { readCore } from "./core.js";
+import { loadArchival } from "./archival.js";
+import { loadGraph } from "./graph.js";
+import { loadEpisodes } from "./episodes.js";
+import { loadEmbeddingCache } from "./embedding.js";
+import { analyzePatterns } from "./reflection.js";
+
+/**
+ * Generate a self-contained HTML dashboard file.
+ * @returns {string} output path
+ */
+export function generateDashboard(ws, outputPath = null) {
+  const outPath = outputPath || join(ws, "memory", "dashboard.html");
+
+  const core = readCore(ws);
+  const archival = loadArchival(ws);
+  const graph = loadGraph(ws);
+  const episodes = loadEpisodes(ws);
+  const embCache = loadEmbeddingCache(ws);
+  const reflection = analyzePatterns(ws, 30);
+
+  const embCount = Object.keys(embCache).length;
+
+  // Build data for the HTML
+  const data = {
+    generatedAt: new Date().toISOString(),
+    core,
+    stats: {
+      archival: archival.length,
+      graph: graph.length,
+      episodes: episodes.length,
+      embeddings: embCount,
+    },
+    reflection,
+    // Limit data size for HTML
+    recentArchival: archival.slice(-50).reverse(),
+    recentEpisodes: episodes.slice(-20).reverse(),
+    graphTriples: graph.slice(-100),
+    entities: [...new Set([
+      ...archival.map((r) => r.entity).filter(Boolean),
+      ...graph.map((t) => t.s),
+      ...graph.map((t) => t.o),
+    ])].sort(),
+  };
+
+  const html = renderHtml(data);
+  writeFileSync(outPath, html, "utf-8");
+  return outPath;
+}
+
+function renderHtml(data) {
+  const coreJson = JSON.stringify(data.core, null, 2);
+  const archivalJson = JSON.stringify(data.recentArchival);
+  const episodesJson = JSON.stringify(data.recentEpisodes);
+  const graphJson = JSON.stringify(data.graphTriples);
+  const reflectionJson = JSON.stringify(data.reflection);
+  const entitiesJson = JSON.stringify(data.entities);
+
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Memory Engine Dashboard</title>
+<style>
+  :root { --bg: #0d1117; --card: #161b22; --border: #30363d; --text: #c9d1d9; --accent: #58a6ff; --green: #3fb950; --yellow: #d29922; --red: #f85149; }
+  * { box-sizing: border-box; margin: 0; padding: 0; }
+  body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: var(--bg); color: var(--text); padding: 20px; max-width: 1200px; margin: 0 auto; }
+  h1 { color: var(--accent); margin-bottom: 8px; font-size: 24px; }
+  h2 { color: var(--accent); margin: 24px 0 12px; font-size: 18px; border-bottom: 1px solid var(--border); padding-bottom: 8px; }
+  h3 { color: var(--text); margin: 16px 0 8px; font-size: 15px; }
+  .subtitle { color: #8b949e; font-size: 13px; margin-bottom: 20px; }
+  .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 12px; margin-bottom: 20px; }
+  .stat-card { background: var(--card); border: 1px solid var(--border); border-radius: 8px; padding: 16px; text-align: center; }
+  .stat-num { font-size: 28px; font-weight: 700; color: var(--accent); }
+  .stat-label { font-size: 12px; color: #8b949e; margin-top: 4px; }
+  .card { background: var(--card); border: 1px solid var(--border); border-radius: 8px; padding: 16px; margin-bottom: 12px; }
+  .card-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 8px; }
+  .card-date { color: #8b949e; font-size: 12px; }
+  .card-entity { background: var(--accent); color: #000; padding: 2px 8px; border-radius: 12px; font-size: 11px; font-weight: 600; }
+  .card-content { font-size: 14px; line-height: 1.5; }
+  .tag { display: inline-block; background: #21262d; border: 1px solid var(--border); color: #8b949e; padding: 1px 6px; border-radius: 4px; font-size: 11px; margin: 2px; }
+  .importance { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; }
+  .importance.high { background: var(--red); }
+  .importance.mid { background: var(--yellow); }
+  .importance.low { background: var(--green); }
+  .triple { font-family: monospace; font-size: 13px; padding: 6px 0; border-bottom: 1px solid var(--border); }
+  .triple:last-child { border-bottom: none; }
+  .episode { border-left: 3px solid var(--accent); padding-left: 12px; margin-bottom: 16px; }
+  .episode-mood { color: var(--yellow); font-size: 12px; }
+  .episode-decisions { color: var(--green); font-size: 13px; margin-top: 4px; }
+  pre { background: #0d1117; border: 1px solid var(--border); border-radius: 6px; padding: 12px; overflow-x: auto; font-size: 12px; color: var(--green); }
+  .tabs { display: flex; gap: 0; margin-bottom: 0; border-bottom: 1px solid var(--border); }
+  .tab { padding: 8px 16px; cursor: pointer; color: #8b949e; border-bottom: 2px solid transparent; font-size: 14px; }
+  .tab.active { color: var(--accent); border-bottom-color: var(--accent); }
+  .tab-content { display: none; padding-top: 16px; }
+  .tab-content.active { display: block; }
+  .search-box { width: 100%; padding: 8px 12px; background: var(--bg); border: 1px solid var(--border); border-radius: 6px; color: var(--text); font-size: 14px; margin-bottom: 12px; }
+  .bar { display: inline-block; height: 16px; background: var(--accent); border-radius: 2px; margin-right: 4px; vertical-align: middle; }
+</style>
+</head>
+<body>
+
+<h1>Memory Engine Dashboard</h1>
+<div class="subtitle">Generated: ${data.generatedAt.slice(0, 19).replace("T", " ")} UTC</div>
+
+<div class="grid">
+  <div class="stat-card"><div class="stat-num">${data.stats.archival}</div><div class="stat-label">Facts</div></div>
+  <div class="stat-card"><div class="stat-num">${data.stats.graph}</div><div class="stat-label">Graph Triples</div></div>
+  <div class="stat-card"><div class="stat-num">${data.stats.episodes}</div><div class="stat-label">Episodes</div></div>
+  <div class="stat-card"><div class="stat-num">${data.stats.embeddings}</div><div class="stat-label">Embeddings</div></div>
+</div>
+
+<div class="tabs">
+  <div class="tab active" onclick="switchTab('facts')">Facts</div>
+  <div class="tab" onclick="switchTab('graph')">Graph</div>
+  <div class="tab" onclick="switchTab('episodes')">Episodes</div>
+  <div class="tab" onclick="switchTab('core')">Core Memory</div>
+  <div class="tab" onclick="switchTab('reflection')">Reflection</div>
+</div>
+
+<div id="tab-facts" class="tab-content active">
+  <input class="search-box" id="fact-search" placeholder="Search facts..." oninput="filterFacts()">
+  <div id="fact-list"></div>
+</div>
+
+<div id="tab-graph" class="tab-content">
+  <h3>Knowledge Graph (${data.stats.graph} triples)</h3>
+  <div id="graph-list"></div>
+</div>
+
+<div id="tab-episodes" class="tab-content">
+  <h3>Conversation Episodes (${data.stats.episodes})</h3>
+  <div id="episode-list"></div>
+</div>
+
+<div id="tab-core" class="tab-content">
+  <h3>Core Memory Block</h3>
+  <pre>${escapeHtml(coreJson)}</pre>
+</div>
+
+<div id="tab-reflection" class="tab-content">
+  <h3>30-Day Reflection</h3>
+  <div id="reflection-content"></div>
+</div>
+
+<script>
+const archival = ${archivalJson};
+const episodes = ${episodesJson};
+const graph = ${graphJson};
+const reflection = ${reflectionJson};
+const coreJson = ${JSON.stringify(coreJson)};
+
+function escapeHtml(s) { return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;'); }
+
+function switchTab(name) {
+  document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
+  document.querySelectorAll('.tab-content').forEach(t => t.classList.remove('active'));
+  event.target.classList.add('active');
+  document.getElementById('tab-' + name).classList.add('active');
+}
+
+function impClass(v) { return v >= 7 ? 'high' : v <= 3 ? 'low' : 'mid'; }
+
+function renderFacts(facts) {
+  return facts.map(r => \`
+    <div class="card">
+      <div class="card-header">
+        <span><span class="importance \${impClass(r.importance||5)}"></span>\${r.entity ? '<span class="card-entity">'+r.entity+'</span>' : ''}</span>
+        <span class="card-date">\${(r.ts||'').slice(0,10)} · imp=\${r.importance||5} · accessed=\${r.access_count||0}</span>
+      </div>
+      <div class="card-content">\${escapeHtml(r.content)}</div>
+      <div>\${(r.tags||[]).map(t => '<span class="tag">#'+t+'</span>').join('')}</div>
+    </div>
+  \`).join('');
+}
+
+function filterFacts() {
+  const q = document.getElementById('fact-search').value.toLowerCase();
+  const filtered = q ? archival.filter(r =>
+    (r.content||'').toLowerCase().includes(q) ||
+    (r.entity||'').toLowerCase().includes(q) ||
+    (r.tags||[]).some(t => t.toLowerCase().includes(q))
+  ) : archival;
+  document.getElementById('fact-list').innerHTML = renderFacts(filtered);
+}
+
+// Init facts
+document.getElementById('fact-list').innerHTML = renderFacts(archival);
+
+// Init graph
+document.getElementById('graph-list').innerHTML = graph.map(t =>
+  '<div class="triple">(' + escapeHtml(t.s) + ') <span style="color:var(--accent)">—' + escapeHtml(t.r) + '→</span> (' + escapeHtml(t.o) + ')</div>'
+).join('') || '<p style="color:#8b949e">No graph triples yet.</p>';
+
+// Init episodes
+document.getElementById('episode-list').innerHTML = episodes.map(ep => \`
+  <div class="episode">
+    <div class="card-date">\${(ep.ts||'').slice(0,10)} \${ep.mood ? '<span class="episode-mood">[\${ep.mood}]</span>' : ''}</div>
+    <div class="card-content" style="margin:4px 0">\${escapeHtml(ep.summary||'')}</div>
+    \${(ep.decisions||[]).length ? '<div class="episode-decisions">Decisions: '+(ep.decisions||[]).join('; ')+'</div>' : ''}
+    <div>\${(ep.topics||[]).map(t => '<span class="tag">'+t+'</span>').join('')}</div>
+  </div>
+\`).join('') || '<p style="color:#8b949e">No episodes yet.</p>';
+
+// Init reflection
+const ref = reflection;
+const topTopics = (ref.top_topics||[]).map(([t,c]) => '<div>'+t+': <span class="bar" style="width:'+Math.min(c*20,200)+'px"></span> '+c+'</div>').join('');
+document.getElementById('reflection-content').innerHTML = \`
+  <div class="grid">
+    <div class="stat-card"><div class="stat-num">\${ref.activity?.new_facts||0}</div><div class="stat-label">New Facts (30d)</div></div>
+    <div class="stat-card"><div class="stat-num">\${ref.activity?.new_episodes||0}</div><div class="stat-label">New Episodes</div></div>
+    <div class="stat-card"><div class="stat-num">\${ref.activity?.total_decisions||0}</div><div class="stat-label">Decisions</div></div>
+    <div class="stat-card"><div class="stat-num">\${ref.health?.forgetting_candidates||0}</div><div class="stat-label">Forgetting Candidates</div></div>
+  </div>
+  <h3>Top Topics</h3>\${topTopics||'<p style="color:#8b949e">No data</p>'}
+  <h3>Time Distribution</h3>
+  <div>Morning: \${ref.time_distribution?.morning||0} · Afternoon: \${ref.time_distribution?.afternoon||0} · Evening: \${ref.time_distribution?.evening||0} · Night: \${ref.time_distribution?.night||0}</div>
+  \${ref.health?.neglected_entities?.length ? '<h3>Neglected Entities</h3><div>'+(ref.health.neglected_entities||[]).join(', ')+'</div>' : ''}
+\`;
+</script>
+</body>
+</html>`;
+}
+
+function escapeHtml(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}

package/lib/dedup.js

@@ -0,0 +1,68 @@
+/**
+ * Deduplication via embedding cosine similarity.
+ */
+
+import { loadArchival, rewriteArchival } from "./archival.js";
+import {
+  loadEmbeddingCache, saveEmbeddingCache, getEmbedding, cosineSimilarity,
+} from "./embedding.js";
+import { DEDUP_SIMILARITY_THRESHOLD } from "./paths.js";
+
+/**
+ * Find near-duplicate pairs in archival memory.
+ * @returns {Array<{ keep: object, drop: object, similarity: number }>}
+ */
+export async function findDuplicates(ws) {
+  const records = loadArchival(ws);
+  const embCache = loadEmbeddingCache(ws);
+
+  // Build missing embeddings
+  for (const r of records) {
+    if (!embCache[r.id]) {
+      const emb = await getEmbedding(
+        [r.content, r.entity, ...(r.tags || [])].filter(Boolean).join(" "),
+      );
+      if (emb) embCache[r.id] = emb;
+    }
+  }
+  saveEmbeddingCache(ws);
+
+  // O(n²) pairwise comparison
+  const dupes = [];
+  for (let i = 0; i < records.length; i++) {
+    for (let j = i + 1; j < records.length; j++) {
+      const embA = embCache[records[i].id];
+      const embB = embCache[records[j].id];
+      if (!embA || !embB) continue;
+      const sim = cosineSimilarity(embA, embB);
+      if (sim >= DEDUP_SIMILARITY_THRESHOLD) {
+        const keepIdx =
+          (records[j].access_count || 0) >= (records[i].access_count || 0) ? j : i;
+        const dropIdx = keepIdx === i ? j : i;
+        dupes.push({
+          keep: records[keepIdx],
+          drop: records[dropIdx],
+          similarity: Math.round(sim * 1000) / 1000,
+        });
+      }
+    }
+  }
+  return dupes;
+}
+
+/**
+ * Remove duplicate records from archival.
+ * @returns {{ removed: number, remaining: number }}
+ */
+export function applyDedup(ws, dupes) {
+  const records = loadArchival(ws);
+  const dropIds = new Set(dupes.map((d) => d.drop.id));
+  const cleaned = records.filter((r) => !dropIds.has(r.id));
+  rewriteArchival(ws, cleaned);
+
+  const embCache = loadEmbeddingCache(ws);
+  for (const id of dropIds) delete embCache[id];
+  saveEmbeddingCache(ws);
+
+  return { removed: dupes.length, remaining: cleaned.length };
+}

package/lib/embedding.js

@@ -0,0 +1,70 @@
+/**
+ * OpenAI embedding API + local file cache.
+ */
+
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { embeddingCachePath, EMBEDDING_MODEL, EMBEDDING_DIM } from "./paths.js";
+
+/** In-memory embedding cache keyed by workspace path. */
+const cacheMap = new Map();
+
+export function loadEmbeddingCache(ws) {
+  if (cacheMap.has(ws)) return cacheMap.get(ws);
+  const p = embeddingCachePath(ws);
+  let data = {};
+  if (existsSync(p)) {
+    try { data = JSON.parse(readFileSync(p, "utf-8")); } catch { /* ignore */ }
+  }
+  cacheMap.set(ws, data);
+  return data;
+}
+
+export function saveEmbeddingCache(ws) {
+  const data = cacheMap.get(ws);
+  if (!data) return;
+  writeFileSync(embeddingCachePath(ws), JSON.stringify(data), "utf-8");
+}
+
+export function resolveApiKey() {
+  return process.env.OPENAI_API_KEY || null;
+}
+
+/** Fetch embedding vector from OpenAI. Returns float[] or null. */
+export async function getEmbedding(text) {
+  const apiKey = resolveApiKey();
+  if (!apiKey) return null;
+  try {
+    const res = await fetch("https://api.openai.com/v1/embeddings", {
+      method: "POST",
+      headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+      body: JSON.stringify({ input: text, model: EMBEDDING_MODEL, dimensions: EMBEDDING_DIM }),
+    });
+    if (!res.ok) return null;
+    const data = await res.json();
+    return data?.data?.[0]?.embedding || null;
+  } catch { return null; }
+}
+
+export function cosineSimilarity(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+/** Compute and cache embedding for a record (non-blocking). */
+export async function indexEmbedding(ws, record) {
+  if (!resolveApiKey()) return;
+  const text = [record.content, record.entity, ...(record.tags || [])].filter(Boolean).join(" ");
+  const emb = await getEmbedding(text);
+  if (emb) {
+    const cache = loadEmbeddingCache(ws);
+    cache[record.id] = emb;
+    saveEmbeddingCache(ws);
+  }
+}