memory-lancedb-pro 1.0.30 → 1.0.32

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 1.0.32
+
+- Fix: strip OpenClaw `Conversation info` / `Sender` metadata noise before auto-capture matching and adaptive retrieval normalization, reducing false captures and noisy retrieval triggers.
+- Fix: parse `autoRecallMinRepeated` from plugin config so repeated-memory suppression works when configured.
+
+PR: #50
+
+---
+
+
+## 1.0.31
+
+- Fix: `memory-pro import` now preserves provided IDs and is idempotent (skips if ID already exists).
+
 ## 1.0.26
 
 **Access Reinforcement for Time Decay**

package/cli.ts

@@ -379,25 +379,78 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void {
               continue;
             }
 
-            // Check for duplicates
-            const existing = await context.retriever.retrieve({
-              query: text,
-              limit: 1,
-              scopeFilter: [targetScope],
-            });
-            if (existing.length > 0 && existing[0].score > 0.95) {
+            const categoryRaw = memory.category;
+            const category: MemoryEntry["category"] =
+              categoryRaw === "preference" ||
+              categoryRaw === "fact" ||
+              categoryRaw === "decision" ||
+              categoryRaw === "entity" ||
+              categoryRaw === "other"
+                ? categoryRaw
+                : "other";
+
+            const importanceRaw = Number(memory.importance);
+            const importance = Number.isFinite(importanceRaw)
+              ? Math.max(0, Math.min(1, importanceRaw))
+              : 0.7;
+
+            const timestampRaw = Number(memory.timestamp);
+            const timestamp = Number.isFinite(timestampRaw) ? timestampRaw : Date.now();
+
+            const metadataRaw = memory.metadata;
+            const metadata =
+              typeof metadataRaw === "string"
+                ? metadataRaw
+                : metadataRaw != null
+                  ? JSON.stringify(metadataRaw)
+                  : "{}";
+
+            const idRaw = memory.id;
+            const id = typeof idRaw === "string" && idRaw.length > 0 ? idRaw : undefined;
+
+            // Idempotency: if the import file includes an id and we already have it, skip.
+            if (id && (await context.store.hasId(id))) {
               skipped++;
               continue;
             }
 
+            // Back-compat dedupe: if no id provided, do a best-effort similarity check.
+            if (!id) {
+              const existing = await context.retriever.retrieve({
+                query: text,
+                limit: 1,
+                scopeFilter: [targetScope],
+              });
+              if (existing.length > 0 && existing[0].score > 0.95) {
+                skipped++;
+                continue;
+              }
+            }
+
             const vector = await context.embedder.embedPassage(text);
-            await context.store.store({
-              text,
-              vector,
-              importance: memory.importance ?? 0.7,
-              category: memory.category || "other",
-              scope: targetScope,
-            });
+
+            if (id) {
+              await context.store.importEntry({
+                id,
+                text,
+                vector,
+                category,
+                scope: targetScope,
+                importance,
+                timestamp,
+                metadata,
+              });
+            } else {
+              await context.store.store({
+                text,
+                vector,
+                importance,
+                category,
+                scope: targetScope,
+                metadata,
+              });
+            }
+
             imported++;
           } catch (error) {
             console.warn(`Failed to import memory: ${error}`);

package/index.ts

@@ -6,7 +6,7 @@
 import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
 import { homedir } from "node:os";
 import { join, dirname, basename } from "node:path";
-import { readFile, readdir, writeFile, mkdir } from "node:fs/promises";
+import { readFile, readdir, writeFile, mkdir, appendFile } from "node:fs/promises";
 import { readFileSync } from "node:fs";
 
 // Import core components
@@ -16,6 +16,7 @@ import { createRetriever, DEFAULT_RETRIEVAL_CONFIG } from "./src/retriever.js";
 import { createScopeManager } from "./src/scopes.js";
 import { createMigrator } from "./src/migrate.js";
 import { registerAllMemoryTools } from "./src/tools.js";
+import type { MdMirrorWriter } from "./src/tools.js";
 import { shouldSkipRetrieval } from "./src/adaptive-retrieval.js";
 import { AccessTracker } from "./src/access-tracker.js";
 import { createMemoryCLI } from "./cli.js";
@@ -39,6 +40,7 @@ interface PluginConfig {
   autoCapture?: boolean;
   autoRecall?: boolean;
   autoRecallMinLength?: number;
+  autoRecallMinRepeated?: number;
   captureAssistant?: boolean;
   retrieval?: {
     mode?: "hybrid" | "vector";
@@ -67,6 +69,7 @@ interface PluginConfig {
   };
   enableManagementTools?: boolean;
   sessionMemory?: { enabled?: boolean; messageCount?: number };
+  mdMirror?: { enabled?: boolean; dir?: string };
 }
 
 // ============================================================================
@@ -138,7 +141,11 @@ const CAPTURE_EXCLUDE_PATTERNS = [
 ];
 
 export function shouldCapture(text: string): boolean {
-  const s = text.trim();
+  let s = text.trim();
+
+  // Strip OpenClaw metadata headers (Conversation info or Sender)
+  const metadataPattern = /^(Conversation info|Sender) \(untrusted metadata\):[\s\S]*?\n\s*\n/gim;
+  s = s.replace(metadataPattern, "");
 
   // CJK characters carry more meaning per character, use lower minimum threshold
   const hasCJK = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(
@@ -337,6 +344,92 @@ async function findPreviousSessionFile(
   } catch {}
 }
 
+// ============================================================================
+// Markdown Mirror (dual-write)
+// ============================================================================
+
+type AgentWorkspaceMap = Record<string, string>;
+
+function resolveAgentWorkspaceMap(api: OpenClawPluginApi): AgentWorkspaceMap {
+  const map: AgentWorkspaceMap = {};
+
+  // Try api.config first (runtime config)
+  const agents = Array.isArray((api as any).config?.agents?.list)
+    ? (api as any).config.agents.list
+    : [];
+
+  for (const agent of agents) {
+    if (agent?.id && typeof agent.workspace === "string") {
+      map[String(agent.id)] = agent.workspace;
+    }
+  }
+
+  // Fallback: read from openclaw.json (respect OPENCLAW_HOME if set)
+  if (Object.keys(map).length === 0) {
+    try {
+      const openclawHome = process.env.OPENCLAW_HOME || join(homedir(), ".openclaw");
+      const configPath = join(openclawHome, "openclaw.json");
+      const raw = readFileSync(configPath, "utf8");
+      const parsed = JSON.parse(raw);
+      const list = parsed?.agents?.list;
+      if (Array.isArray(list)) {
+        for (const agent of list) {
+          if (agent?.id && typeof agent.workspace === "string") {
+            map[String(agent.id)] = agent.workspace;
+          }
+        }
+      }
+    } catch {
+      /* silent */
+    }
+  }
+
+  return map;
+}
+
+function createMdMirrorWriter(
+  api: OpenClawPluginApi,
+  config: PluginConfig,
+): MdMirrorWriter | null {
+  if (config.mdMirror?.enabled !== true) return null;
+
+  const fallbackDir = api.resolvePath(config.mdMirror.dir || "memory-md");
+  const workspaceMap = resolveAgentWorkspaceMap(api);
+
+  if (Object.keys(workspaceMap).length > 0) {
+    api.logger.info(
+      `mdMirror: resolved ${Object.keys(workspaceMap).length} agent workspace(s)`,
+    );
+  } else {
+    api.logger.warn(
+      `mdMirror: no agent workspaces found, writes will use fallback dir: ${fallbackDir}`,
+    );
+  }
+
+  return async (entry, meta) => {
+    try {
+      const ts = new Date(entry.timestamp || Date.now());
+      const dateStr = ts.toISOString().split("T")[0];
+
+      let mirrorDir = fallbackDir;
+      if (meta?.agentId && workspaceMap[meta.agentId]) {
+        mirrorDir = join(workspaceMap[meta.agentId], "memory");
+      }
+
+      const filePath = join(mirrorDir, `${dateStr}.md`);
+      const agentLabel = meta?.agentId ? ` agent=${meta.agentId}` : "";
+      const sourceLabel = meta?.source ? ` source=${meta.source}` : "";
+      const safeText = entry.text.replace(/\n/g, " ").slice(0, 500);
+      const line = `- ${ts.toISOString()} [${entry.category}:${entry.scope}]${agentLabel}${sourceLabel} ${safeText}\n`;
+
+      await mkdir(mirrorDir, { recursive: true });
+      await appendFile(filePath, line, "utf8");
+    } catch (err) {
+      api.logger.warn(`mdMirror: write failed: ${String(err)}`);
+    }
+  };
+}
+
 // ============================================================================
 // Version
 // ============================================================================
@@ -427,6 +520,12 @@ const memoryLanceDBProPlugin = {
       `memory-lancedb-pro@${pluginVersion}: plugin registered (db: ${resolvedDbPath}, model: ${config.embedding.model || "text-embedding-3-small"})`,
     );
 
+    // ========================================================================
+    // Markdown Mirror
+    // ========================================================================
+
+    const mdMirror = createMdMirrorWriter(api, config);
+
     // ========================================================================
     // Register Tools
     // ========================================================================
@@ -439,6 +538,7 @@ const memoryLanceDBProPlugin = {
         scopeManager,
         embedder,
         agentId: undefined, // Will be determined at runtime from context
+        mdMirror,
       },
       {
         enableManagementTools: config.enableManagementTools,
@@ -649,6 +749,14 @@ const memoryLanceDBProPlugin = {
               scope: defaultScope,
             });
             stored++;
+
+            // Dual-write to Markdown mirror if enabled
+            if (mdMirror) {
+              await mdMirror(
+                { text, category, scope: defaultScope, timestamp: Date.now() },
+                { source: "auto-capture", agentId },
+              );
+            }
           }
 
           if (stored > 0) {
@@ -758,6 +866,14 @@ const memoryLanceDBProPlugin = {
             }),
           });
 
+          // Dual-write to Markdown mirror if enabled
+          if (mdMirror) {
+            await mdMirror(
+              { text: memoryText.replace(/\n/g, " ").slice(0, 500), category: "fact", scope: "global", timestamp: Date.now() },
+              { source: "session-memory" },
+            );
+          }
+
           api.logger.info(
             `session-memory: stored session summary for ${currentSessionId || "unknown"}`,
           );
@@ -987,6 +1103,7 @@ function parsePluginConfig(value: unknown): PluginConfig {
     // Default OFF: only enable when explicitly set to true.
     autoRecall: cfg.autoRecall === true,
     autoRecallMinLength: parsePositiveInt(cfg.autoRecallMinLength),
+    autoRecallMinRepeated: parsePositiveInt(cfg.autoRecallMinRepeated),
     captureAssistant: cfg.captureAssistant === true,
     retrieval:
       typeof cfg.retrieval === "object" && cfg.retrieval !== null
@@ -1010,6 +1127,17 @@ function parsePluginConfig(value: unknown): PluginConfig {
                 : undefined,
           }
         : undefined,
+    mdMirror:
+      typeof cfg.mdMirror === "object" && cfg.mdMirror !== null
+        ? {
+            enabled:
+              (cfg.mdMirror as Record<string, unknown>).enabled === true,
+            dir:
+              typeof (cfg.mdMirror as Record<string, unknown>).dir === "string"
+                ? ((cfg.mdMirror as Record<string, unknown>).dir as string)
+                : undefined,
+          }
+        : undefined,
   };
 }
 

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "memory-lancedb-pro",
   "name": "Memory (LanceDB Pro)",
   "description": "Enhanced LanceDB-backed long-term memory with hybrid retrieval, multi-scope isolation, long-context chunking, and management CLI",
-  "version": "1.0.26",
+  "version": "1.0.32",
   "kind": "memory",
   "configSchema": {
     "type": "object",
@@ -270,6 +270,21 @@
             }
           }
         }
+      },
+      "mdMirror": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "enabled": {
+            "type": "boolean",
+            "default": false,
+            "description": "Enable dual-write: store memories in both LanceDB and human-readable Markdown files"
+          },
+          "dir": {
+            "type": "string",
+            "description": "Fallback directory for Markdown mirror files when agent workspace is unknown"
+          }
+        }
       }
     },
     "required": [
@@ -448,6 +463,15 @@
       "label": "Management Tools",
       "help": "Enable memory_list and memory_stats tools for debugging and auditing",
       "advanced": true
+    },
+    "mdMirror.enabled": {
+      "label": "Markdown Mirror",
+      "help": "Write a human-readable Markdown copy alongside LanceDB storage (dual-write mode)"
+    },
+    "mdMirror.dir": {
+      "label": "Mirror Fallback Directory",
+      "help": "Fallback directory when agent workspace mapping is unavailable",
+      "advanced": true
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-lancedb-pro",
-  "version": "1.0.30",
+  "version": "1.0.32",
   "description": "OpenClaw enhanced LanceDB memory plugin with hybrid retrieval (Vector + BM25), cross-encoder rerank, multi-scope isolation, long-context chunking, and management CLI",
   "type": "module",
   "main": "index.ts",

package/src/adaptive-retrieval.ts

@@ -46,15 +46,10 @@ const FORCE_RETRIEVE_PATTERNS = [
 function normalizeQuery(query: string): string {
   let s = query.trim();
 
-  // 1. Strip OpenClaw injected metadata header (Conversation info or Sender).
-  if (/^(Conversation info|Sender) \(untrusted metadata\):/i.test(s)) {
-    s = s.replace(/^(Conversation info|Sender) \(untrusted metadata\):\s*/i, "");
-    // If there is a blank-line separator (after JSON block), keep only the part after it.
-    const parts = s.split(/\n\s*\n/, 2);
-    if (parts.length === 2) {
-      s = parts[1].trim();
-    }
-  }
+  // 1. Strip OpenClaw injected metadata headers (Conversation info or Sender).
+  // Use a global regex to strip all metadata blocks including following blank lines.
+  const metadataPattern = /^(Conversation info|Sender) \(untrusted metadata\):[\s\S]*?\n\s*\n/gim;
+  s = s.replace(metadataPattern, "");
 
   // 2. Strip OpenClaw cron wrapper prefix.
   s = s.trim().replace(/^\[cron:[^\]]+\]\s*/i, "");

package/src/tools.ts

@@ -24,12 +24,18 @@ export const MEMORY_CATEGORIES = [
   "other",
 ] as const;
 
+export type MdMirrorWriter = (
+  entry: { text: string; category: string; scope: string; timestamp?: number },
+  meta?: { source?: string; agentId?: string },
+) => Promise<void>;
+
 interface ToolContext {
   retriever: MemoryRetriever;
   store: MemoryStore;
   scopeManager: MemoryScopeManager;
   embedder: Embedder;
   agentId?: string;
+  mdMirror?: MdMirrorWriter | null;
 }
 
 function resolveAgentId(runtimeAgentId: unknown, fallback?: string): string | undefined {
@@ -301,6 +307,14 @@ export function registerMemoryStoreTool(
             scope: targetScope,
           });
 
+          // Dual-write to Markdown mirror if enabled
+          if (context.mdMirror) {
+            await context.mdMirror(
+              { text, category: category as string, scope: targetScope, timestamp: entry.timestamp },
+              { source: "memory_store", agentId },
+            );
+          }
+
           return {
             content: [
               {

package/test/cli-smoke.mjs

@@ -1,5 +1,5 @@
 import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 
@@ -47,14 +47,23 @@ async function runCliSmoke() {
   const program = new Command();
   program.exitOverride();
 
+  const { MemoryStore } = jiti("../src/store.ts");
+
+  const store = new MemoryStore({
+    dbPath: path.join(workDir, "target-db"),
+    vectorDim: 4,
+  });
+
   const context = {
-    // Minimal store interface for reembed dry-run.
-    store: { dbPath: path.join(workDir, "target-db") },
-    retriever: {},
-    scopeManager: {},
+    store,
+    // Only used for similarity-based dedupe when the import file has no id.
+    retriever: { retrieve: async () => [] },
+    scopeManager: { getDefaultScope: () => "global" },
     migrator: {},
-    // Presence required, but dry-run exits before embeddings.
-    embedder: {},
+    // Stub embedder used by import/reembed.
+    embedder: {
+      embedPassage: async () => [0, 0, 0, 0],
+    },
   };
 
   // Register commands under `memory-pro`
@@ -78,7 +87,71 @@ async function runCliSmoke() {
     "--dry-run",
   ]);
 
-  // 3) Access reinforcement formula smoke test
+  // 3) import should preserve id and be idempotent (skip on second import)
+  const importId = "smoke_import_id_1";
+  const importPhrase = `smoke-import-${Date.now()}`;
+  const importFile = path.join(workDir, "import-test.json");
+
+  writeFileSync(
+    importFile,
+    JSON.stringify(
+      {
+        version: "1.0",
+        exportedAt: new Date().toISOString(),
+        count: 1,
+        filters: {},
+        memories: [
+          {
+            id: importId,
+            text: `Import smoke test. UniquePhrase=${importPhrase}.`,
+            category: "other",
+            scope: "global",
+            importance: 0.3,
+            timestamp: Date.now(),
+            metadata: "{}",
+          },
+        ],
+      },
+      null,
+      2,
+    ),
+  );
+
+  const captureLogs = async (argv) => {
+    const logs = [];
+    const origLog = console.log;
+    console.log = (...args) => logs.push(args.join(" "));
+    try {
+      await program.parseAsync(argv);
+    } finally {
+      console.log = origLog;
+    }
+    return logs.join("\n");
+  };
+
+  const out1 = await captureLogs([
+    "node",
+    "openclaw",
+    "memory-pro",
+    "import",
+    importFile,
+    "--scope",
+    "agent:smoke",
+  ]);
+  assert.match(out1, /Import completed: 1 imported/, out1);
+
+  const out2 = await captureLogs([
+    "node",
+    "openclaw",
+    "memory-pro",
+    "import",
+    importFile,
+    "--scope",
+    "agent:smoke",
+  ]);
+  assert.match(out2, /Import completed: 0 imported, 1 skipped/, out2);
+
+  // 4) Access reinforcement formula smoke test
   const { parseAccessMetadata, buildUpdatedMetadata, computeEffectiveHalfLife } =
     jiti("../src/access-tracker.ts");