mr-memory 3.7.0 → 3.7.1

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "mr-memory",
-  "version": "3.7.0",
-  "description": "MemoryRouter persistent memory plugin for OpenClaw — your AI remembers every conversation",
+  "version": "3.7.1",
+  "description": "mr-memory is the MemoryRouter plugin for OpenClaw — persistent memory across every conversation",
   "type": "module",
   "files": [
     "README.md",
+    "index.js",
+    "upload.js",
     "index.ts",
-    "openclaw.plugin.json",
-    "upload.ts"
+    "upload.ts",
+    "openclaw.plugin.json"
   ],
   "keywords": [
     "openclaw",
@@ -32,7 +34,8 @@
   },
   "openclaw": {
     "extensions": [
-      "./index.ts"
+      "./index.js"
     ]
-  }
+  },
+  "main": "./index.js"
 }

package/upload.js

@@ -0,0 +1,469 @@
+/**
+ * MemoryRouter Upload — File discovery, parsing, batching, and upload.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+// ── Sanitization utilities (shared with index.ts ingest logic)
+// These MUST stay in sync with the patterns in index.ts
+const MEMORY_TAG_RE = /<mr-memory>[\s\S]*?<\/mr-memory>\s*/g;
+const LEGACY_TAG_RE = /<memory_context>[\s\S]*?<\/memory_context>\s*(?:The above are retrieved memories|IMPORTANT: The above block contains retrieved memories)[^\n]*\n*/g;
+function stripOldMemory(text) {
+    return text.replace(MEMORY_TAG_RE, "").replace(LEGACY_TAG_RE, "").trim();
+}
+const ENVELOPE_METADATA_RE = /Conversation info \(untrusted metadata\):\s*```json\s*\{[^}]*\}\s*```\s*/g;
+const SENDER_METADATA_RE = /Sender \(untrusted metadata\):\s*```json\s*\{[^}]*\}\s*```\s*/g;
+const REPLY_CONTEXT_RE = /Replied message \(untrusted, for context\):\s*```json\s*\{[^}]*\}\s*```\s*/g;
+const MEMORY_INSTRUCTION_RE = /The above are retrieved memories from past conversations[^\n]*\n*/g;
+const IMPORTANT_MEMORY_RE = /IMPORTANT: The above (?:are retrieved memories|block contains retrieved memories)[^\n]*\n*/g;
+const MEDIA_ATTACHED_RE = /\[media attached[^\]]*\]/gi;
+const MEDIA_TAGS_RE = /<media:(?:audio|image|video|document)>/gi;
+const MEDIA_INSTRUCTION_RE = /To send an image back,[\s\S]*?Keep caption in the text body\./gi;
+const MEDIA_INBOUND_PATH_RE = /\/Users\/[^\s]*\/media\/inbound\/[^\s)\]"]*/gi;
+const IMAGE_DATA_RE = /\[image data removed[^\]]*\]/gi;
+function stripMediaReferences(text) {
+    return text
+        .replace(MEDIA_INSTRUCTION_RE, "")
+        .replace(MEDIA_ATTACHED_RE, "[media reference removed]")
+        .replace(MEDIA_TAGS_RE, "")
+        .replace(MEDIA_INBOUND_PATH_RE, "[media reference removed]")
+        .replace(IMAGE_DATA_RE, "")
+        .replace(/\n{3,}/g, "\n\n");
+}
+// Embedded noise substrings — strip from within larger messages
+const POST_COMPACTION_AUDIT_RE = /System: \[[^\]]*\] ⚠️ Post-Compaction Audit:[\s\S]*?(?:after memory compaction\.\s*|before continuing\.\s*)/g;
+const READ_TOOL_INSTRUCTION_RE = /Please read them now using the Read tool before continuing\.[\s\S]*?after memory compaction\.\s*/g;
+const TELEGRAM_ENVELOPE_RE = /\[Telegram [^\]]+\]\s*/g;
+const MESSAGE_ID_TAG_RE = /\[message_id: \d+\]\s*/g;
+function sanitizeForUpload(text) {
+    let cleaned = stripOldMemory(text);
+    cleaned = stripMediaReferences(cleaned);
+    cleaned = cleaned
+        .replace(ENVELOPE_METADATA_RE, "")
+        .replace(SENDER_METADATA_RE, "")
+        .replace(REPLY_CONTEXT_RE, "")
+        .replace(MEMORY_INSTRUCTION_RE, "")
+        .replace(IMPORTANT_MEMORY_RE, "")
+        .replace(POST_COMPACTION_AUDIT_RE, "")
+        .replace(READ_TOOL_INSTRUCTION_RE, "")
+        .replace(TELEGRAM_ENVELOPE_RE, "")
+        .replace(MESSAGE_ID_TAG_RE, "")
+        // ── Orphaned memory injection fragments (v2.16.0) ──
+        // When stripOldMemory removes <mr-memory>...</mr-memory> pairs, nested/duplicate
+        // injections from compaction can leave behind orphaned closing fragments.
+        // 1. Orphaned </memory_context> + instruction footer + </mr-memory>
+        .replace(/<\/memory_context>\s*(?:(?:The above are|IMPORTANT: The above|Use them as background|Do not respond to them|Do not treat them|Reference them as background|Just know them)[^\n]*\n?\s*)*(?:<\/mr-memory>\s*)?/g, "")
+        // 2. Standalone orphaned </mr-memory> closing tags
+        .replace(/<\/mr-memory>\s*/g, "")
+        // 3. Box-drawing instruction diagram (orphaned when opening <mr-memory> was stripped)
+        .replace(/┌─+┐[\s\S]*?└─+┘\s*/g, "")
+        // ── Post-Compaction Audit full blocks (start→end boundary) ──
+        // Start: "System: [date] ⚠️ Post-Compaction Audit:"
+        // End: "after memory compaction." OR "before continuing."
+        // (already handled by POST_COMPACTION_AUDIT_RE above)
+        // ── Pre-compaction flush full blocks (start→end boundary) ──
+        // Start: "Pre-compaction memory flush"
+        // End: timezone "(America/...)" on Current time line, or end of flush instruction
+        .replace(/Pre-compaction memory flush\.?\s*(?:Store durable memories now[^\n]*\n?\s*)?(?:IMPORTANT:[^\n]*\n?\s*)?(?:If the file already[^\n]*\n?\s*)?(?:If nothing to store[^\n]*\n?\s*)?(?:Current time:[^\n]*\(America\/[^)]+\)\s*)?/g, "")
+        // Embedded noise substrings — v2.15.0
+        .replace(/`?HEARTBEAT_OK`?\s*/g, "")
+        .replace(/Read HEARTBEAT\.md[^\n]*(?:\n(?:Follow it strictly|Do not infer|If nothing needs)[^\n]*)*/g, "")
+        .replace(/\n*\s*NO_REPLY\s*$/g, "")
+        .replace(/You can respond with NO_REPLY[^\n]*\n*/g, "")
+        .replace(/If nothing to store,? reply (?:with )?NO_REPLY[^\n]*\n*/g, "")
+        .replace(/Store durable memories now[^\n]*\n*/g, "")
+        .replace(/This ensures your operating protocols are restored[^\n]*\n*/g, "")
+        .replace(/Current time:[^\n]*\n*/g, "")
+        .replace(/\bset:\s*\n\s*-\s*WORKFLOW_AUTO\.md[^\n]*\n(?:\s*-\s*memory[^\n]*\n)*/g, "")
+        .replace(/Do not mention technical details[^\n]*\n*/g, "");
+    cleaned = cleaned.replace(/\n{3,}/g, "\n\n").trim();
+    return cleaned;
+}
+const SYSTEM_NOISE_PATTERNS = [
+    /^Read HEARTBEAT\.md if it exists/,
+    /^Read HEARTBEAT\.md/,
+    /HEARTBEAT_OK\s*$/,
+    /^Pre-compaction memory flush/,
+    /⚠️ Post-Compaction Audit:/,
+    /^NO_REPLY\s*$/,
+    /^Note: The previous agent run was aborted/,
+    /^\[Queued messages while agent was busy\]/,
+    /^\[media reference removed\]\s*$/,
+    /^System: \[\d{4}-\d{2}-\d{2}/,
+    /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [A-Z]{2,4}\] ⚠️/,
+    /^Store durable memories now/,
+    /^Current time: .+\(America\//,
+    /^A new session was started via \/new or \/reset/,
+    /^Please read them now using the Read tool/,
+    /^GatewayRestart:\s*\{/,
+    /^This ensures your operating protocols are restored/,
+    // Cron job prompts — automated system instructions, not human conversation
+    /^\[cron:[^\]]+\]/,
+    // Sub-agent completion announcements (system-generated)
+    /^A background task "[^"]*" just completed/,
+    // Sub-agent stats lines
+    /^Stats: runtime \d+/,
+    // Summarize instruction for sub-agent completions
+    /^Summarize this naturally for the user/,
+    // Session reset greet prompts
+    /^Greet the user in your configured persona/,
+];
+function isSystemNoise(text) {
+    const trimmed = text.trim();
+    if (!trimmed)
+        return true;
+    return SYSTEM_NOISE_PATTERNS.some(pattern => pattern.test(trimmed));
+}
+/**
+ * Detect OpenClaw compaction summary blobs — large user messages containing
+ * interleaved [USER] and [ASSISTANT] markers from conversation history dumps.
+ * These are 100% redundant since individual messages are already uploaded.
+ */
+function isCompactionSummary(text, role) {
+    if (role !== "user")
+        return false;
+    if (text.length < 2000)
+        return false;
+    const markers = (text.match(/\[(?:USER|ASSISTANT)[\] ]/g) || []).length;
+    return markers >= 3;
+}
+const MAX_ITEM_CHARS = 8000;
+const TARGET_CHUNK_CHARS = 4000;
+const MAX_BATCH_BYTES = 2_000_000;
+const MAX_BATCH_COUNT_DEFAULT = 100;
+const MAX_BATCH_COUNT_QWEN = 25;
+const BATCH_SLEEP_MS = 150;
+const MAX_HTTP_RETRIES = 3;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function chunkText(text, targetChars) {
+    const chunks = [];
+    let remaining = text;
+    while (remaining.length > targetChars) {
+        let splitAt = remaining.lastIndexOf("\n\n", targetChars);
+        if (splitAt < targetChars * 0.5) {
+            splitAt = remaining.lastIndexOf("\n", targetChars);
+        }
+        if (splitAt < targetChars * 0.5) {
+            splitAt = remaining.lastIndexOf(" ", targetChars);
+        }
+        if (splitAt < targetChars * 0.3) {
+            splitAt = targetChars;
+        }
+        chunks.push(remaining.slice(0, splitAt).trim());
+        remaining = remaining.slice(splitAt).trim();
+    }
+    if (remaining) {
+        chunks.push(remaining);
+    }
+    return chunks;
+}
+const BINARY_EXTENSIONS = new Set([".pdf", ".docx"]);
+const DOC_EXTENSIONS = new Set([".md", ".pdf", ".docx"]);
+async function extractText(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    try {
+        if (ext === ".pdf") {
+            const pdfParse = (await import("pdf-parse")).default;
+            const buffer = await fs.readFile(filePath);
+            return (await pdfParse(buffer)).text;
+        }
+        if (ext === ".docx") {
+            const mammoth = await import("mammoth");
+            return (await mammoth.extractRawText({ path: filePath })).value;
+        }
+    }
+    catch (err) {
+        console.error(`  ⚠ Failed to extract ${path.basename(filePath)}: ${err instanceof Error ? err.message : err}`);
+        return "";
+    }
+    return fs.readFile(filePath, "utf-8");
+}
+async function fileToJsonl(filePath) {
+    const content = await extractText(filePath);
+    const stat = await fs.stat(filePath);
+    const timestamp = Math.floor(stat.mtimeMs);
+    const filename = path.basename(filePath);
+    const trimmed = content.trim();
+    if (trimmed.length < 50)
+        return [];
+    if (trimmed.length <= MAX_ITEM_CHARS) {
+        return [{ content: `[${filename}] ${trimmed}`, role: "user", timestamp }];
+    }
+    const chunks = chunkText(trimmed, TARGET_CHUNK_CHARS);
+    return chunks.map((chunk, i) => ({
+        content: `[${filename} part ${i + 1}/${chunks.length}] ${chunk}`,
+        role: "user",
+        timestamp,
+    }));
+}
+async function sessionToJsonl(filePath) {
+    const content = await fs.readFile(filePath, "utf-8");
+    const lines = [];
+    for (const line of content.split("\n")) {
+        if (!line.trim())
+            continue;
+        try {
+            const parsed = JSON.parse(line);
+            if (parsed.type !== "message")
+                continue;
+            const msg = parsed.message;
+            if (!msg || !msg.role)
+                continue;
+            if (msg.role !== "user" && msg.role !== "assistant")
+                continue;
+            let text = "";
+            if (typeof msg.content === "string") {
+                text = msg.content;
+            }
+            else if (Array.isArray(msg.content)) {
+                text = msg.content
+                    .filter((block) => block.type === "text")
+                    .map((block) => block.text)
+                    .join("\n");
+            }
+            if (!text || text.trim().length < 20)
+                continue;
+            // Sanitize: strip memory injections, envelope metadata, media refs, system noise
+            text = sanitizeForUpload(text);
+            if (!text || text.trim().length < 20)
+                continue;
+            if (isSystemNoise(text))
+                continue;
+            if (isCompactionSummary(text, msg.role))
+                continue;
+            let timestamp;
+            if (typeof parsed.timestamp === "string") {
+                timestamp = new Date(parsed.timestamp).getTime();
+            }
+            else if (typeof parsed.timestamp === "number") {
+                timestamp = parsed.timestamp;
+            }
+            else {
+                timestamp = Date.now();
+            }
+            const trimmedText = text.trim();
+            // Chunk oversized messages to avoid blowing up the embedding service
+            if (trimmedText.length > MAX_ITEM_CHARS) {
+                const chunks = chunkText(trimmedText, TARGET_CHUNK_CHARS);
+                for (const chunk of chunks) {
+                    lines.push({ content: chunk, role: msg.role, timestamp });
+                }
+            }
+            else {
+                lines.push({ content: trimmedText, role: msg.role, timestamp });
+            }
+        }
+        catch {
+            // Skip invalid lines
+        }
+    }
+    return lines;
+}
+async function exists(p) {
+    try {
+        await fs.access(p);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function discoverWorkspaceFiles(workspacePath) {
+    const files = [];
+    const memoryMd = path.join(workspacePath, "MEMORY.md");
+    if (await exists(memoryMd))
+        files.push(memoryMd);
+    const memoryDir = path.join(workspacePath, "memory");
+    if (await exists(memoryDir)) {
+        const allMemFiles = await fs.readdir(memoryDir, { recursive: true });
+        const mdFiles = allMemFiles.filter((f) => {
+            const ext = path.extname(f).toLowerCase();
+            return DOC_EXTENSIONS.has(ext);
+        });
+        files.push(...mdFiles.map((f) => path.join(memoryDir, f)));
+    }
+    for (const contextFile of ["AGENTS.md", "TOOLS.md"]) {
+        const p = path.join(workspacePath, contextFile);
+        if (await exists(p))
+            files.push(p);
+    }
+    return files;
+}
+async function discoverBrainFiles(stateDir) {
+    const files = [];
+    const sessionsDir = path.join(stateDir, "agents", "main", "sessions");
+    if (await exists(sessionsDir)) {
+        const allSessionFiles = await fs.readdir(sessionsDir);
+        // Match both active sessions (.jsonl) and soft-deleted ones (.jsonl.deleted.*)
+        // OpenClaw renames old sessions instead of deleting them — the data is still valid
+        const sessionFiles = allSessionFiles.filter((f) => f.endsWith(".jsonl") || f.includes(".jsonl.deleted."));
+        files.push(...sessionFiles.map((f) => path.join(sessionsDir, f)));
+    }
+    return files;
+}
+async function fetchWithRetry(url, init, label) {
+    let lastError = null;
+    for (let attempt = 1; attempt <= MAX_HTTP_RETRIES; attempt++) {
+        try {
+            const res = await fetch(url, { ...init, signal: AbortSignal.timeout(30000) });
+            if (res.ok || res.status < 500)
+                return res;
+            lastError = new Error(`HTTP ${res.status}`);
+        }
+        catch (err) {
+            lastError = err instanceof Error ? err : new Error(String(err));
+        }
+        if (attempt < MAX_HTTP_RETRIES)
+            await sleep(1000 * attempt);
+    }
+    throw lastError ?? new Error(`${label}: failed after ${MAX_HTTP_RETRIES} attempts`);
+}
+export async function runUpload(params) {
+    const { memoryKey, endpoint, targetPath, stateDir, embeddings } = params;
+    const uploadUrl = `${endpoint}/v1/memory/upload`;
+    // Validate API reachability
+    try {
+        const res = await fetch(`${endpoint}/health`, { signal: AbortSignal.timeout(5000) });
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status}`);
+    }
+    catch {
+        console.error("Error: Could not reach MemoryRouter API.");
+        return;
+    }
+    const workspacePath = params.workspacePath ?? process.cwd();
+    let files;
+    if (targetPath) {
+        const resolved = path.resolve(targetPath);
+        const stat = await fs.stat(resolved);
+        if (stat.isDirectory()) {
+            const allDirFiles = await fs.readdir(resolved, { recursive: true });
+            files = allDirFiles
+                .filter((f) => {
+                const ext = path.extname(f).toLowerCase();
+                return DOC_EXTENSIONS.has(ext) || ext === ".jsonl";
+            })
+                .map((f) => path.join(resolved, f));
+        }
+        else {
+            files = [resolved];
+        }
+    }
+    else if (params.hasBrainFlag && !params.hasWorkspaceFlag) {
+        // --brain only: upload sessions from brain path
+        files = await discoverBrainFiles(stateDir);
+    }
+    else if (params.hasWorkspaceFlag && !params.hasBrainFlag) {
+        // --workspace only: upload workspace files only
+        files = await discoverWorkspaceFiles(workspacePath);
+    }
+    else {
+        // No flags or both flags: upload both workspace + sessions
+        const wsFiles = await discoverWorkspaceFiles(workspacePath);
+        const brainFiles = await discoverBrainFiles(stateDir);
+        files = [...wsFiles, ...brainFiles];
+    }
+    if (files.length === 0) {
+        console.log("No files found to upload.");
+        return;
+    }
+    const modelLabel = embeddings ? `(model: ${embeddings})` : "(model: bge)";
+    console.log(`Uploading ${files.length} files to MemoryRouter ${modelLabel}...`);
+    const allLines = [];
+    let skippedEmpty = 0;
+    for (const file of files) {
+        const displayName = path.basename(file);
+        try {
+            const isSession = file.endsWith(".jsonl") || file.includes(".jsonl.deleted.");
+            const lines = isSession ? await sessionToJsonl(file) : await fileToJsonl(file);
+            if (lines.length === 0) {
+                skippedEmpty++;
+                continue;
+            }
+            allLines.push(...lines);
+            console.log(`  ${displayName.padEnd(40, ".")} ✓ (${lines.length} chunks)`);
+        }
+        catch (err) {
+            console.log(`  ${displayName.padEnd(40, ".")} ✗ ${err instanceof Error ? err.message : "Error"}`);
+        }
+    }
+    if (skippedEmpty > 0)
+        console.log(`  Skipped ${skippedEmpty} empty files`);
+    if (allLines.length === 0) {
+        console.log("\nNo content to upload.");
+        return;
+    }
+    // Batch — smaller batches for Qwen (4096-dim vectors are 4x heavier)
+    const isQwen = embeddings && ['qwen', 'qwen3', 'qwen3-8b', 'qwen3-embedding', 'qwen3-embedding-8b'].includes(embeddings.toLowerCase());
+    const maxBatchCount = isQwen ? MAX_BATCH_COUNT_QWEN : MAX_BATCH_COUNT_DEFAULT;
+    const batches = [];
+    let currentBatch = [];
+    let currentBytes = 0;
+    for (const line of allLines) {
+        const lineBytes = JSON.stringify(line).length + 1;
+        if (currentBytes + lineBytes > MAX_BATCH_BYTES || currentBatch.length >= maxBatchCount) {
+            if (currentBatch.length > 0)
+                batches.push(currentBatch);
+            currentBatch = [line];
+            currentBytes = lineBytes;
+        }
+        else {
+            currentBatch.push(line);
+            currentBytes += lineBytes;
+        }
+    }
+    if (currentBatch.length > 0)
+        batches.push(currentBatch);
+    console.log(`\nSending ${allLines.length} chunks in ${batches.length} batches...`);
+    let totalProcessed = 0;
+    let totalFailed = 0;
+    for (let i = 0; i < batches.length; i++) {
+        if (i > 0)
+            await sleep(BATCH_SLEEP_MS);
+        const batch = batches[i];
+        const jsonlBody = batch.map((line) => JSON.stringify(line)).join("\n");
+        if (batches.length > 1) {
+            process.stdout.write(`  Batch ${i + 1}/${batches.length} (${batch.length} items)... `);
+        }
+        try {
+            const response = await fetchWithRetry(uploadUrl, {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${memoryKey}`,
+                    "Content-Type": "text/plain",
+                    ...(embeddings && { "X-Embedding-Model": embeddings }),
+                },
+                body: jsonlBody,
+            }, `Batch ${i + 1}`);
+            const result = (await response.json());
+            const batchStored = result.stats?.stored ?? result.stats?.inputItems ?? batch.length;
+            const batchFailed = result.stats?.failed ?? 0;
+            totalProcessed += batchStored;
+            totalFailed += batchFailed;
+            if (batchFailed > 0) {
+                const errHint = result.errors?.[0] ? ` (${result.errors[0].slice(0, 120)})` : "";
+                console.log(`⚠ ${batchStored} stored, ${batchFailed} skipped${errHint}`);
+                // Show detailed error diagnostics (up to 5 per batch)
+                if (result.errors && result.errors.length > 1) {
+                    for (const err of result.errors.slice(0, 5)) {
+                        console.log(`    → ${err.slice(0, 200)}`);
+                    }
+                    if (result.errors.length > 5)
+                        console.log(`    → ... and ${result.errors.length - 5} more`);
+                }
+            }
+            else {
+                console.log(`✓ ${batchStored} stored`);
+            }
+        }
+        catch (err) {
+            console.log(`✗ Failed: ${err instanceof Error ? err.message : String(err)}`);
+            totalFailed += batch.length;
+        }
+    }
+    console.log(`\n✅ ${totalProcessed} memories stored in vault`);
+    if (totalFailed > 0) {
+        console.log(`⚠️ ${totalFailed} failed (${((totalFailed / (totalProcessed + totalFailed)) * 100).toFixed(0)}%)`);
+    }
+}