mr-memory 3.1.0 → 3.2.0

package/index.ts

@@ -10,9 +10,11 @@
 
 import { readFile, readdir, stat, lstat } from "node:fs/promises";
 import { join, resolve, relative, isAbsolute, sep } from "node:path";
+import { homedir } from "node:os";
 import path from "node:path";
 import { spawn } from "node:child_process";
 import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
+import { syncWorkspaceFiles, runSyncCli, showManifestStatus } from "./sync.js";
 
 const DEFAULT_ENDPOINT = "https://api.memoryrouter.ai";
 
@@ -1192,6 +1194,33 @@ const memoryRouterPlugin = {
               console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
             }
           });
+
+        // ── Workspace Sync ──
+        mr.command("sync")
+          .description("Sync workspace files to vault using source_hash tracking")
+          .option("--workspace <dir>", "Workspace directory")
+          .option("--status", "Show manifest status instead of syncing")
+          .action(async (opts: { workspace?: string; status?: boolean }) => {
+            if (opts.status) {
+              await showManifestStatus();
+              return;
+            }
+            if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
+            const os = await import("node:os");
+            const configWorkspace = (api.config as any).workspace || (api.config as any).agents?.defaults?.workspace;
+            const workspaceDir = opts.workspace
+              ? path.resolve(opts.workspace)
+              : configWorkspace
+                ? path.resolve(configWorkspace.replace(/^~/, os.homedir()))
+                : path.join(os.homedir(), ".openclaw", "workspace");
+            await runSyncCli({
+              workspaceDir,
+              endpoint,
+              memoryKey,
+              embeddings,
+              logging: true,
+            });
+          });
       },
       { commands: ["mr"] },
     );
@@ -1202,8 +1231,41 @@ const memoryRouterPlugin = {
 
     api.registerService({
       id: "mr-memory",
-      start: () => {
-        if (memoryKey) api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
+      start: async () => {
+        if (memoryKey) {
+          api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
+
+          // ── Auto-sync workspace files (non-blocking) ──
+          // Resolve workspace directory
+          const configWorkspace = (api.config as any).workspace || (api.config as any).agents?.defaults?.workspace;
+          const workspaceDir = configWorkspace
+            ? path.resolve(configWorkspace.replace(/^~/, homedir()))
+            : path.join(homedir(), ".openclaw", "workspace");
+
+          // Fire and forget — don't block service startup
+          syncWorkspaceFiles({
+            workspaceDir,
+            endpoint,
+            memoryKey,
+            embeddings,
+            logging,
+          })
+            .then((result) => {
+              if (result.uploaded > 0 || result.deleted > 0) {
+                api.logger.info?.(
+                  `memoryrouter: sync complete — ${result.uploaded} uploaded, ${result.deleted} deleted, ${result.unchanged} unchanged`,
+                );
+              } else {
+                log(`memoryrouter: sync complete — ${result.unchanged} files unchanged`);
+              }
+              if (result.errors.length > 0) {
+                api.logger.warn?.(`memoryrouter: sync had ${result.errors.length} errors`);
+              }
+            })
+            .catch((err) => {
+              api.logger.warn?.(`memoryrouter: sync error — ${err instanceof Error ? err.message : String(err)}`);
+            });
+        }
       },
       stop: () => {
         api.logger.info?.("memoryrouter: stopped");

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "mr-memory",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "MemoryRouter persistent memory plugin for OpenClaw — your AI remembers every conversation",
   "type": "module",
   "files": [
     "README.md",
     "index.ts",
     "openclaw.plugin.json",
-    "upload.ts"
+    "upload.ts",
+    "sync.ts"
   ],
   "keywords": [
     "openclaw",

package/sync.ts

@@ -0,0 +1,367 @@
+/**
+ * MemoryRouter Workspace Sync — source_hash-based file tracking.
+ *
+ * Tracks workspace files via SHA-256 content hashes. When files change,
+ * old chunks are deleted and new content is uploaded with the new hash.
+ * This enables clean replacement without stale chunk accumulation.
+ *
+ * IMPORTANT: source_hash is ONLY for workspace files, NEVER for sessions.
+ */
+
+import { readFile, writeFile, readdir, lstat, mkdir } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { join, relative, sep } from "node:path";
+import { homedir } from "node:os";
+
+// ── Types ──
+
+export interface ManifestFileEntry {
+  source_hash: string;
+  size: number;
+  chunks: number;
+  uploaded_at: number;
+}
+
+export interface Manifest {
+  version: number;
+  lastSync: number;
+  files: Record<string, ManifestFileEntry>;
+}
+
+export interface SyncResult {
+  uploaded: number;
+  deleted: number;
+  unchanged: number;
+  errors: string[];
+}
+
+export interface SyncOptions {
+  workspaceDir: string;
+  endpoint: string;
+  memoryKey: string;
+  embeddings?: string;
+  logging?: boolean;
+}
+
+// ── Constants ──
+
+const MANIFEST_PATH = join(homedir(), ".openclaw", "mr-memory-manifest.json");
+const MAX_FILE_SIZE = 1_000_000; // 1MB limit
+const TEXT_EXTENSIONS = new Set(["md", "txt"]);
+
+// ── Manifest I/O ──
+
+export async function loadManifest(): Promise<Manifest> {
+  try {
+    const raw = await readFile(MANIFEST_PATH, "utf-8");
+    return JSON.parse(raw) as Manifest;
+  } catch {
+    // First run or corrupted — create empty manifest
+    return { version: 1, lastSync: 0, files: {} };
+  }
+}
+
+export async function saveManifest(manifest: Manifest): Promise<void> {
+  manifest.lastSync = Date.now();
+  // Ensure directory exists
+  const dir = join(homedir(), ".openclaw");
+  await mkdir(dir, { recursive: true });
+  await writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
+}
+
+// ── Hash Computation ──
+
+export function computeSourceHash(content: string): string {
+  const hash = createHash("sha256").update(content).digest("hex");
+  return hash.substring(0, 16); // First 16 hex chars, matching MR API format
+}
+
+// ── File Discovery ──
+
+/**
+ * Discover ALL workspace files to sync:
+ * - All .md and .txt files in workspace root
+ * - All files recursively under memory/ directory
+ * - Excludes: hidden files/dirs, node_modules/, files > 1MB, binary files
+ */
+export async function discoverWorkspaceFiles(workspaceDir: string): Promise<string[]> {
+  const files: string[] = [];
+
+  async function walk(dir: string) {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return; // Directory doesn't exist or not readable
+    }
+
+    for (const entry of entries) {
+      // Skip hidden dirs/files and node_modules
+      if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
+
+      const fullPath = join(dir, entry.name);
+
+      if (entry.isDirectory()) {
+        await walk(fullPath);
+      } else if (entry.isFile()) {
+        // Only text files we want to index
+        const ext = entry.name.toLowerCase().split(".").pop() || "";
+        if (!TEXT_EXTENSIONS.has(ext)) continue;
+
+        // Check file size
+        try {
+          const stat = await lstat(fullPath);
+          if (stat.size > MAX_FILE_SIZE) continue;
+          if (stat.size === 0) continue; // Skip empty files
+          files.push(relative(workspaceDir, fullPath));
+        } catch {
+          continue; // Skip unreadable files
+        }
+      }
+    }
+  }
+
+  await walk(workspaceDir);
+  return files.sort();
+}
+
+// ── API Helpers ──
+
+/**
+ * Upload file content with source_hash using isolated chunking.
+ * Uses POST /v1/memory/upload-source endpoint.
+ */
+export async function uploadSourceFile(
+  endpoint: string,
+  memoryKey: string,
+  content: string,
+  sourceHash: string,
+  embeddings?: string,
+): Promise<{ stored: number; chunks: number }> {
+  const res = await fetch(`${endpoint}/v1/memory/upload-source`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${memoryKey}`,
+      "Content-Type": "application/json",
+      ...(embeddings && { "X-Embedding-Model": embeddings }),
+    },
+    body: JSON.stringify({ content, source_hash: sourceHash }),
+  });
+
+  if (!res.ok) {
+    const errText = await res.text().catch(() => "");
+    throw new Error(`Upload failed: HTTP ${res.status} — ${errText.slice(0, 200)}`);
+  }
+
+  const data = (await res.json()) as { stored: number; chunks: number; source_hash: string };
+  return { stored: data.stored || 0, chunks: data.chunks || data.stored || 0 };
+}
+
+/**
+ * Delete all chunks with a given source_hash.
+ * Uses POST /v1/memory/source/delete (POST fallback for clients that can't send DELETE with body).
+ */
+export async function deleteBySourceHash(
+  endpoint: string,
+  memoryKey: string,
+  sourceHash: string,
+): Promise<number> {
+  const res = await fetch(`${endpoint}/v1/memory/source/delete`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${memoryKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ source_hash: sourceHash }),
+  });
+
+  if (!res.ok) {
+    const errText = await res.text().catch(() => "");
+    throw new Error(`Delete failed: HTTP ${res.status} — ${errText.slice(0, 200)}`);
+  }
+
+  const data = (await res.json()) as { deleted: number; source_hash: string };
+  return data.deleted || 0;
+}
+
+// ── Main Sync Flow ──
+
+/**
+ * Sync workspace files to MemoryRouter using source_hash tracking.
+ *
+ * Algorithm:
+ * 1. Load manifest (previous state)
+ * 2. Discover current workspace files, compute hashes
+ * 3. Diff current vs manifest:
+ *    - NEW: upload with source_hash
+ *    - CHANGED: delete old hash, upload new
+ *    - DELETED: delete from MR
+ *    - UNCHANGED: skip
+ * 4. Save updated manifest
+ */
+export async function syncWorkspaceFiles(options: SyncOptions): Promise<SyncResult> {
+  const { workspaceDir, endpoint, memoryKey, embeddings, logging } = options;
+  const log = (msg: string) => {
+    if (logging) console.log(msg);
+  };
+
+  const result: SyncResult = { uploaded: 0, deleted: 0, unchanged: 0, errors: [] };
+
+  // 1. Load manifest
+  const manifest = await loadManifest();
+  log(`memoryrouter: sync starting (${Object.keys(manifest.files).length} files in manifest)`);
+
+  // 2. Discover workspace files and compute hashes
+  const diskFiles = await discoverWorkspaceFiles(workspaceDir);
+  log(`memoryrouter: discovered ${diskFiles.length} workspace files`);
+
+  const diskState = new Map<string, { hash: string; size: number; content: string }>();
+
+  for (const relPath of diskFiles) {
+    const absPath = join(workspaceDir, relPath);
+    try {
+      const content = await readFile(absPath, "utf-8");
+      const trimmed = content.trim();
+      if (!trimmed) continue; // Skip empty files
+
+      const hash = computeSourceHash(content);
+      const size = Buffer.byteLength(content, "utf-8");
+      diskState.set(relPath, { hash, size, content });
+    } catch (err) {
+      result.errors.push(`Read error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  // 3a. Process disk files (new + changed)
+  for (const [relPath, { hash, size, content }] of diskState) {
+    const manifestEntry = manifest.files[relPath];
+
+    if (!manifestEntry) {
+      // NEW FILE — upload
+      try {
+        const { chunks } = await uploadSourceFile(endpoint, memoryKey, content, hash, embeddings);
+        manifest.files[relPath] = {
+          source_hash: hash,
+          size,
+          chunks,
+          uploaded_at: Date.now(),
+        };
+        result.uploaded++;
+        log(`memoryrouter: sync uploaded new file ${relPath} (${chunks} chunks)`);
+      } catch (err) {
+        result.errors.push(`Upload error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
+      }
+    } else if (manifestEntry.source_hash !== hash) {
+      // CHANGED FILE — delete old, upload new
+      try {
+        const deleted = await deleteBySourceHash(endpoint, memoryKey, manifestEntry.source_hash);
+        log(`memoryrouter: sync deleted old version of ${relPath} (${deleted} chunks)`);
+      } catch (err) {
+        // Log but continue — old chunks may have been deleted manually
+        log(`memoryrouter: sync delete warning for ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
+      }
+
+      try {
+        const { chunks } = await uploadSourceFile(endpoint, memoryKey, content, hash, embeddings);
+        manifest.files[relPath] = {
+          source_hash: hash,
+          size,
+          chunks,
+          uploaded_at: Date.now(),
+        };
+        result.uploaded++;
+        log(`memoryrouter: sync updated file ${relPath} (${chunks} new chunks)`);
+      } catch (err) {
+        result.errors.push(`Upload error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
+      }
+    } else {
+      // UNCHANGED — skip
+      result.unchanged++;
+    }
+  }
+
+  // 3b. Handle deleted files (in manifest but not on disk)
+  for (const relPath of Object.keys(manifest.files)) {
+    if (!diskState.has(relPath)) {
+      try {
+        const deleted = await deleteBySourceHash(endpoint, memoryKey, manifest.files[relPath].source_hash);
+        delete manifest.files[relPath];
+        result.deleted++;
+        log(`memoryrouter: sync deleted removed file ${relPath} (${deleted} chunks)`);
+      } catch (err) {
+        result.errors.push(`Delete error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+  }
+
+  // 4. Save updated manifest
+  await saveManifest(manifest);
+
+  log(
+    `memoryrouter: sync complete — ${result.uploaded} uploaded, ${result.deleted} deleted, ${result.unchanged} unchanged`,
+  );
+
+  return result;
+}
+
+// ── CLI Helper ──
+
+/**
+ * Run sync from CLI with console output.
+ */
+export async function runSyncCli(options: SyncOptions): Promise<void> {
+  const { workspaceDir, embeddings } = options;
+
+  const modelLabel = embeddings ? `(model: ${embeddings})` : "(model: bge)";
+  console.log(`Syncing workspace to MemoryRouter ${modelLabel}...`);
+  console.log(`  Workspace: ${workspaceDir}`);
+
+  try {
+    const result = await syncWorkspaceFiles({ ...options, logging: true });
+
+    console.log("\n────────────────────────────────");
+    console.log(`✅ Sync complete`);
+    console.log(`   Uploaded: ${result.uploaded}`);
+    console.log(`   Deleted:  ${result.deleted}`);
+    console.log(`   Unchanged: ${result.unchanged}`);
+
+    if (result.errors.length > 0) {
+      console.log(`\n⚠️ ${result.errors.length} errors:`);
+      for (const err of result.errors.slice(0, 5)) {
+        console.log(`   • ${err}`);
+      }
+      if (result.errors.length > 5) {
+        console.log(`   • ... and ${result.errors.length - 5} more`);
+      }
+    }
+  } catch (err) {
+    console.error(`\n❌ Sync failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+/**
+ * Show manifest status from CLI.
+ */
+export async function showManifestStatus(): Promise<void> {
+  const manifest = await loadManifest();
+  const fileCount = Object.keys(manifest.files).length;
+
+  console.log("Workspace Sync Manifest");
+  console.log("───────────────────────────────");
+  console.log(`  Path:       ${MANIFEST_PATH}`);
+  console.log(`  Version:    ${manifest.version}`);
+  console.log(`  Last sync:  ${manifest.lastSync ? new Date(manifest.lastSync).toLocaleString() : "never"}`);
+  console.log(`  Files:      ${fileCount}`);
+
+  if (fileCount > 0) {
+    console.log("\n  Tracked files:");
+    const files = Object.entries(manifest.files).sort((a, b) => a[0].localeCompare(b[0]));
+    for (const [path, entry] of files.slice(0, 20)) {
+      const date = new Date(entry.uploaded_at).toLocaleDateString();
+      console.log(`    • ${path} (${entry.chunks} chunks, ${date})`);
+    }
+    if (files.length > 20) {
+      console.log(`    ... and ${files.length - 20} more`);
+    }
+  }
+}