mr-memory 3.2.2 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/index.ts +36 -165
  2. package/package.json +2 -3
  3. package/upload.ts +13 -52
  4. package/sync.ts +0 -367
package/index.ts CHANGED
@@ -10,11 +10,9 @@
10
10
 
11
11
  import { readFile, readdir, stat, lstat } from "node:fs/promises";
12
12
  import { join, resolve, relative, isAbsolute, sep } from "node:path";
13
- import { homedir } from "node:os";
14
13
  import path from "node:path";
15
14
  import { spawn } from "node:child_process";
16
15
  import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
17
- import { syncWorkspaceFiles, runSyncCli, showManifestStatus } from "./sync.js";
18
16
 
19
17
  const DEFAULT_ENDPOINT = "https://api.memoryrouter.ai";
20
18
 
@@ -385,43 +383,12 @@ const memoryRouterPlugin = {
385
383
  const log = (msg: string) => { if (logging) api.logger.info?.(msg); };
386
384
 
387
385
  if (memoryKey) {
388
- const modelLabel = embeddings ? `, embeddings: ${embeddings}` : "";
389
- api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}..., mode: ${mode}${modelLabel})`);
386
+ api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
390
387
  } else {
391
388
  api.logger.info?.("memoryrouter: no key configured — run: openclaw mr <key>");
392
389
  api.logger.info?.("memoryrouter: get your free API key at https://memoryrouter.ai");
393
390
  }
394
391
 
395
- // ==================================================================
396
- // Workspace sync (debounced, fire-and-forget)
397
- // ==================================================================
398
-
399
- let lastSyncMs = 0;
400
- const SYNC_DEBOUNCE_MS = 60_000; // At most once per 60 seconds
401
-
402
- const triggerSync = (workspaceDir: string) => {
403
- if (!memoryKey) return;
404
- const now = Date.now();
405
- if (now - lastSyncMs < SYNC_DEBOUNCE_MS) return;
406
- lastSyncMs = now;
407
- syncWorkspaceFiles({ workspaceDir, endpoint, memoryKey, embeddings, logging })
408
- .then((result) => {
409
- if (result.uploaded > 0 || result.deleted > 0) {
410
- api.logger.info?.(
411
- `memoryrouter: sync complete — ${result.uploaded} uploaded, ${result.deleted} deleted, ${result.unchanged} unchanged`,
412
- );
413
- } else {
414
- log(`memoryrouter: sync complete — ${result.unchanged} files unchanged`);
415
- }
416
- if (result.errors.length > 0) {
417
- api.logger.warn?.(`memoryrouter: sync had ${result.errors.length} errors`);
418
- }
419
- })
420
- .catch((err) => {
421
- log(`memoryrouter: sync error — ${err instanceof Error ? err.message : String(err)}`);
422
- });
423
- };
424
-
425
392
  // ==================================================================
426
393
  // Core: before_agent_start — search memories, inject context
427
394
  // ==================================================================
@@ -437,7 +404,6 @@ const memoryRouterPlugin = {
437
404
  // When PR #24122 merges, OpenClaw will use the returned prependContext.
438
405
  // This gives forward compatibility — no plugin update needed.
439
406
  api.on("llm_input", async (event, ctx) => {
440
- log(`memoryrouter: llm_input fired (sessionKey=${ctx.sessionKey}, promptBuildFired=${promptBuildFiredThisRun})`);
441
407
  // Skip the first call — before_prompt_build already handled it
442
408
  // (before_prompt_build includes workspace+tools+skills for accurate billing)
443
409
  if (promptBuildFiredThisRun) {
@@ -446,6 +412,7 @@ const memoryRouterPlugin = {
446
412
  }
447
413
 
448
414
  try {
415
+ const startMs = Date.now();
449
416
  const prompt = event.prompt;
450
417
  if (prompt === lastPreparedPrompt && lastPreparedPrompt !== "") return;
451
418
  lastPreparedPrompt = prompt;
@@ -511,9 +478,8 @@ const memoryRouterPlugin = {
511
478
  };
512
479
 
513
480
  if (data.context) {
514
- log(
515
- `memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens)`,
516
- );
481
+ const elapsed = Date.now() - startMs;
482
+ api.logger.info?.(`memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens, ${elapsed}ms) [llm_input]`);
517
483
  const wrapped = wrapForInjection(data.context);
518
484
  return { appendSystemContext: wrapped };
519
485
  }
@@ -525,12 +491,8 @@ const memoryRouterPlugin = {
525
491
  // ── before_prompt_build: fires once per run (primary, includes full billing context)
526
492
  api.on("before_prompt_build", async (event, ctx) => {
527
493
  promptBuildFiredThisRun = true;
528
- log(`memoryrouter: before_prompt_build fired (sessionKey=${ctx.sessionKey}, promptLen=${event.prompt?.length})`);
529
-
530
- // Trigger workspace sync (fire-and-forget, debounced — catches new files mid-run)
531
- if (ctx.workspaceDir) triggerSync(ctx.workspaceDir);
532
-
533
494
  try {
495
+ const startMs = Date.now();
534
496
  const prompt = event.prompt;
535
497
 
536
498
  // Deduplicate — if we already prepared this exact prompt, skip
@@ -625,16 +587,11 @@ const memoryRouterPlugin = {
625
587
  memory_tokens?: number;
626
588
  };
627
589
 
628
- log(`memoryrouter: prepare response — memories_found=${data.memories_found || 0}, context_length=${data.context?.length || 0}, tokens=${data.memory_tokens || 0}, embeddings=${embeddings || 'bge'}`);
629
-
630
590
  if (data.context) {
631
- log(
632
- `memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens)`,
633
- );
591
+ const elapsed = Date.now() - startMs;
592
+ api.logger.info?.(`memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens, ${elapsed}ms)`);
634
593
  const wrapped = wrapForInjection(data.context);
635
594
  return { appendSystemContext: wrapped };
636
- } else {
637
- log(`memoryrouter: prepare returned no context (memories_found=${data.memories_found || 0})`);
638
595
  }
639
596
  } catch (err) {
640
597
  log(
@@ -770,7 +727,7 @@ const memoryRouterPlugin = {
770
727
  execute: async (_toolCallId: string, params: Record<string, unknown>) => {
771
728
  const query = typeof params.query === "string" ? params.query.trim() : "";
772
729
  if (!query) return jsonToolResult({ results: [], error: "query required" });
773
- const limit = typeof params.maxResults === "number" ? params.maxResults : 10;
730
+ const limit = typeof params.maxResults === "number" ? params.maxResults : 50;
774
731
  try {
775
732
  const res = await fetch(`${endpoint}/v1/memory/search`, {
776
733
  method: "POST",
@@ -1011,71 +968,17 @@ const memoryRouterPlugin = {
1011
968
  }
1012
969
  });
1013
970
 
1014
- // Mode commands
1015
- for (const [modeName, modeDesc] of [
1016
- ["relay", "Relay mode — hooks only, works on stock OpenClaw [default]"],
1017
- ["proxy", "Proxy mode — memory on every LLM call (requires registerStreamFnWrapper)"],
1018
- ] as const) {
1019
- mr.command(modeName)
1020
- .description(modeDesc)
1021
- .action(async () => {
1022
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1023
- try {
1024
- await setPluginConfig(api, { key: memoryKey, endpoint: cfg?.endpoint, density, mode: modeName });
1025
- console.log(`✓ Mode set to ${modeName}`);
1026
- } catch (err) {
1027
- console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
1028
- }
1029
- });
1030
- }
1031
971
 
1032
- // Embedding model command
1033
- mr.command("embeddings")
1034
- .description("Get or set embedding model (bge, qwen)")
1035
- .argument("[model]", "Embedding model: bge (default, 1024d) or qwen (4096d)")
1036
- .action(async (model: string | undefined) => {
1037
- if (!model) {
1038
- console.log(`Current embedding model: ${embeddings || "bge (default)"}`);
1039
- console.log(`\nAvailable models:`);
1040
- console.log(` bge BGE-M3 1024 dims (Cloudflare Workers AI, ~18ms, free)`);
1041
- console.log(` qwen Qwen3-8B 4096 dims (HuggingFace, ~69ms, $0.80/hr)`);
1042
- console.log(`\nUsage: openclaw mr embeddings <model>`);
1043
- return;
1044
- }
1045
- const normalized = model.toLowerCase().trim();
1046
- const valid = ["bge", "bge-m3", "qwen", "qwen3", "qwen3-8b", "default"];
1047
- if (!valid.includes(normalized)) {
1048
- console.error(`Unknown model: "${model}". Valid options: bge, qwen`);
1049
- return;
1050
- }
1051
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1052
- // "bge" / "default" → clear embeddings (use server default)
1053
- const newEmbeddings = (normalized === "bge" || normalized === "bge-m3" || normalized === "default")
1054
- ? undefined
1055
- : normalized.startsWith("qwen") ? "qwen" : normalized;
1056
- try {
1057
- await setPluginConfig(api, {
1058
- key: memoryKey,
1059
- endpoint: cfg?.endpoint,
1060
- density,
1061
- mode,
1062
- logging,
1063
- ...(newEmbeddings && { embeddings: newEmbeddings }),
1064
- });
1065
- const label = newEmbeddings || "bge (default)";
1066
- console.log(`✓ Embedding model set to ${label}`);
1067
- console.log(` All future prepare/ingest/search will use ${label}`);
1068
- console.log(`\n Restart gateway to apply: openclaw gateway restart`);
1069
- } catch (err) {
1070
- console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
1071
- }
1072
- });
972
+
973
+
1073
974
 
1074
975
  mr.command("status")
1075
976
  .description("Show MemoryRouter vault stats")
1076
977
  .option("--json", "JSON output")
1077
- .action(async (opts) => {
1078
- if (!memoryKey) {
978
+ .option("--key <key>", "Override memory key (for subagent vaults)")
979
+ .action(async (opts: { json?: boolean; key?: string }) => {
980
+ const effectiveKey = opts.key || memoryKey;
981
+ if (!effectiveKey) {
1079
982
  if (opts.json) {
1080
983
  console.log(JSON.stringify({ enabled: false, key: null }, null, 2));
1081
984
  } else {
@@ -1092,19 +995,17 @@ const memoryRouterPlugin = {
1092
995
  ? `${endpoint}/v1/memory/stats?embeddings=${encodeURIComponent(embeddings)}`
1093
996
  : `${endpoint}/v1/memory/stats`;
1094
997
  const res = await fetch(statsUrl, {
1095
- headers: { Authorization: `Bearer ${memoryKey}` },
998
+ headers: { Authorization: `Bearer ${effectiveKey}` },
1096
999
  });
1097
1000
  const data = (await res.json()) as { totalVectors?: number; totalTokens?: number };
1098
1001
  if (opts.json) {
1099
- console.log(JSON.stringify({ enabled: true, key: memoryKey, density, mode, embeddings: embeddings || "bge", stats: data }, null, 2));
1002
+ console.log(JSON.stringify({ enabled: true, key: effectiveKey, density, stats: data }, null, 2));
1100
1003
  } else {
1101
1004
  console.log("MemoryRouter Status");
1102
1005
  console.log("───────────────────────────");
1103
1006
  console.log(`Enabled: ✓ Yes`);
1104
- console.log(`Key: ${memoryKey.slice(0, 6)}...${memoryKey.slice(-3)}`);
1105
- console.log(`Mode: ${mode}`);
1007
+ console.log(`Key: ${effectiveKey.slice(0, 6)}...${effectiveKey.slice(-3)}`);
1106
1008
  console.log(`Density: ${density}`);
1107
- console.log(`Embeddings: ${embeddings || "bge (default)"}`);
1108
1009
  console.log(`Endpoint: ${endpoint}`);
1109
1010
  console.log(`Memories: ${data.totalVectors ?? 0}`);
1110
1011
  console.log(`Tokens: ${data.totalTokens ?? 0}`);
@@ -1119,8 +1020,10 @@ const memoryRouterPlugin = {
1119
1020
  .argument("[path]", "Specific file or directory to upload")
1120
1021
  .option("--workspace <dir>", "Workspace directory")
1121
1022
  .option("--brain <dir>", "State directory with sessions")
1122
- .action(async (targetPath: string | undefined, opts: { workspace?: string; brain?: string }) => {
1123
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1023
+ .option("--key <key>", "Override memory key (for subagent vaults)")
1024
+ .action(async (targetPath: string | undefined, opts: { workspace?: string; brain?: string; key?: string }) => {
1025
+ const effectiveKey = opts.key || memoryKey;
1026
+ if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
1124
1027
  const os = await import("node:os");
1125
1028
  const path = await import("node:path");
1126
1029
  const stateDir = opts.brain ? path.resolve(opts.brain) : path.join(os.homedir(), ".openclaw");
@@ -1131,22 +1034,24 @@ const memoryRouterPlugin = {
1131
1034
  ? path.resolve(configWorkspace.replace(/^~/, os.homedir()))
1132
1035
  : path.join(os.homedir(), ".openclaw", "workspace");
1133
1036
  const { runUpload } = await import("./upload.js");
1134
- await runUpload({ memoryKey, endpoint, targetPath, stateDir, workspacePath, hasWorkspaceFlag: !!opts.workspace, hasBrainFlag: !!opts.brain, embeddings });
1037
+ await runUpload({ memoryKey: effectiveKey, endpoint, targetPath, stateDir, workspacePath, hasWorkspaceFlag: !!opts.workspace, hasBrainFlag: !!opts.brain, embeddings });
1135
1038
  });
1136
1039
 
1137
1040
  mr.command("search")
1138
1041
  .description("Search memories in vault")
1139
1042
  .argument("<query>", "Search query")
1140
- .option("-n, --limit <number>", "Number of results", "5")
1043
+ .option("-n, --limit <number>", "Number of results", "50")
1141
1044
  .option("--json", "Output raw JSON response")
1142
- .action(async (query: string, opts: { limit: string; json?: boolean }) => {
1143
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1144
- const limit = parseInt(opts.limit, 10) || 5;
1045
+ .option("--key <key>", "Override memory key (for subagent vaults)")
1046
+ .action(async (query: string, opts: { limit: string; json?: boolean; key?: string }) => {
1047
+ const effectiveKey = opts.key || memoryKey;
1048
+ if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
1049
+ const limit = parseInt(opts.limit, 10) || 50;
1145
1050
  try {
1146
1051
  const res = await fetch(`${endpoint}/v1/memory/search`, {
1147
1052
  method: "POST",
1148
1053
  headers: {
1149
- Authorization: `Bearer ${memoryKey}`,
1054
+ Authorization: `Bearer ${effectiveKey}`,
1150
1055
  "Content-Type": "application/json",
1151
1056
  ...(embeddings && { "X-Embedding-Model": embeddings }),
1152
1057
  },
@@ -1211,15 +1116,17 @@ const memoryRouterPlugin = {
1211
1116
 
1212
1117
  mr.command("delete")
1213
1118
  .description("Clear all memories from vault")
1214
- .action(async () => {
1215
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1119
+ .option("--key <key>", "Override memory key (for subagent vaults)")
1120
+ .action(async (opts: { key?: string }) => {
1121
+ const effectiveKey = opts.key || memoryKey;
1122
+ if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
1216
1123
  try {
1217
1124
  const deleteUrl = embeddings
1218
1125
  ? `${endpoint}/v1/memory?embeddings=${encodeURIComponent(embeddings)}`
1219
1126
  : `${endpoint}/v1/memory`;
1220
1127
  const res = await fetch(deleteUrl, {
1221
1128
  method: "DELETE",
1222
- headers: { Authorization: `Bearer ${memoryKey}` },
1129
+ headers: { Authorization: `Bearer ${effectiveKey}` },
1223
1130
  });
1224
1131
  const data = (await res.json()) as { message?: string };
1225
1132
  const modelLabel = embeddings ? ` (${embeddings})` : "";
@@ -1228,33 +1135,6 @@ const memoryRouterPlugin = {
1228
1135
  console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
1229
1136
  }
1230
1137
  });
1231
-
1232
- // ── Workspace Sync ──
1233
- mr.command("sync")
1234
- .description("Sync workspace files to vault using source_hash tracking")
1235
- .option("--workspace <dir>", "Workspace directory")
1236
- .option("--status", "Show manifest status instead of syncing")
1237
- .action(async (opts: { workspace?: string; status?: boolean }) => {
1238
- if (opts.status) {
1239
- await showManifestStatus();
1240
- return;
1241
- }
1242
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1243
- const os = await import("node:os");
1244
- const configWorkspace = (api.config as any).workspace || (api.config as any).agents?.defaults?.workspace;
1245
- const workspaceDir = opts.workspace
1246
- ? path.resolve(opts.workspace)
1247
- : configWorkspace
1248
- ? path.resolve(configWorkspace.replace(/^~/, os.homedir()))
1249
- : path.join(os.homedir(), ".openclaw", "workspace");
1250
- await runSyncCli({
1251
- workspaceDir,
1252
- endpoint,
1253
- memoryKey,
1254
- embeddings,
1255
- logging: true,
1256
- });
1257
- });
1258
1138
  },
1259
1139
  { commands: ["mr"] },
1260
1140
  );
@@ -1265,17 +1145,8 @@ const memoryRouterPlugin = {
1265
1145
 
1266
1146
  api.registerService({
1267
1147
  id: "mr-memory",
1268
- start: async () => {
1269
- if (memoryKey) {
1270
- api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
1271
-
1272
- // ── Auto-sync workspace files (non-blocking) ──
1273
- const configWorkspace = (api.config as any).workspace || (api.config as any).agents?.defaults?.workspace;
1274
- const workspaceDir = configWorkspace
1275
- ? path.resolve(configWorkspace.replace(/^~/, homedir()))
1276
- : path.join(homedir(), ".openclaw", "workspace");
1277
- triggerSync(workspaceDir);
1278
- }
1148
+ start: () => {
1149
+ if (memoryKey) api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
1279
1150
  },
1280
1151
  stop: () => {
1281
1152
  api.logger.info?.("memoryrouter: stopped");
package/package.json CHANGED
@@ -1,14 +1,13 @@
1
1
  {
2
2
  "name": "mr-memory",
3
- "version": "3.2.2",
3
+ "version": "3.3.0",
4
4
  "description": "MemoryRouter persistent memory plugin for OpenClaw — your AI remembers every conversation",
5
5
  "type": "module",
6
6
  "files": [
7
7
  "README.md",
8
8
  "index.ts",
9
9
  "openclaw.plugin.json",
10
- "upload.ts",
11
- "sync.ts"
10
+ "upload.ts"
12
11
  ],
13
12
  "keywords": [
14
13
  "openclaw",
package/upload.ts CHANGED
@@ -370,11 +370,10 @@ export async function runUpload(params: {
370
370
 
371
371
  const workspacePath = params.workspacePath ?? process.cwd();
372
372
 
373
- // ── Target path: specific file/dir upload (old behavior, no sync) ──
373
+ let files: string[];
374
374
  if (targetPath) {
375
375
  const resolved = path.resolve(targetPath);
376
376
  const stat = await fs.stat(resolved);
377
- let files: string[];
378
377
  if (stat.isDirectory()) {
379
378
  const allDirFiles = await fs.readdir(resolved, { recursive: true });
380
379
  files = (allDirFiles as string[])
@@ -386,58 +385,19 @@ export async function runUpload(params: {
386
385
  } else {
387
386
  files = [resolved];
388
387
  }
389
- await uploadFilesLegacy(files, memoryKey, endpoint, embeddings);
390
- return;
391
- }
392
-
393
- // ── Determine what to upload ──
394
- const doWorkspace = !params.hasBrainFlag || params.hasWorkspaceFlag; // default: yes
395
- const doSessions = !params.hasWorkspaceFlag || params.hasBrainFlag; // default: yes
396
-
397
- // ── Workspace files: use source_hash sync (smart, tracks changes) ──
398
- if (doWorkspace) {
399
- console.log("📁 Syncing workspace files...");
400
- const { syncWorkspaceFiles } = await import("./sync.js");
401
- const syncResult = await syncWorkspaceFiles({
402
- workspaceDir: workspacePath,
403
- endpoint,
404
- memoryKey,
405
- embeddings,
406
- logging: true,
407
- });
408
- console.log(`\n Workspace: ${syncResult.uploaded} uploaded, ${syncResult.deleted} deleted, ${syncResult.unchanged} unchanged`);
409
- if (syncResult.errors.length > 0) {
410
- console.log(` ⚠️ ${syncResult.errors.length} errors`);
411
- for (const err of syncResult.errors.slice(0, 3)) {
412
- console.log(` • ${err}`);
413
- }
414
- }
388
+ } else if (params.hasBrainFlag && !params.hasWorkspaceFlag) {
389
+ // --brain only: upload sessions from brain path
390
+ files = await discoverBrainFiles(stateDir);
391
+ } else if (params.hasWorkspaceFlag && !params.hasBrainFlag) {
392
+ // --workspace only: upload workspace files only
393
+ files = await discoverWorkspaceFiles(workspacePath);
394
+ } else {
395
+ // No flags or both flags: upload both workspace + sessions
396
+ const wsFiles = await discoverWorkspaceFiles(workspacePath);
397
+ const brainFiles = await discoverBrainFiles(stateDir);
398
+ files = [...wsFiles, ...brainFiles];
415
399
  }
416
400
 
417
- // ── Session transcripts: batch upload (append-only, no sync needed) ──
418
- if (doSessions) {
419
- console.log("\n📝 Uploading session transcripts...");
420
- const sessionFiles = await discoverBrainFiles(stateDir);
421
- if (sessionFiles.length === 0) {
422
- console.log(" No session files found.");
423
- } else {
424
- await uploadFilesLegacy(sessionFiles, memoryKey, endpoint, embeddings);
425
- }
426
- }
427
- }
428
-
429
- /**
430
- * Legacy batch upload for session transcripts and specific-path uploads.
431
- * Sessions are append-only, so they don't need source_hash tracking.
432
- */
433
- async function uploadFilesLegacy(
434
- files: string[],
435
- memoryKey: string,
436
- endpoint: string,
437
- embeddings?: string,
438
- ): Promise<void> {
439
- const uploadUrl = `${endpoint}/v1/memory/upload`;
440
-
441
401
  if (files.length === 0) {
442
402
  console.log("No files found to upload.");
443
403
  return;
@@ -536,6 +496,7 @@ async function uploadFilesLegacy(
536
496
  if (batchFailed > 0) {
537
497
  const errHint = result.errors?.[0] ? ` (${result.errors[0].slice(0, 120)})` : "";
538
498
  console.log(`⚠ ${batchStored} stored, ${batchFailed} skipped${errHint}`);
499
+ // Show detailed error diagnostics (up to 5 per batch)
539
500
  if (result.errors && result.errors.length > 1) {
540
501
  for (const err of result.errors.slice(0, 5)) {
541
502
  console.log(` → ${err.slice(0, 200)}`);
package/sync.ts DELETED
@@ -1,367 +0,0 @@
1
- /**
2
- * MemoryRouter Workspace Sync — source_hash-based file tracking.
3
- *
4
- * Tracks workspace files via SHA-256 content hashes. When files change,
5
- * old chunks are deleted and new content is uploaded with the new hash.
6
- * This enables clean replacement without stale chunk accumulation.
7
- *
8
- * IMPORTANT: source_hash is ONLY for workspace files, NEVER for sessions.
9
- */
10
-
11
- import { readFile, writeFile, readdir, lstat, mkdir } from "node:fs/promises";
12
- import { createHash } from "node:crypto";
13
- import { join, relative, sep } from "node:path";
14
- import { homedir } from "node:os";
15
-
16
- // ── Types ──
17
-
18
- export interface ManifestFileEntry {
19
- source_hash: string;
20
- size: number;
21
- chunks: number;
22
- uploaded_at: number;
23
- }
24
-
25
- export interface Manifest {
26
- version: number;
27
- lastSync: number;
28
- files: Record<string, ManifestFileEntry>;
29
- }
30
-
31
- export interface SyncResult {
32
- uploaded: number;
33
- deleted: number;
34
- unchanged: number;
35
- errors: string[];
36
- }
37
-
38
- export interface SyncOptions {
39
- workspaceDir: string;
40
- endpoint: string;
41
- memoryKey: string;
42
- embeddings?: string;
43
- logging?: boolean;
44
- }
45
-
46
- // ── Constants ──
47
-
48
- const MANIFEST_PATH = join(homedir(), ".openclaw", "mr-memory-manifest.json");
49
- const MAX_FILE_SIZE = 1_000_000; // 1MB limit
50
- const TEXT_EXTENSIONS = new Set(["md", "txt"]);
51
-
52
- // ── Manifest I/O ──
53
-
54
- export async function loadManifest(): Promise<Manifest> {
55
- try {
56
- const raw = await readFile(MANIFEST_PATH, "utf-8");
57
- return JSON.parse(raw) as Manifest;
58
- } catch {
59
- // First run or corrupted — create empty manifest
60
- return { version: 1, lastSync: 0, files: {} };
61
- }
62
- }
63
-
64
- export async function saveManifest(manifest: Manifest): Promise<void> {
65
- manifest.lastSync = Date.now();
66
- // Ensure directory exists
67
- const dir = join(homedir(), ".openclaw");
68
- await mkdir(dir, { recursive: true });
69
- await writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
70
- }
71
-
72
- // ── Hash Computation ──
73
-
74
- export function computeSourceHash(content: string): string {
75
- const hash = createHash("sha256").update(content).digest("hex");
76
- return hash.substring(0, 16); // First 16 hex chars, matching MR API format
77
- }
78
-
79
- // ── File Discovery ──
80
-
81
- /**
82
- * Discover ALL workspace files to sync:
83
- * - All .md and .txt files in workspace root
84
- * - All files recursively under memory/ directory
85
- * - Excludes: hidden files/dirs, node_modules/, files > 1MB, binary files
86
- */
87
- export async function discoverWorkspaceFiles(workspaceDir: string): Promise<string[]> {
88
- const files: string[] = [];
89
-
90
- async function walk(dir: string) {
91
- let entries;
92
- try {
93
- entries = await readdir(dir, { withFileTypes: true });
94
- } catch {
95
- return; // Directory doesn't exist or not readable
96
- }
97
-
98
- for (const entry of entries) {
99
- // Skip hidden dirs/files and node_modules
100
- if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
101
-
102
- const fullPath = join(dir, entry.name);
103
-
104
- if (entry.isDirectory()) {
105
- await walk(fullPath);
106
- } else if (entry.isFile()) {
107
- // Only text files we want to index
108
- const ext = entry.name.toLowerCase().split(".").pop() || "";
109
- if (!TEXT_EXTENSIONS.has(ext)) continue;
110
-
111
- // Check file size
112
- try {
113
- const stat = await lstat(fullPath);
114
- if (stat.size > MAX_FILE_SIZE) continue;
115
- if (stat.size === 0) continue; // Skip empty files
116
- files.push(relative(workspaceDir, fullPath));
117
- } catch {
118
- continue; // Skip unreadable files
119
- }
120
- }
121
- }
122
- }
123
-
124
- await walk(workspaceDir);
125
- return files.sort();
126
- }
127
-
128
- // ── API Helpers ──
129
-
130
- /**
131
- * Upload file content with source_hash using isolated chunking.
132
- * Uses POST /v1/memory/upload-source endpoint.
133
- */
134
- export async function uploadSourceFile(
135
- endpoint: string,
136
- memoryKey: string,
137
- content: string,
138
- sourceHash: string,
139
- embeddings?: string,
140
- ): Promise<{ stored: number; chunks: number }> {
141
- const res = await fetch(`${endpoint}/v1/memory/upload-source`, {
142
- method: "POST",
143
- headers: {
144
- Authorization: `Bearer ${memoryKey}`,
145
- "Content-Type": "application/json",
146
- ...(embeddings && { "X-Embedding-Model": embeddings }),
147
- },
148
- body: JSON.stringify({ content, source_hash: sourceHash }),
149
- });
150
-
151
- if (!res.ok) {
152
- const errText = await res.text().catch(() => "");
153
- throw new Error(`Upload failed: HTTP ${res.status} — ${errText.slice(0, 200)}`);
154
- }
155
-
156
- const data = (await res.json()) as { stored: number; chunks: number; source_hash: string };
157
- return { stored: data.stored || 0, chunks: data.chunks || data.stored || 0 };
158
- }
159
-
160
- /**
161
- * Delete all chunks with a given source_hash.
162
- * Uses POST /v1/memory/source/delete (POST fallback for clients that can't send DELETE with body).
163
- */
164
- export async function deleteBySourceHash(
165
- endpoint: string,
166
- memoryKey: string,
167
- sourceHash: string,
168
- ): Promise<number> {
169
- const res = await fetch(`${endpoint}/v1/memory/source/delete`, {
170
- method: "POST",
171
- headers: {
172
- Authorization: `Bearer ${memoryKey}`,
173
- "Content-Type": "application/json",
174
- },
175
- body: JSON.stringify({ source_hash: sourceHash }),
176
- });
177
-
178
- if (!res.ok) {
179
- const errText = await res.text().catch(() => "");
180
- throw new Error(`Delete failed: HTTP ${res.status} — ${errText.slice(0, 200)}`);
181
- }
182
-
183
- const data = (await res.json()) as { deleted: number; source_hash: string };
184
- return data.deleted || 0;
185
- }
186
-
187
- // ── Main Sync Flow ──
188
-
189
- /**
190
- * Sync workspace files to MemoryRouter using source_hash tracking.
191
- *
192
- * Algorithm:
193
- * 1. Load manifest (previous state)
194
- * 2. Discover current workspace files, compute hashes
195
- * 3. Diff current vs manifest:
196
- * - NEW: upload with source_hash
197
- * - CHANGED: delete old hash, upload new
198
- * - DELETED: delete from MR
199
- * - UNCHANGED: skip
200
- * 4. Save updated manifest
201
- */
202
- export async function syncWorkspaceFiles(options: SyncOptions): Promise<SyncResult> {
203
- const { workspaceDir, endpoint, memoryKey, embeddings, logging } = options;
204
- const log = (msg: string) => {
205
- if (logging) console.log(msg);
206
- };
207
-
208
- const result: SyncResult = { uploaded: 0, deleted: 0, unchanged: 0, errors: [] };
209
-
210
- // 1. Load manifest
211
- const manifest = await loadManifest();
212
- log(`memoryrouter: sync starting (${Object.keys(manifest.files).length} files in manifest)`);
213
-
214
- // 2. Discover workspace files and compute hashes
215
- const diskFiles = await discoverWorkspaceFiles(workspaceDir);
216
- log(`memoryrouter: discovered ${diskFiles.length} workspace files`);
217
-
218
- const diskState = new Map<string, { hash: string; size: number; content: string }>();
219
-
220
- for (const relPath of diskFiles) {
221
- const absPath = join(workspaceDir, relPath);
222
- try {
223
- const content = await readFile(absPath, "utf-8");
224
- const trimmed = content.trim();
225
- if (!trimmed) continue; // Skip empty files
226
-
227
- const hash = computeSourceHash(content);
228
- const size = Buffer.byteLength(content, "utf-8");
229
- diskState.set(relPath, { hash, size, content });
230
- } catch (err) {
231
- result.errors.push(`Read error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
232
- }
233
- }
234
-
235
- // 3a. Process disk files (new + changed)
236
- for (const [relPath, { hash, size, content }] of diskState) {
237
- const manifestEntry = manifest.files[relPath];
238
-
239
- if (!manifestEntry) {
240
- // NEW FILE — upload
241
- try {
242
- const { chunks } = await uploadSourceFile(endpoint, memoryKey, content, hash, embeddings);
243
- manifest.files[relPath] = {
244
- source_hash: hash,
245
- size,
246
- chunks,
247
- uploaded_at: Date.now(),
248
- };
249
- result.uploaded++;
250
- log(`memoryrouter: sync uploaded new file ${relPath} (${chunks} chunks)`);
251
- } catch (err) {
252
- result.errors.push(`Upload error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
253
- }
254
- } else if (manifestEntry.source_hash !== hash) {
255
- // CHANGED FILE — delete old, upload new
256
- try {
257
- const deleted = await deleteBySourceHash(endpoint, memoryKey, manifestEntry.source_hash);
258
- log(`memoryrouter: sync deleted old version of ${relPath} (${deleted} chunks)`);
259
- } catch (err) {
260
- // Log but continue — old chunks may have been deleted manually
261
- log(`memoryrouter: sync delete warning for ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
262
- }
263
-
264
- try {
265
- const { chunks } = await uploadSourceFile(endpoint, memoryKey, content, hash, embeddings);
266
- manifest.files[relPath] = {
267
- source_hash: hash,
268
- size,
269
- chunks,
270
- uploaded_at: Date.now(),
271
- };
272
- result.uploaded++;
273
- log(`memoryrouter: sync updated file ${relPath} (${chunks} new chunks)`);
274
- } catch (err) {
275
- result.errors.push(`Upload error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
276
- }
277
- } else {
278
- // UNCHANGED — skip
279
- result.unchanged++;
280
- }
281
- }
282
-
283
- // 3b. Handle deleted files (in manifest but not on disk)
284
- for (const relPath of Object.keys(manifest.files)) {
285
- if (!diskState.has(relPath)) {
286
- try {
287
- const deleted = await deleteBySourceHash(endpoint, memoryKey, manifest.files[relPath].source_hash);
288
- delete manifest.files[relPath];
289
- result.deleted++;
290
- log(`memoryrouter: sync deleted removed file ${relPath} (${deleted} chunks)`);
291
- } catch (err) {
292
- result.errors.push(`Delete error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
293
- }
294
- }
295
- }
296
-
297
- // 4. Save updated manifest
298
- await saveManifest(manifest);
299
-
300
- log(
301
- `memoryrouter: sync complete — ${result.uploaded} uploaded, ${result.deleted} deleted, ${result.unchanged} unchanged`,
302
- );
303
-
304
- return result;
305
- }
306
-
307
- // ── CLI Helper ──
308
-
309
- /**
310
- * Run sync from CLI with console output.
311
- */
312
- export async function runSyncCli(options: SyncOptions): Promise<void> {
313
- const { workspaceDir, embeddings } = options;
314
-
315
- const modelLabel = embeddings ? `(model: ${embeddings})` : "(model: bge)";
316
- console.log(`Syncing workspace to MemoryRouter ${modelLabel}...`);
317
- console.log(` Workspace: ${workspaceDir}`);
318
-
319
- try {
320
- const result = await syncWorkspaceFiles({ ...options, logging: true });
321
-
322
- console.log("\n────────────────────────────────");
323
- console.log(`✅ Sync complete`);
324
- console.log(` Uploaded: ${result.uploaded}`);
325
- console.log(` Deleted: ${result.deleted}`);
326
- console.log(` Unchanged: ${result.unchanged}`);
327
-
328
- if (result.errors.length > 0) {
329
- console.log(`\n⚠️ ${result.errors.length} errors:`);
330
- for (const err of result.errors.slice(0, 5)) {
331
- console.log(` • ${err}`);
332
- }
333
- if (result.errors.length > 5) {
334
- console.log(` • ... and ${result.errors.length - 5} more`);
335
- }
336
- }
337
- } catch (err) {
338
- console.error(`\n❌ Sync failed: ${err instanceof Error ? err.message : String(err)}`);
339
- }
340
- }
341
-
342
- /**
343
- * Show manifest status from CLI.
344
- */
345
- export async function showManifestStatus(): Promise<void> {
346
- const manifest = await loadManifest();
347
- const fileCount = Object.keys(manifest.files).length;
348
-
349
- console.log("Workspace Sync Manifest");
350
- console.log("───────────────────────────────");
351
- console.log(` Path: ${MANIFEST_PATH}`);
352
- console.log(` Version: ${manifest.version}`);
353
- console.log(` Last sync: ${manifest.lastSync ? new Date(manifest.lastSync).toLocaleString() : "never"}`);
354
- console.log(` Files: ${fileCount}`);
355
-
356
- if (fileCount > 0) {
357
- console.log("\n Tracked files:");
358
- const files = Object.entries(manifest.files).sort((a, b) => a[0].localeCompare(b[0]));
359
- for (const [path, entry] of files.slice(0, 20)) {
360
- const date = new Date(entry.uploaded_at).toLocaleDateString();
361
- console.log(` • ${path} (${entry.chunks} chunks, ${date})`);
362
- }
363
- if (files.length > 20) {
364
- console.log(` ... and ${files.length - 20} more`);
365
- }
366
- }
367
- }