mr-memory 3.2.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/index.ts +36 -155
  2. package/package.json +2 -3
  3. package/upload.ts +13 -52
  4. package/sync.ts +0 -367
package/index.ts CHANGED
@@ -10,11 +10,9 @@
10
10
 
11
11
  import { readFile, readdir, stat, lstat } from "node:fs/promises";
12
12
  import { join, resolve, relative, isAbsolute, sep } from "node:path";
13
- import { homedir } from "node:os";
14
13
  import path from "node:path";
15
14
  import { spawn } from "node:child_process";
16
15
  import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
17
- import { syncWorkspaceFiles, runSyncCli, showManifestStatus } from "./sync.js";
18
16
 
19
17
  const DEFAULT_ENDPOINT = "https://api.memoryrouter.ai";
20
18
 
@@ -385,8 +383,7 @@ const memoryRouterPlugin = {
385
383
  const log = (msg: string) => { if (logging) api.logger.info?.(msg); };
386
384
 
387
385
  if (memoryKey) {
388
- const modelLabel = embeddings ? `, embeddings: ${embeddings}` : "";
389
- api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}..., mode: ${mode}${modelLabel})`);
386
+ api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
390
387
  } else {
391
388
  api.logger.info?.("memoryrouter: no key configured — run: openclaw mr <key>");
392
389
  api.logger.info?.("memoryrouter: get your free API key at https://memoryrouter.ai");
@@ -407,7 +404,6 @@ const memoryRouterPlugin = {
407
404
  // When PR #24122 merges, OpenClaw will use the returned prependContext.
408
405
  // This gives forward compatibility — no plugin update needed.
409
406
  api.on("llm_input", async (event, ctx) => {
410
- log(`memoryrouter: llm_input fired (sessionKey=${ctx.sessionKey}, promptBuildFired=${promptBuildFiredThisRun})`);
411
407
  // Skip the first call — before_prompt_build already handled it
412
408
  // (before_prompt_build includes workspace+tools+skills for accurate billing)
413
409
  if (promptBuildFiredThisRun) {
@@ -416,6 +412,7 @@ const memoryRouterPlugin = {
416
412
  }
417
413
 
418
414
  try {
415
+ const startMs = Date.now();
419
416
  const prompt = event.prompt;
420
417
  if (prompt === lastPreparedPrompt && lastPreparedPrompt !== "") return;
421
418
  lastPreparedPrompt = prompt;
@@ -481,9 +478,8 @@ const memoryRouterPlugin = {
481
478
  };
482
479
 
483
480
  if (data.context) {
484
- log(
485
- `memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens)`,
486
- );
481
+ const elapsed = Date.now() - startMs;
482
+ api.logger.info?.(`memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens, ${elapsed}ms) [llm_input]`);
487
483
  const wrapped = wrapForInjection(data.context);
488
484
  return { appendSystemContext: wrapped };
489
485
  }
@@ -495,8 +491,8 @@ const memoryRouterPlugin = {
495
491
  // ── before_prompt_build: fires once per run (primary, includes full billing context)
496
492
  api.on("before_prompt_build", async (event, ctx) => {
497
493
  promptBuildFiredThisRun = true;
498
- log(`memoryrouter: before_prompt_build fired (sessionKey=${ctx.sessionKey}, promptLen=${event.prompt?.length})`);
499
494
  try {
495
+ const startMs = Date.now();
500
496
  const prompt = event.prompt;
501
497
 
502
498
  // Deduplicate — if we already prepared this exact prompt, skip
@@ -591,16 +587,11 @@ const memoryRouterPlugin = {
591
587
  memory_tokens?: number;
592
588
  };
593
589
 
594
- log(`memoryrouter: prepare response — memories_found=${data.memories_found || 0}, context_length=${data.context?.length || 0}, tokens=${data.memory_tokens || 0}, embeddings=${embeddings || 'bge'}`);
595
-
596
590
  if (data.context) {
597
- log(
598
- `memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens)`,
599
- );
591
+ const elapsed = Date.now() - startMs;
592
+ api.logger.info?.(`memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens, ${elapsed}ms)`);
600
593
  const wrapped = wrapForInjection(data.context);
601
594
  return { appendSystemContext: wrapped };
602
- } else {
603
- log(`memoryrouter: prepare returned no context (memories_found=${data.memories_found || 0})`);
604
595
  }
605
596
  } catch (err) {
606
597
  log(
@@ -736,7 +727,7 @@ const memoryRouterPlugin = {
736
727
  execute: async (_toolCallId: string, params: Record<string, unknown>) => {
737
728
  const query = typeof params.query === "string" ? params.query.trim() : "";
738
729
  if (!query) return jsonToolResult({ results: [], error: "query required" });
739
- const limit = typeof params.maxResults === "number" ? params.maxResults : 10;
730
+ const limit = typeof params.maxResults === "number" ? params.maxResults : 50;
740
731
  try {
741
732
  const res = await fetch(`${endpoint}/v1/memory/search`, {
742
733
  method: "POST",
@@ -977,71 +968,17 @@ const memoryRouterPlugin = {
977
968
  }
978
969
  });
979
970
 
980
- // Mode commands
981
- for (const [modeName, modeDesc] of [
982
- ["relay", "Relay mode — hooks only, works on stock OpenClaw [default]"],
983
- ["proxy", "Proxy mode — memory on every LLM call (requires registerStreamFnWrapper)"],
984
- ] as const) {
985
- mr.command(modeName)
986
- .description(modeDesc)
987
- .action(async () => {
988
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
989
- try {
990
- await setPluginConfig(api, { key: memoryKey, endpoint: cfg?.endpoint, density, mode: modeName });
991
- console.log(`✓ Mode set to ${modeName}`);
992
- } catch (err) {
993
- console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
994
- }
995
- });
996
- }
997
971
 
998
- // Embedding model command
999
- mr.command("embeddings")
1000
- .description("Get or set embedding model (bge, qwen)")
1001
- .argument("[model]", "Embedding model: bge (default, 1024d) or qwen (4096d)")
1002
- .action(async (model: string | undefined) => {
1003
- if (!model) {
1004
- console.log(`Current embedding model: ${embeddings || "bge (default)"}`);
1005
- console.log(`\nAvailable models:`);
1006
- console.log(` bge BGE-M3 1024 dims (Cloudflare Workers AI, ~18ms, free)`);
1007
- console.log(` qwen Qwen3-8B 4096 dims (HuggingFace, ~69ms, $0.80/hr)`);
1008
- console.log(`\nUsage: openclaw mr embeddings <model>`);
1009
- return;
1010
- }
1011
- const normalized = model.toLowerCase().trim();
1012
- const valid = ["bge", "bge-m3", "qwen", "qwen3", "qwen3-8b", "default"];
1013
- if (!valid.includes(normalized)) {
1014
- console.error(`Unknown model: "${model}". Valid options: bge, qwen`);
1015
- return;
1016
- }
1017
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1018
- // "bge" / "default" → clear embeddings (use server default)
1019
- const newEmbeddings = (normalized === "bge" || normalized === "bge-m3" || normalized === "default")
1020
- ? undefined
1021
- : normalized.startsWith("qwen") ? "qwen" : normalized;
1022
- try {
1023
- await setPluginConfig(api, {
1024
- key: memoryKey,
1025
- endpoint: cfg?.endpoint,
1026
- density,
1027
- mode,
1028
- logging,
1029
- ...(newEmbeddings && { embeddings: newEmbeddings }),
1030
- });
1031
- const label = newEmbeddings || "bge (default)";
1032
- console.log(`✓ Embedding model set to ${label}`);
1033
- console.log(` All future prepare/ingest/search will use ${label}`);
1034
- console.log(`\n Restart gateway to apply: openclaw gateway restart`);
1035
- } catch (err) {
1036
- console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
1037
- }
1038
- });
972
+
973
+
1039
974
 
1040
975
  mr.command("status")
1041
976
  .description("Show MemoryRouter vault stats")
1042
977
  .option("--json", "JSON output")
1043
- .action(async (opts) => {
1044
- if (!memoryKey) {
978
+ .option("--key <key>", "Override memory key (for subagent vaults)")
979
+ .action(async (opts: { json?: boolean; key?: string }) => {
980
+ const effectiveKey = opts.key || memoryKey;
981
+ if (!effectiveKey) {
1045
982
  if (opts.json) {
1046
983
  console.log(JSON.stringify({ enabled: false, key: null }, null, 2));
1047
984
  } else {
@@ -1058,19 +995,17 @@ const memoryRouterPlugin = {
1058
995
  ? `${endpoint}/v1/memory/stats?embeddings=${encodeURIComponent(embeddings)}`
1059
996
  : `${endpoint}/v1/memory/stats`;
1060
997
  const res = await fetch(statsUrl, {
1061
- headers: { Authorization: `Bearer ${memoryKey}` },
998
+ headers: { Authorization: `Bearer ${effectiveKey}` },
1062
999
  });
1063
1000
  const data = (await res.json()) as { totalVectors?: number; totalTokens?: number };
1064
1001
  if (opts.json) {
1065
- console.log(JSON.stringify({ enabled: true, key: memoryKey, density, mode, embeddings: embeddings || "bge", stats: data }, null, 2));
1002
+ console.log(JSON.stringify({ enabled: true, key: effectiveKey, density, stats: data }, null, 2));
1066
1003
  } else {
1067
1004
  console.log("MemoryRouter Status");
1068
1005
  console.log("───────────────────────────");
1069
1006
  console.log(`Enabled: ✓ Yes`);
1070
- console.log(`Key: ${memoryKey.slice(0, 6)}...${memoryKey.slice(-3)}`);
1071
- console.log(`Mode: ${mode}`);
1007
+ console.log(`Key: ${effectiveKey.slice(0, 6)}...${effectiveKey.slice(-3)}`);
1072
1008
  console.log(`Density: ${density}`);
1073
- console.log(`Embeddings: ${embeddings || "bge (default)"}`);
1074
1009
  console.log(`Endpoint: ${endpoint}`);
1075
1010
  console.log(`Memories: ${data.totalVectors ?? 0}`);
1076
1011
  console.log(`Tokens: ${data.totalTokens ?? 0}`);
@@ -1085,8 +1020,10 @@ const memoryRouterPlugin = {
1085
1020
  .argument("[path]", "Specific file or directory to upload")
1086
1021
  .option("--workspace <dir>", "Workspace directory")
1087
1022
  .option("--brain <dir>", "State directory with sessions")
1088
- .action(async (targetPath: string | undefined, opts: { workspace?: string; brain?: string }) => {
1089
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1023
+ .option("--key <key>", "Override memory key (for subagent vaults)")
1024
+ .action(async (targetPath: string | undefined, opts: { workspace?: string; brain?: string; key?: string }) => {
1025
+ const effectiveKey = opts.key || memoryKey;
1026
+ if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
1090
1027
  const os = await import("node:os");
1091
1028
  const path = await import("node:path");
1092
1029
  const stateDir = opts.brain ? path.resolve(opts.brain) : path.join(os.homedir(), ".openclaw");
@@ -1097,22 +1034,24 @@ const memoryRouterPlugin = {
1097
1034
  ? path.resolve(configWorkspace.replace(/^~/, os.homedir()))
1098
1035
  : path.join(os.homedir(), ".openclaw", "workspace");
1099
1036
  const { runUpload } = await import("./upload.js");
1100
- await runUpload({ memoryKey, endpoint, targetPath, stateDir, workspacePath, hasWorkspaceFlag: !!opts.workspace, hasBrainFlag: !!opts.brain, embeddings });
1037
+ await runUpload({ memoryKey: effectiveKey, endpoint, targetPath, stateDir, workspacePath, hasWorkspaceFlag: !!opts.workspace, hasBrainFlag: !!opts.brain, embeddings });
1101
1038
  });
1102
1039
 
1103
1040
  mr.command("search")
1104
1041
  .description("Search memories in vault")
1105
1042
  .argument("<query>", "Search query")
1106
- .option("-n, --limit <number>", "Number of results", "5")
1043
+ .option("-n, --limit <number>", "Number of results", "50")
1107
1044
  .option("--json", "Output raw JSON response")
1108
- .action(async (query: string, opts: { limit: string; json?: boolean }) => {
1109
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1110
- const limit = parseInt(opts.limit, 10) || 5;
1045
+ .option("--key <key>", "Override memory key (for subagent vaults)")
1046
+ .action(async (query: string, opts: { limit: string; json?: boolean; key?: string }) => {
1047
+ const effectiveKey = opts.key || memoryKey;
1048
+ if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
1049
+ const limit = parseInt(opts.limit, 10) || 50;
1111
1050
  try {
1112
1051
  const res = await fetch(`${endpoint}/v1/memory/search`, {
1113
1052
  method: "POST",
1114
1053
  headers: {
1115
- Authorization: `Bearer ${memoryKey}`,
1054
+ Authorization: `Bearer ${effectiveKey}`,
1116
1055
  "Content-Type": "application/json",
1117
1056
  ...(embeddings && { "X-Embedding-Model": embeddings }),
1118
1057
  },
@@ -1177,15 +1116,17 @@ const memoryRouterPlugin = {
1177
1116
 
1178
1117
  mr.command("delete")
1179
1118
  .description("Clear all memories from vault")
1180
- .action(async () => {
1181
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1119
+ .option("--key <key>", "Override memory key (for subagent vaults)")
1120
+ .action(async (opts: { key?: string }) => {
1121
+ const effectiveKey = opts.key || memoryKey;
1122
+ if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
1182
1123
  try {
1183
1124
  const deleteUrl = embeddings
1184
1125
  ? `${endpoint}/v1/memory?embeddings=${encodeURIComponent(embeddings)}`
1185
1126
  : `${endpoint}/v1/memory`;
1186
1127
  const res = await fetch(deleteUrl, {
1187
1128
  method: "DELETE",
1188
- headers: { Authorization: `Bearer ${memoryKey}` },
1129
+ headers: { Authorization: `Bearer ${effectiveKey}` },
1189
1130
  });
1190
1131
  const data = (await res.json()) as { message?: string };
1191
1132
  const modelLabel = embeddings ? ` (${embeddings})` : "";
@@ -1194,33 +1135,6 @@ const memoryRouterPlugin = {
1194
1135
  console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
1195
1136
  }
1196
1137
  });
1197
-
1198
- // ── Workspace Sync ──
1199
- mr.command("sync")
1200
- .description("Sync workspace files to vault using source_hash tracking")
1201
- .option("--workspace <dir>", "Workspace directory")
1202
- .option("--status", "Show manifest status instead of syncing")
1203
- .action(async (opts: { workspace?: string; status?: boolean }) => {
1204
- if (opts.status) {
1205
- await showManifestStatus();
1206
- return;
1207
- }
1208
- if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
1209
- const os = await import("node:os");
1210
- const configWorkspace = (api.config as any).workspace || (api.config as any).agents?.defaults?.workspace;
1211
- const workspaceDir = opts.workspace
1212
- ? path.resolve(opts.workspace)
1213
- : configWorkspace
1214
- ? path.resolve(configWorkspace.replace(/^~/, os.homedir()))
1215
- : path.join(os.homedir(), ".openclaw", "workspace");
1216
- await runSyncCli({
1217
- workspaceDir,
1218
- endpoint,
1219
- memoryKey,
1220
- embeddings,
1221
- logging: true,
1222
- });
1223
- });
1224
1138
  },
1225
1139
  { commands: ["mr"] },
1226
1140
  );
@@ -1231,41 +1145,8 @@ const memoryRouterPlugin = {
1231
1145
 
1232
1146
  api.registerService({
1233
1147
  id: "mr-memory",
1234
- start: async () => {
1235
- if (memoryKey) {
1236
- api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
1237
-
1238
- // ── Auto-sync workspace files (non-blocking) ──
1239
- // Resolve workspace directory
1240
- const configWorkspace = (api.config as any).workspace || (api.config as any).agents?.defaults?.workspace;
1241
- const workspaceDir = configWorkspace
1242
- ? path.resolve(configWorkspace.replace(/^~/, homedir()))
1243
- : path.join(homedir(), ".openclaw", "workspace");
1244
-
1245
- // Fire and forget — don't block service startup
1246
- syncWorkspaceFiles({
1247
- workspaceDir,
1248
- endpoint,
1249
- memoryKey,
1250
- embeddings,
1251
- logging,
1252
- })
1253
- .then((result) => {
1254
- if (result.uploaded > 0 || result.deleted > 0) {
1255
- api.logger.info?.(
1256
- `memoryrouter: sync complete — ${result.uploaded} uploaded, ${result.deleted} deleted, ${result.unchanged} unchanged`,
1257
- );
1258
- } else {
1259
- log(`memoryrouter: sync complete — ${result.unchanged} files unchanged`);
1260
- }
1261
- if (result.errors.length > 0) {
1262
- api.logger.warn?.(`memoryrouter: sync had ${result.errors.length} errors`);
1263
- }
1264
- })
1265
- .catch((err) => {
1266
- api.logger.warn?.(`memoryrouter: sync error — ${err instanceof Error ? err.message : String(err)}`);
1267
- });
1268
- }
1148
+ start: () => {
1149
+ if (memoryKey) api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
1269
1150
  },
1270
1151
  stop: () => {
1271
1152
  api.logger.info?.("memoryrouter: stopped");
package/package.json CHANGED
@@ -1,14 +1,13 @@
1
1
  {
2
2
  "name": "mr-memory",
3
- "version": "3.2.1",
3
+ "version": "3.3.0",
4
4
  "description": "MemoryRouter persistent memory plugin for OpenClaw — your AI remembers every conversation",
5
5
  "type": "module",
6
6
  "files": [
7
7
  "README.md",
8
8
  "index.ts",
9
9
  "openclaw.plugin.json",
10
- "upload.ts",
11
- "sync.ts"
10
+ "upload.ts"
12
11
  ],
13
12
  "keywords": [
14
13
  "openclaw",
package/upload.ts CHANGED
@@ -370,11 +370,10 @@ export async function runUpload(params: {
370
370
 
371
371
  const workspacePath = params.workspacePath ?? process.cwd();
372
372
 
373
- // ── Target path: specific file/dir upload (old behavior, no sync) ──
373
+ let files: string[];
374
374
  if (targetPath) {
375
375
  const resolved = path.resolve(targetPath);
376
376
  const stat = await fs.stat(resolved);
377
- let files: string[];
378
377
  if (stat.isDirectory()) {
379
378
  const allDirFiles = await fs.readdir(resolved, { recursive: true });
380
379
  files = (allDirFiles as string[])
@@ -386,58 +385,19 @@ export async function runUpload(params: {
386
385
  } else {
387
386
  files = [resolved];
388
387
  }
389
- await uploadFilesLegacy(files, memoryKey, endpoint, embeddings);
390
- return;
391
- }
392
-
393
- // ── Determine what to upload ──
394
- const doWorkspace = !params.hasBrainFlag || params.hasWorkspaceFlag; // default: yes
395
- const doSessions = !params.hasWorkspaceFlag || params.hasBrainFlag; // default: yes
396
-
397
- // ── Workspace files: use source_hash sync (smart, tracks changes) ──
398
- if (doWorkspace) {
399
- console.log("📁 Syncing workspace files...");
400
- const { syncWorkspaceFiles } = await import("./sync.js");
401
- const syncResult = await syncWorkspaceFiles({
402
- workspaceDir: workspacePath,
403
- endpoint,
404
- memoryKey,
405
- embeddings,
406
- logging: true,
407
- });
408
- console.log(`\n Workspace: ${syncResult.uploaded} uploaded, ${syncResult.deleted} deleted, ${syncResult.unchanged} unchanged`);
409
- if (syncResult.errors.length > 0) {
410
- console.log(` ⚠️ ${syncResult.errors.length} errors`);
411
- for (const err of syncResult.errors.slice(0, 3)) {
412
- console.log(` • ${err}`);
413
- }
414
- }
388
+ } else if (params.hasBrainFlag && !params.hasWorkspaceFlag) {
389
+ // --brain only: upload sessions from brain path
390
+ files = await discoverBrainFiles(stateDir);
391
+ } else if (params.hasWorkspaceFlag && !params.hasBrainFlag) {
392
+ // --workspace only: upload workspace files only
393
+ files = await discoverWorkspaceFiles(workspacePath);
394
+ } else {
395
+ // No flags or both flags: upload both workspace + sessions
396
+ const wsFiles = await discoverWorkspaceFiles(workspacePath);
397
+ const brainFiles = await discoverBrainFiles(stateDir);
398
+ files = [...wsFiles, ...brainFiles];
415
399
  }
416
400
 
417
- // ── Session transcripts: batch upload (append-only, no sync needed) ──
418
- if (doSessions) {
419
- console.log("\n📝 Uploading session transcripts...");
420
- const sessionFiles = await discoverBrainFiles(stateDir);
421
- if (sessionFiles.length === 0) {
422
- console.log(" No session files found.");
423
- } else {
424
- await uploadFilesLegacy(sessionFiles, memoryKey, endpoint, embeddings);
425
- }
426
- }
427
- }
428
-
429
- /**
430
- * Legacy batch upload for session transcripts and specific-path uploads.
431
- * Sessions are append-only, so they don't need source_hash tracking.
432
- */
433
- async function uploadFilesLegacy(
434
- files: string[],
435
- memoryKey: string,
436
- endpoint: string,
437
- embeddings?: string,
438
- ): Promise<void> {
439
- const uploadUrl = `${endpoint}/v1/memory/upload`;
440
-
441
401
  if (files.length === 0) {
442
402
  console.log("No files found to upload.");
443
403
  return;
@@ -536,6 +496,7 @@ async function uploadFilesLegacy(
536
496
  if (batchFailed > 0) {
537
497
  const errHint = result.errors?.[0] ? ` (${result.errors[0].slice(0, 120)})` : "";
538
498
  console.log(`⚠ ${batchStored} stored, ${batchFailed} skipped${errHint}`);
499
+ // Show detailed error diagnostics (up to 5 per batch)
539
500
  if (result.errors && result.errors.length > 1) {
540
501
  for (const err of result.errors.slice(0, 5)) {
541
502
  console.log(` → ${err.slice(0, 200)}`);
package/sync.ts DELETED
@@ -1,367 +0,0 @@
1
- /**
2
- * MemoryRouter Workspace Sync — source_hash-based file tracking.
3
- *
4
- * Tracks workspace files via SHA-256 content hashes. When files change,
5
- * old chunks are deleted and new content is uploaded with the new hash.
6
- * This enables clean replacement without stale chunk accumulation.
7
- *
8
- * IMPORTANT: source_hash is ONLY for workspace files, NEVER for sessions.
9
- */
10
-
11
- import { readFile, writeFile, readdir, lstat, mkdir } from "node:fs/promises";
12
- import { createHash } from "node:crypto";
13
- import { join, relative, sep } from "node:path";
14
- import { homedir } from "node:os";
15
-
16
- // ── Types ──
17
-
18
- export interface ManifestFileEntry {
19
- source_hash: string;
20
- size: number;
21
- chunks: number;
22
- uploaded_at: number;
23
- }
24
-
25
- export interface Manifest {
26
- version: number;
27
- lastSync: number;
28
- files: Record<string, ManifestFileEntry>;
29
- }
30
-
31
- export interface SyncResult {
32
- uploaded: number;
33
- deleted: number;
34
- unchanged: number;
35
- errors: string[];
36
- }
37
-
38
- export interface SyncOptions {
39
- workspaceDir: string;
40
- endpoint: string;
41
- memoryKey: string;
42
- embeddings?: string;
43
- logging?: boolean;
44
- }
45
-
46
- // ── Constants ──
47
-
48
- const MANIFEST_PATH = join(homedir(), ".openclaw", "mr-memory-manifest.json");
49
- const MAX_FILE_SIZE = 1_000_000; // 1MB limit
50
- const TEXT_EXTENSIONS = new Set(["md", "txt"]);
51
-
52
- // ── Manifest I/O ──
53
-
54
- export async function loadManifest(): Promise<Manifest> {
55
- try {
56
- const raw = await readFile(MANIFEST_PATH, "utf-8");
57
- return JSON.parse(raw) as Manifest;
58
- } catch {
59
- // First run or corrupted — create empty manifest
60
- return { version: 1, lastSync: 0, files: {} };
61
- }
62
- }
63
-
64
- export async function saveManifest(manifest: Manifest): Promise<void> {
65
- manifest.lastSync = Date.now();
66
- // Ensure directory exists
67
- const dir = join(homedir(), ".openclaw");
68
- await mkdir(dir, { recursive: true });
69
- await writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
70
- }
71
-
72
- // ── Hash Computation ──
73
-
74
- export function computeSourceHash(content: string): string {
75
- const hash = createHash("sha256").update(content).digest("hex");
76
- return hash.substring(0, 16); // First 16 hex chars, matching MR API format
77
- }
78
-
79
- // ── File Discovery ──
80
-
81
- /**
82
- * Discover ALL workspace files to sync:
83
- * - All .md and .txt files in workspace root
84
- * - All files recursively under memory/ directory
85
- * - Excludes: hidden files/dirs, node_modules/, files > 1MB, binary files
86
- */
87
- export async function discoverWorkspaceFiles(workspaceDir: string): Promise<string[]> {
88
- const files: string[] = [];
89
-
90
- async function walk(dir: string) {
91
- let entries;
92
- try {
93
- entries = await readdir(dir, { withFileTypes: true });
94
- } catch {
95
- return; // Directory doesn't exist or not readable
96
- }
97
-
98
- for (const entry of entries) {
99
- // Skip hidden dirs/files and node_modules
100
- if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
101
-
102
- const fullPath = join(dir, entry.name);
103
-
104
- if (entry.isDirectory()) {
105
- await walk(fullPath);
106
- } else if (entry.isFile()) {
107
- // Only text files we want to index
108
- const ext = entry.name.toLowerCase().split(".").pop() || "";
109
- if (!TEXT_EXTENSIONS.has(ext)) continue;
110
-
111
- // Check file size
112
- try {
113
- const stat = await lstat(fullPath);
114
- if (stat.size > MAX_FILE_SIZE) continue;
115
- if (stat.size === 0) continue; // Skip empty files
116
- files.push(relative(workspaceDir, fullPath));
117
- } catch {
118
- continue; // Skip unreadable files
119
- }
120
- }
121
- }
122
- }
123
-
124
- await walk(workspaceDir);
125
- return files.sort();
126
- }
127
-
128
- // ── API Helpers ──
129
-
130
- /**
131
- * Upload file content with source_hash using isolated chunking.
132
- * Uses POST /v1/memory/upload-source endpoint.
133
- */
134
- export async function uploadSourceFile(
135
- endpoint: string,
136
- memoryKey: string,
137
- content: string,
138
- sourceHash: string,
139
- embeddings?: string,
140
- ): Promise<{ stored: number; chunks: number }> {
141
- const res = await fetch(`${endpoint}/v1/memory/upload-source`, {
142
- method: "POST",
143
- headers: {
144
- Authorization: `Bearer ${memoryKey}`,
145
- "Content-Type": "application/json",
146
- ...(embeddings && { "X-Embedding-Model": embeddings }),
147
- },
148
- body: JSON.stringify({ content, source_hash: sourceHash }),
149
- });
150
-
151
- if (!res.ok) {
152
- const errText = await res.text().catch(() => "");
153
- throw new Error(`Upload failed: HTTP ${res.status} — ${errText.slice(0, 200)}`);
154
- }
155
-
156
- const data = (await res.json()) as { stored: number; chunks: number; source_hash: string };
157
- return { stored: data.stored || 0, chunks: data.chunks || data.stored || 0 };
158
- }
159
-
160
- /**
161
- * Delete all chunks with a given source_hash.
162
- * Uses POST /v1/memory/source/delete (POST fallback for clients that can't send DELETE with body).
163
- */
164
- export async function deleteBySourceHash(
165
- endpoint: string,
166
- memoryKey: string,
167
- sourceHash: string,
168
- ): Promise<number> {
169
- const res = await fetch(`${endpoint}/v1/memory/source/delete`, {
170
- method: "POST",
171
- headers: {
172
- Authorization: `Bearer ${memoryKey}`,
173
- "Content-Type": "application/json",
174
- },
175
- body: JSON.stringify({ source_hash: sourceHash }),
176
- });
177
-
178
- if (!res.ok) {
179
- const errText = await res.text().catch(() => "");
180
- throw new Error(`Delete failed: HTTP ${res.status} — ${errText.slice(0, 200)}`);
181
- }
182
-
183
- const data = (await res.json()) as { deleted: number; source_hash: string };
184
- return data.deleted || 0;
185
- }
186
-
187
- // ── Main Sync Flow ──
188
-
189
- /**
190
- * Sync workspace files to MemoryRouter using source_hash tracking.
191
- *
192
- * Algorithm:
193
- * 1. Load manifest (previous state)
194
- * 2. Discover current workspace files, compute hashes
195
- * 3. Diff current vs manifest:
196
- * - NEW: upload with source_hash
197
- * - CHANGED: delete old hash, upload new
198
- * - DELETED: delete from MR
199
- * - UNCHANGED: skip
200
- * 4. Save updated manifest
201
- */
202
- export async function syncWorkspaceFiles(options: SyncOptions): Promise<SyncResult> {
203
- const { workspaceDir, endpoint, memoryKey, embeddings, logging } = options;
204
- const log = (msg: string) => {
205
- if (logging) console.log(msg);
206
- };
207
-
208
- const result: SyncResult = { uploaded: 0, deleted: 0, unchanged: 0, errors: [] };
209
-
210
- // 1. Load manifest
211
- const manifest = await loadManifest();
212
- log(`memoryrouter: sync starting (${Object.keys(manifest.files).length} files in manifest)`);
213
-
214
- // 2. Discover workspace files and compute hashes
215
- const diskFiles = await discoverWorkspaceFiles(workspaceDir);
216
- log(`memoryrouter: discovered ${diskFiles.length} workspace files`);
217
-
218
- const diskState = new Map<string, { hash: string; size: number; content: string }>();
219
-
220
- for (const relPath of diskFiles) {
221
- const absPath = join(workspaceDir, relPath);
222
- try {
223
- const content = await readFile(absPath, "utf-8");
224
- const trimmed = content.trim();
225
- if (!trimmed) continue; // Skip empty files
226
-
227
- const hash = computeSourceHash(content);
228
- const size = Buffer.byteLength(content, "utf-8");
229
- diskState.set(relPath, { hash, size, content });
230
- } catch (err) {
231
- result.errors.push(`Read error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
232
- }
233
- }
234
-
235
- // 3a. Process disk files (new + changed)
236
- for (const [relPath, { hash, size, content }] of diskState) {
237
- const manifestEntry = manifest.files[relPath];
238
-
239
- if (!manifestEntry) {
240
- // NEW FILE — upload
241
- try {
242
- const { chunks } = await uploadSourceFile(endpoint, memoryKey, content, hash, embeddings);
243
- manifest.files[relPath] = {
244
- source_hash: hash,
245
- size,
246
- chunks,
247
- uploaded_at: Date.now(),
248
- };
249
- result.uploaded++;
250
- log(`memoryrouter: sync uploaded new file ${relPath} (${chunks} chunks)`);
251
- } catch (err) {
252
- result.errors.push(`Upload error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
253
- }
254
- } else if (manifestEntry.source_hash !== hash) {
255
- // CHANGED FILE — delete old, upload new
256
- try {
257
- const deleted = await deleteBySourceHash(endpoint, memoryKey, manifestEntry.source_hash);
258
- log(`memoryrouter: sync deleted old version of ${relPath} (${deleted} chunks)`);
259
- } catch (err) {
260
- // Log but continue — old chunks may have been deleted manually
261
- log(`memoryrouter: sync delete warning for ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
262
- }
263
-
264
- try {
265
- const { chunks } = await uploadSourceFile(endpoint, memoryKey, content, hash, embeddings);
266
- manifest.files[relPath] = {
267
- source_hash: hash,
268
- size,
269
- chunks,
270
- uploaded_at: Date.now(),
271
- };
272
- result.uploaded++;
273
- log(`memoryrouter: sync updated file ${relPath} (${chunks} new chunks)`);
274
- } catch (err) {
275
- result.errors.push(`Upload error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
276
- }
277
- } else {
278
- // UNCHANGED — skip
279
- result.unchanged++;
280
- }
281
- }
282
-
283
- // 3b. Handle deleted files (in manifest but not on disk)
284
- for (const relPath of Object.keys(manifest.files)) {
285
- if (!diskState.has(relPath)) {
286
- try {
287
- const deleted = await deleteBySourceHash(endpoint, memoryKey, manifest.files[relPath].source_hash);
288
- delete manifest.files[relPath];
289
- result.deleted++;
290
- log(`memoryrouter: sync deleted removed file ${relPath} (${deleted} chunks)`);
291
- } catch (err) {
292
- result.errors.push(`Delete error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
293
- }
294
- }
295
- }
296
-
297
- // 4. Save updated manifest
298
- await saveManifest(manifest);
299
-
300
- log(
301
- `memoryrouter: sync complete — ${result.uploaded} uploaded, ${result.deleted} deleted, ${result.unchanged} unchanged`,
302
- );
303
-
304
- return result;
305
- }
306
-
307
- // ── CLI Helper ──
308
-
309
- /**
310
- * Run sync from CLI with console output.
311
- */
312
- export async function runSyncCli(options: SyncOptions): Promise<void> {
313
- const { workspaceDir, embeddings } = options;
314
-
315
- const modelLabel = embeddings ? `(model: ${embeddings})` : "(model: bge)";
316
- console.log(`Syncing workspace to MemoryRouter ${modelLabel}...`);
317
- console.log(` Workspace: ${workspaceDir}`);
318
-
319
- try {
320
- const result = await syncWorkspaceFiles({ ...options, logging: true });
321
-
322
- console.log("\n────────────────────────────────");
323
- console.log(`✅ Sync complete`);
324
- console.log(` Uploaded: ${result.uploaded}`);
325
- console.log(` Deleted: ${result.deleted}`);
326
- console.log(` Unchanged: ${result.unchanged}`);
327
-
328
- if (result.errors.length > 0) {
329
- console.log(`\n⚠️ ${result.errors.length} errors:`);
330
- for (const err of result.errors.slice(0, 5)) {
331
- console.log(` • ${err}`);
332
- }
333
- if (result.errors.length > 5) {
334
- console.log(` • ... and ${result.errors.length - 5} more`);
335
- }
336
- }
337
- } catch (err) {
338
- console.error(`\n❌ Sync failed: ${err instanceof Error ? err.message : String(err)}`);
339
- }
340
- }
341
-
342
- /**
343
- * Show manifest status from CLI.
344
- */
345
- export async function showManifestStatus(): Promise<void> {
346
- const manifest = await loadManifest();
347
- const fileCount = Object.keys(manifest.files).length;
348
-
349
- console.log("Workspace Sync Manifest");
350
- console.log("───────────────────────────────");
351
- console.log(` Path: ${MANIFEST_PATH}`);
352
- console.log(` Version: ${manifest.version}`);
353
- console.log(` Last sync: ${manifest.lastSync ? new Date(manifest.lastSync).toLocaleString() : "never"}`);
354
- console.log(` Files: ${fileCount}`);
355
-
356
- if (fileCount > 0) {
357
- console.log("\n Tracked files:");
358
- const files = Object.entries(manifest.files).sort((a, b) => a[0].localeCompare(b[0]));
359
- for (const [path, entry] of files.slice(0, 20)) {
360
- const date = new Date(entry.uploaded_at).toLocaleDateString();
361
- console.log(` • ${path} (${entry.chunks} chunks, ${date})`);
362
- }
363
- if (files.length > 20) {
364
- console.log(` ... and ${files.length - 20} more`);
365
- }
366
- }
367
- }