mr-memory 3.2.1 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.ts +36 -155
- package/package.json +2 -3
- package/upload.ts +13 -52
- package/sync.ts +0 -367
package/index.ts
CHANGED
|
@@ -10,11 +10,9 @@
|
|
|
10
10
|
|
|
11
11
|
import { readFile, readdir, stat, lstat } from "node:fs/promises";
|
|
12
12
|
import { join, resolve, relative, isAbsolute, sep } from "node:path";
|
|
13
|
-
import { homedir } from "node:os";
|
|
14
13
|
import path from "node:path";
|
|
15
14
|
import { spawn } from "node:child_process";
|
|
16
15
|
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
|
17
|
-
import { syncWorkspaceFiles, runSyncCli, showManifestStatus } from "./sync.js";
|
|
18
16
|
|
|
19
17
|
const DEFAULT_ENDPOINT = "https://api.memoryrouter.ai";
|
|
20
18
|
|
|
@@ -385,8 +383,7 @@ const memoryRouterPlugin = {
|
|
|
385
383
|
const log = (msg: string) => { if (logging) api.logger.info?.(msg); };
|
|
386
384
|
|
|
387
385
|
if (memoryKey) {
|
|
388
|
-
|
|
389
|
-
api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}..., mode: ${mode}${modelLabel})`);
|
|
386
|
+
api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
|
|
390
387
|
} else {
|
|
391
388
|
api.logger.info?.("memoryrouter: no key configured — run: openclaw mr <key>");
|
|
392
389
|
api.logger.info?.("memoryrouter: get your free API key at https://memoryrouter.ai");
|
|
@@ -407,7 +404,6 @@ const memoryRouterPlugin = {
|
|
|
407
404
|
// When PR #24122 merges, OpenClaw will use the returned prependContext.
|
|
408
405
|
// This gives forward compatibility — no plugin update needed.
|
|
409
406
|
api.on("llm_input", async (event, ctx) => {
|
|
410
|
-
log(`memoryrouter: llm_input fired (sessionKey=${ctx.sessionKey}, promptBuildFired=${promptBuildFiredThisRun})`);
|
|
411
407
|
// Skip the first call — before_prompt_build already handled it
|
|
412
408
|
// (before_prompt_build includes workspace+tools+skills for accurate billing)
|
|
413
409
|
if (promptBuildFiredThisRun) {
|
|
@@ -416,6 +412,7 @@ const memoryRouterPlugin = {
|
|
|
416
412
|
}
|
|
417
413
|
|
|
418
414
|
try {
|
|
415
|
+
const startMs = Date.now();
|
|
419
416
|
const prompt = event.prompt;
|
|
420
417
|
if (prompt === lastPreparedPrompt && lastPreparedPrompt !== "") return;
|
|
421
418
|
lastPreparedPrompt = prompt;
|
|
@@ -481,9 +478,8 @@ const memoryRouterPlugin = {
|
|
|
481
478
|
};
|
|
482
479
|
|
|
483
480
|
if (data.context) {
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
);
|
|
481
|
+
const elapsed = Date.now() - startMs;
|
|
482
|
+
api.logger.info?.(`memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens, ${elapsed}ms) [llm_input]`);
|
|
487
483
|
const wrapped = wrapForInjection(data.context);
|
|
488
484
|
return { appendSystemContext: wrapped };
|
|
489
485
|
}
|
|
@@ -495,8 +491,8 @@ const memoryRouterPlugin = {
|
|
|
495
491
|
// ── before_prompt_build: fires once per run (primary, includes full billing context)
|
|
496
492
|
api.on("before_prompt_build", async (event, ctx) => {
|
|
497
493
|
promptBuildFiredThisRun = true;
|
|
498
|
-
log(`memoryrouter: before_prompt_build fired (sessionKey=${ctx.sessionKey}, promptLen=${event.prompt?.length})`);
|
|
499
494
|
try {
|
|
495
|
+
const startMs = Date.now();
|
|
500
496
|
const prompt = event.prompt;
|
|
501
497
|
|
|
502
498
|
// Deduplicate — if we already prepared this exact prompt, skip
|
|
@@ -591,16 +587,11 @@ const memoryRouterPlugin = {
|
|
|
591
587
|
memory_tokens?: number;
|
|
592
588
|
};
|
|
593
589
|
|
|
594
|
-
log(`memoryrouter: prepare response — memories_found=${data.memories_found || 0}, context_length=${data.context?.length || 0}, tokens=${data.memory_tokens || 0}, embeddings=${embeddings || 'bge'}`);
|
|
595
|
-
|
|
596
590
|
if (data.context) {
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
);
|
|
591
|
+
const elapsed = Date.now() - startMs;
|
|
592
|
+
api.logger.info?.(`memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens, ${elapsed}ms)`);
|
|
600
593
|
const wrapped = wrapForInjection(data.context);
|
|
601
594
|
return { appendSystemContext: wrapped };
|
|
602
|
-
} else {
|
|
603
|
-
log(`memoryrouter: prepare returned no context (memories_found=${data.memories_found || 0})`);
|
|
604
595
|
}
|
|
605
596
|
} catch (err) {
|
|
606
597
|
log(
|
|
@@ -736,7 +727,7 @@ const memoryRouterPlugin = {
|
|
|
736
727
|
execute: async (_toolCallId: string, params: Record<string, unknown>) => {
|
|
737
728
|
const query = typeof params.query === "string" ? params.query.trim() : "";
|
|
738
729
|
if (!query) return jsonToolResult({ results: [], error: "query required" });
|
|
739
|
-
const limit = typeof params.maxResults === "number" ? params.maxResults :
|
|
730
|
+
const limit = typeof params.maxResults === "number" ? params.maxResults : 50;
|
|
740
731
|
try {
|
|
741
732
|
const res = await fetch(`${endpoint}/v1/memory/search`, {
|
|
742
733
|
method: "POST",
|
|
@@ -977,71 +968,17 @@ const memoryRouterPlugin = {
|
|
|
977
968
|
}
|
|
978
969
|
});
|
|
979
970
|
|
|
980
|
-
// Mode commands
|
|
981
|
-
for (const [modeName, modeDesc] of [
|
|
982
|
-
["relay", "Relay mode — hooks only, works on stock OpenClaw [default]"],
|
|
983
|
-
["proxy", "Proxy mode — memory on every LLM call (requires registerStreamFnWrapper)"],
|
|
984
|
-
] as const) {
|
|
985
|
-
mr.command(modeName)
|
|
986
|
-
.description(modeDesc)
|
|
987
|
-
.action(async () => {
|
|
988
|
-
if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
|
|
989
|
-
try {
|
|
990
|
-
await setPluginConfig(api, { key: memoryKey, endpoint: cfg?.endpoint, density, mode: modeName });
|
|
991
|
-
console.log(`✓ Mode set to ${modeName}`);
|
|
992
|
-
} catch (err) {
|
|
993
|
-
console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
994
|
-
}
|
|
995
|
-
});
|
|
996
|
-
}
|
|
997
971
|
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
.description("Get or set embedding model (bge, qwen)")
|
|
1001
|
-
.argument("[model]", "Embedding model: bge (default, 1024d) or qwen (4096d)")
|
|
1002
|
-
.action(async (model: string | undefined) => {
|
|
1003
|
-
if (!model) {
|
|
1004
|
-
console.log(`Current embedding model: ${embeddings || "bge (default)"}`);
|
|
1005
|
-
console.log(`\nAvailable models:`);
|
|
1006
|
-
console.log(` bge BGE-M3 1024 dims (Cloudflare Workers AI, ~18ms, free)`);
|
|
1007
|
-
console.log(` qwen Qwen3-8B 4096 dims (HuggingFace, ~69ms, $0.80/hr)`);
|
|
1008
|
-
console.log(`\nUsage: openclaw mr embeddings <model>`);
|
|
1009
|
-
return;
|
|
1010
|
-
}
|
|
1011
|
-
const normalized = model.toLowerCase().trim();
|
|
1012
|
-
const valid = ["bge", "bge-m3", "qwen", "qwen3", "qwen3-8b", "default"];
|
|
1013
|
-
if (!valid.includes(normalized)) {
|
|
1014
|
-
console.error(`Unknown model: "${model}". Valid options: bge, qwen`);
|
|
1015
|
-
return;
|
|
1016
|
-
}
|
|
1017
|
-
if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
|
|
1018
|
-
// "bge" / "default" → clear embeddings (use server default)
|
|
1019
|
-
const newEmbeddings = (normalized === "bge" || normalized === "bge-m3" || normalized === "default")
|
|
1020
|
-
? undefined
|
|
1021
|
-
: normalized.startsWith("qwen") ? "qwen" : normalized;
|
|
1022
|
-
try {
|
|
1023
|
-
await setPluginConfig(api, {
|
|
1024
|
-
key: memoryKey,
|
|
1025
|
-
endpoint: cfg?.endpoint,
|
|
1026
|
-
density,
|
|
1027
|
-
mode,
|
|
1028
|
-
logging,
|
|
1029
|
-
...(newEmbeddings && { embeddings: newEmbeddings }),
|
|
1030
|
-
});
|
|
1031
|
-
const label = newEmbeddings || "bge (default)";
|
|
1032
|
-
console.log(`✓ Embedding model set to ${label}`);
|
|
1033
|
-
console.log(` All future prepare/ingest/search will use ${label}`);
|
|
1034
|
-
console.log(`\n Restart gateway to apply: openclaw gateway restart`);
|
|
1035
|
-
} catch (err) {
|
|
1036
|
-
console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1037
|
-
}
|
|
1038
|
-
});
|
|
972
|
+
|
|
973
|
+
|
|
1039
974
|
|
|
1040
975
|
mr.command("status")
|
|
1041
976
|
.description("Show MemoryRouter vault stats")
|
|
1042
977
|
.option("--json", "JSON output")
|
|
1043
|
-
.
|
|
1044
|
-
|
|
978
|
+
.option("--key <key>", "Override memory key (for subagent vaults)")
|
|
979
|
+
.action(async (opts: { json?: boolean; key?: string }) => {
|
|
980
|
+
const effectiveKey = opts.key || memoryKey;
|
|
981
|
+
if (!effectiveKey) {
|
|
1045
982
|
if (opts.json) {
|
|
1046
983
|
console.log(JSON.stringify({ enabled: false, key: null }, null, 2));
|
|
1047
984
|
} else {
|
|
@@ -1058,19 +995,17 @@ const memoryRouterPlugin = {
|
|
|
1058
995
|
? `${endpoint}/v1/memory/stats?embeddings=${encodeURIComponent(embeddings)}`
|
|
1059
996
|
: `${endpoint}/v1/memory/stats`;
|
|
1060
997
|
const res = await fetch(statsUrl, {
|
|
1061
|
-
headers: { Authorization: `Bearer ${
|
|
998
|
+
headers: { Authorization: `Bearer ${effectiveKey}` },
|
|
1062
999
|
});
|
|
1063
1000
|
const data = (await res.json()) as { totalVectors?: number; totalTokens?: number };
|
|
1064
1001
|
if (opts.json) {
|
|
1065
|
-
console.log(JSON.stringify({ enabled: true, key:
|
|
1002
|
+
console.log(JSON.stringify({ enabled: true, key: effectiveKey, density, stats: data }, null, 2));
|
|
1066
1003
|
} else {
|
|
1067
1004
|
console.log("MemoryRouter Status");
|
|
1068
1005
|
console.log("───────────────────────────");
|
|
1069
1006
|
console.log(`Enabled: ✓ Yes`);
|
|
1070
|
-
console.log(`Key: ${
|
|
1071
|
-
console.log(`Mode: ${mode}`);
|
|
1007
|
+
console.log(`Key: ${effectiveKey.slice(0, 6)}...${effectiveKey.slice(-3)}`);
|
|
1072
1008
|
console.log(`Density: ${density}`);
|
|
1073
|
-
console.log(`Embeddings: ${embeddings || "bge (default)"}`);
|
|
1074
1009
|
console.log(`Endpoint: ${endpoint}`);
|
|
1075
1010
|
console.log(`Memories: ${data.totalVectors ?? 0}`);
|
|
1076
1011
|
console.log(`Tokens: ${data.totalTokens ?? 0}`);
|
|
@@ -1085,8 +1020,10 @@ const memoryRouterPlugin = {
|
|
|
1085
1020
|
.argument("[path]", "Specific file or directory to upload")
|
|
1086
1021
|
.option("--workspace <dir>", "Workspace directory")
|
|
1087
1022
|
.option("--brain <dir>", "State directory with sessions")
|
|
1088
|
-
.
|
|
1089
|
-
|
|
1023
|
+
.option("--key <key>", "Override memory key (for subagent vaults)")
|
|
1024
|
+
.action(async (targetPath: string | undefined, opts: { workspace?: string; brain?: string; key?: string }) => {
|
|
1025
|
+
const effectiveKey = opts.key || memoryKey;
|
|
1026
|
+
if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
|
|
1090
1027
|
const os = await import("node:os");
|
|
1091
1028
|
const path = await import("node:path");
|
|
1092
1029
|
const stateDir = opts.brain ? path.resolve(opts.brain) : path.join(os.homedir(), ".openclaw");
|
|
@@ -1097,22 +1034,24 @@ const memoryRouterPlugin = {
|
|
|
1097
1034
|
? path.resolve(configWorkspace.replace(/^~/, os.homedir()))
|
|
1098
1035
|
: path.join(os.homedir(), ".openclaw", "workspace");
|
|
1099
1036
|
const { runUpload } = await import("./upload.js");
|
|
1100
|
-
await runUpload({ memoryKey, endpoint, targetPath, stateDir, workspacePath, hasWorkspaceFlag: !!opts.workspace, hasBrainFlag: !!opts.brain, embeddings });
|
|
1037
|
+
await runUpload({ memoryKey: effectiveKey, endpoint, targetPath, stateDir, workspacePath, hasWorkspaceFlag: !!opts.workspace, hasBrainFlag: !!opts.brain, embeddings });
|
|
1101
1038
|
});
|
|
1102
1039
|
|
|
1103
1040
|
mr.command("search")
|
|
1104
1041
|
.description("Search memories in vault")
|
|
1105
1042
|
.argument("<query>", "Search query")
|
|
1106
|
-
.option("-n, --limit <number>", "Number of results", "
|
|
1043
|
+
.option("-n, --limit <number>", "Number of results", "50")
|
|
1107
1044
|
.option("--json", "Output raw JSON response")
|
|
1108
|
-
.
|
|
1109
|
-
|
|
1110
|
-
const
|
|
1045
|
+
.option("--key <key>", "Override memory key (for subagent vaults)")
|
|
1046
|
+
.action(async (query: string, opts: { limit: string; json?: boolean; key?: string }) => {
|
|
1047
|
+
const effectiveKey = opts.key || memoryKey;
|
|
1048
|
+
if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
|
|
1049
|
+
const limit = parseInt(opts.limit, 10) || 50;
|
|
1111
1050
|
try {
|
|
1112
1051
|
const res = await fetch(`${endpoint}/v1/memory/search`, {
|
|
1113
1052
|
method: "POST",
|
|
1114
1053
|
headers: {
|
|
1115
|
-
Authorization: `Bearer ${
|
|
1054
|
+
Authorization: `Bearer ${effectiveKey}`,
|
|
1116
1055
|
"Content-Type": "application/json",
|
|
1117
1056
|
...(embeddings && { "X-Embedding-Model": embeddings }),
|
|
1118
1057
|
},
|
|
@@ -1177,15 +1116,17 @@ const memoryRouterPlugin = {
|
|
|
1177
1116
|
|
|
1178
1117
|
mr.command("delete")
|
|
1179
1118
|
.description("Clear all memories from vault")
|
|
1180
|
-
.
|
|
1181
|
-
|
|
1119
|
+
.option("--key <key>", "Override memory key (for subagent vaults)")
|
|
1120
|
+
.action(async (opts: { key?: string }) => {
|
|
1121
|
+
const effectiveKey = opts.key || memoryKey;
|
|
1122
|
+
if (!effectiveKey) { console.error("Not configured. Run: openclaw mr <key> or pass --key <key>"); return; }
|
|
1182
1123
|
try {
|
|
1183
1124
|
const deleteUrl = embeddings
|
|
1184
1125
|
? `${endpoint}/v1/memory?embeddings=${encodeURIComponent(embeddings)}`
|
|
1185
1126
|
: `${endpoint}/v1/memory`;
|
|
1186
1127
|
const res = await fetch(deleteUrl, {
|
|
1187
1128
|
method: "DELETE",
|
|
1188
|
-
headers: { Authorization: `Bearer ${
|
|
1129
|
+
headers: { Authorization: `Bearer ${effectiveKey}` },
|
|
1189
1130
|
});
|
|
1190
1131
|
const data = (await res.json()) as { message?: string };
|
|
1191
1132
|
const modelLabel = embeddings ? ` (${embeddings})` : "";
|
|
@@ -1194,33 +1135,6 @@ const memoryRouterPlugin = {
|
|
|
1194
1135
|
console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1195
1136
|
}
|
|
1196
1137
|
});
|
|
1197
|
-
|
|
1198
|
-
// ── Workspace Sync ──
|
|
1199
|
-
mr.command("sync")
|
|
1200
|
-
.description("Sync workspace files to vault using source_hash tracking")
|
|
1201
|
-
.option("--workspace <dir>", "Workspace directory")
|
|
1202
|
-
.option("--status", "Show manifest status instead of syncing")
|
|
1203
|
-
.action(async (opts: { workspace?: string; status?: boolean }) => {
|
|
1204
|
-
if (opts.status) {
|
|
1205
|
-
await showManifestStatus();
|
|
1206
|
-
return;
|
|
1207
|
-
}
|
|
1208
|
-
if (!memoryKey) { console.error("Not configured. Run: openclaw mr <key>"); return; }
|
|
1209
|
-
const os = await import("node:os");
|
|
1210
|
-
const configWorkspace = (api.config as any).workspace || (api.config as any).agents?.defaults?.workspace;
|
|
1211
|
-
const workspaceDir = opts.workspace
|
|
1212
|
-
? path.resolve(opts.workspace)
|
|
1213
|
-
: configWorkspace
|
|
1214
|
-
? path.resolve(configWorkspace.replace(/^~/, os.homedir()))
|
|
1215
|
-
: path.join(os.homedir(), ".openclaw", "workspace");
|
|
1216
|
-
await runSyncCli({
|
|
1217
|
-
workspaceDir,
|
|
1218
|
-
endpoint,
|
|
1219
|
-
memoryKey,
|
|
1220
|
-
embeddings,
|
|
1221
|
-
logging: true,
|
|
1222
|
-
});
|
|
1223
|
-
});
|
|
1224
1138
|
},
|
|
1225
1139
|
{ commands: ["mr"] },
|
|
1226
1140
|
);
|
|
@@ -1231,41 +1145,8 @@ const memoryRouterPlugin = {
|
|
|
1231
1145
|
|
|
1232
1146
|
api.registerService({
|
|
1233
1147
|
id: "mr-memory",
|
|
1234
|
-
start:
|
|
1235
|
-
if (memoryKey) {
|
|
1236
|
-
api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
|
|
1237
|
-
|
|
1238
|
-
// ── Auto-sync workspace files (non-blocking) ──
|
|
1239
|
-
// Resolve workspace directory
|
|
1240
|
-
const configWorkspace = (api.config as any).workspace || (api.config as any).agents?.defaults?.workspace;
|
|
1241
|
-
const workspaceDir = configWorkspace
|
|
1242
|
-
? path.resolve(configWorkspace.replace(/^~/, homedir()))
|
|
1243
|
-
: path.join(homedir(), ".openclaw", "workspace");
|
|
1244
|
-
|
|
1245
|
-
// Fire and forget — don't block service startup
|
|
1246
|
-
syncWorkspaceFiles({
|
|
1247
|
-
workspaceDir,
|
|
1248
|
-
endpoint,
|
|
1249
|
-
memoryKey,
|
|
1250
|
-
embeddings,
|
|
1251
|
-
logging,
|
|
1252
|
-
})
|
|
1253
|
-
.then((result) => {
|
|
1254
|
-
if (result.uploaded > 0 || result.deleted > 0) {
|
|
1255
|
-
api.logger.info?.(
|
|
1256
|
-
`memoryrouter: sync complete — ${result.uploaded} uploaded, ${result.deleted} deleted, ${result.unchanged} unchanged`,
|
|
1257
|
-
);
|
|
1258
|
-
} else {
|
|
1259
|
-
log(`memoryrouter: sync complete — ${result.unchanged} files unchanged`);
|
|
1260
|
-
}
|
|
1261
|
-
if (result.errors.length > 0) {
|
|
1262
|
-
api.logger.warn?.(`memoryrouter: sync had ${result.errors.length} errors`);
|
|
1263
|
-
}
|
|
1264
|
-
})
|
|
1265
|
-
.catch((err) => {
|
|
1266
|
-
api.logger.warn?.(`memoryrouter: sync error — ${err instanceof Error ? err.message : String(err)}`);
|
|
1267
|
-
});
|
|
1268
|
-
}
|
|
1148
|
+
start: () => {
|
|
1149
|
+
if (memoryKey) api.logger.info?.(`memoryrouter: active (key: ${memoryKey.slice(0, 6)}...)`);
|
|
1269
1150
|
},
|
|
1270
1151
|
stop: () => {
|
|
1271
1152
|
api.logger.info?.("memoryrouter: stopped");
|
package/package.json
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mr-memory",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.0",
|
|
4
4
|
"description": "MemoryRouter persistent memory plugin for OpenClaw — your AI remembers every conversation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"files": [
|
|
7
7
|
"README.md",
|
|
8
8
|
"index.ts",
|
|
9
9
|
"openclaw.plugin.json",
|
|
10
|
-
"upload.ts"
|
|
11
|
-
"sync.ts"
|
|
10
|
+
"upload.ts"
|
|
12
11
|
],
|
|
13
12
|
"keywords": [
|
|
14
13
|
"openclaw",
|
package/upload.ts
CHANGED
|
@@ -370,11 +370,10 @@ export async function runUpload(params: {
|
|
|
370
370
|
|
|
371
371
|
const workspacePath = params.workspacePath ?? process.cwd();
|
|
372
372
|
|
|
373
|
-
|
|
373
|
+
let files: string[];
|
|
374
374
|
if (targetPath) {
|
|
375
375
|
const resolved = path.resolve(targetPath);
|
|
376
376
|
const stat = await fs.stat(resolved);
|
|
377
|
-
let files: string[];
|
|
378
377
|
if (stat.isDirectory()) {
|
|
379
378
|
const allDirFiles = await fs.readdir(resolved, { recursive: true });
|
|
380
379
|
files = (allDirFiles as string[])
|
|
@@ -386,58 +385,19 @@ export async function runUpload(params: {
|
|
|
386
385
|
} else {
|
|
387
386
|
files = [resolved];
|
|
388
387
|
}
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
const { syncWorkspaceFiles } = await import("./sync.js");
|
|
401
|
-
const syncResult = await syncWorkspaceFiles({
|
|
402
|
-
workspaceDir: workspacePath,
|
|
403
|
-
endpoint,
|
|
404
|
-
memoryKey,
|
|
405
|
-
embeddings,
|
|
406
|
-
logging: true,
|
|
407
|
-
});
|
|
408
|
-
console.log(`\n Workspace: ${syncResult.uploaded} uploaded, ${syncResult.deleted} deleted, ${syncResult.unchanged} unchanged`);
|
|
409
|
-
if (syncResult.errors.length > 0) {
|
|
410
|
-
console.log(` ⚠️ ${syncResult.errors.length} errors`);
|
|
411
|
-
for (const err of syncResult.errors.slice(0, 3)) {
|
|
412
|
-
console.log(` • ${err}`);
|
|
413
|
-
}
|
|
414
|
-
}
|
|
388
|
+
} else if (params.hasBrainFlag && !params.hasWorkspaceFlag) {
|
|
389
|
+
// --brain only: upload sessions from brain path
|
|
390
|
+
files = await discoverBrainFiles(stateDir);
|
|
391
|
+
} else if (params.hasWorkspaceFlag && !params.hasBrainFlag) {
|
|
392
|
+
// --workspace only: upload workspace files only
|
|
393
|
+
files = await discoverWorkspaceFiles(workspacePath);
|
|
394
|
+
} else {
|
|
395
|
+
// No flags or both flags: upload both workspace + sessions
|
|
396
|
+
const wsFiles = await discoverWorkspaceFiles(workspacePath);
|
|
397
|
+
const brainFiles = await discoverBrainFiles(stateDir);
|
|
398
|
+
files = [...wsFiles, ...brainFiles];
|
|
415
399
|
}
|
|
416
400
|
|
|
417
|
-
// ── Session transcripts: batch upload (append-only, no sync needed) ──
|
|
418
|
-
if (doSessions) {
|
|
419
|
-
console.log("\n📝 Uploading session transcripts...");
|
|
420
|
-
const sessionFiles = await discoverBrainFiles(stateDir);
|
|
421
|
-
if (sessionFiles.length === 0) {
|
|
422
|
-
console.log(" No session files found.");
|
|
423
|
-
} else {
|
|
424
|
-
await uploadFilesLegacy(sessionFiles, memoryKey, endpoint, embeddings);
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
/**
|
|
430
|
-
* Legacy batch upload for session transcripts and specific-path uploads.
|
|
431
|
-
* Sessions are append-only, so they don't need source_hash tracking.
|
|
432
|
-
*/
|
|
433
|
-
async function uploadFilesLegacy(
|
|
434
|
-
files: string[],
|
|
435
|
-
memoryKey: string,
|
|
436
|
-
endpoint: string,
|
|
437
|
-
embeddings?: string,
|
|
438
|
-
): Promise<void> {
|
|
439
|
-
const uploadUrl = `${endpoint}/v1/memory/upload`;
|
|
440
|
-
|
|
441
401
|
if (files.length === 0) {
|
|
442
402
|
console.log("No files found to upload.");
|
|
443
403
|
return;
|
|
@@ -536,6 +496,7 @@ async function uploadFilesLegacy(
|
|
|
536
496
|
if (batchFailed > 0) {
|
|
537
497
|
const errHint = result.errors?.[0] ? ` (${result.errors[0].slice(0, 120)})` : "";
|
|
538
498
|
console.log(`⚠ ${batchStored} stored, ${batchFailed} skipped${errHint}`);
|
|
499
|
+
// Show detailed error diagnostics (up to 5 per batch)
|
|
539
500
|
if (result.errors && result.errors.length > 1) {
|
|
540
501
|
for (const err of result.errors.slice(0, 5)) {
|
|
541
502
|
console.log(` → ${err.slice(0, 200)}`);
|
package/sync.ts
DELETED
|
@@ -1,367 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* MemoryRouter Workspace Sync — source_hash-based file tracking.
|
|
3
|
-
*
|
|
4
|
-
* Tracks workspace files via SHA-256 content hashes. When files change,
|
|
5
|
-
* old chunks are deleted and new content is uploaded with the new hash.
|
|
6
|
-
* This enables clean replacement without stale chunk accumulation.
|
|
7
|
-
*
|
|
8
|
-
* IMPORTANT: source_hash is ONLY for workspace files, NEVER for sessions.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
import { readFile, writeFile, readdir, lstat, mkdir } from "node:fs/promises";
|
|
12
|
-
import { createHash } from "node:crypto";
|
|
13
|
-
import { join, relative, sep } from "node:path";
|
|
14
|
-
import { homedir } from "node:os";
|
|
15
|
-
|
|
16
|
-
// ── Types ──
|
|
17
|
-
|
|
18
|
-
export interface ManifestFileEntry {
|
|
19
|
-
source_hash: string;
|
|
20
|
-
size: number;
|
|
21
|
-
chunks: number;
|
|
22
|
-
uploaded_at: number;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
export interface Manifest {
|
|
26
|
-
version: number;
|
|
27
|
-
lastSync: number;
|
|
28
|
-
files: Record<string, ManifestFileEntry>;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export interface SyncResult {
|
|
32
|
-
uploaded: number;
|
|
33
|
-
deleted: number;
|
|
34
|
-
unchanged: number;
|
|
35
|
-
errors: string[];
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export interface SyncOptions {
|
|
39
|
-
workspaceDir: string;
|
|
40
|
-
endpoint: string;
|
|
41
|
-
memoryKey: string;
|
|
42
|
-
embeddings?: string;
|
|
43
|
-
logging?: boolean;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
// ── Constants ──
|
|
47
|
-
|
|
48
|
-
const MANIFEST_PATH = join(homedir(), ".openclaw", "mr-memory-manifest.json");
|
|
49
|
-
const MAX_FILE_SIZE = 1_000_000; // 1MB limit
|
|
50
|
-
const TEXT_EXTENSIONS = new Set(["md", "txt"]);
|
|
51
|
-
|
|
52
|
-
// ── Manifest I/O ──
|
|
53
|
-
|
|
54
|
-
export async function loadManifest(): Promise<Manifest> {
|
|
55
|
-
try {
|
|
56
|
-
const raw = await readFile(MANIFEST_PATH, "utf-8");
|
|
57
|
-
return JSON.parse(raw) as Manifest;
|
|
58
|
-
} catch {
|
|
59
|
-
// First run or corrupted — create empty manifest
|
|
60
|
-
return { version: 1, lastSync: 0, files: {} };
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export async function saveManifest(manifest: Manifest): Promise<void> {
|
|
65
|
-
manifest.lastSync = Date.now();
|
|
66
|
-
// Ensure directory exists
|
|
67
|
-
const dir = join(homedir(), ".openclaw");
|
|
68
|
-
await mkdir(dir, { recursive: true });
|
|
69
|
-
await writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
// ── Hash Computation ──
|
|
73
|
-
|
|
74
|
-
export function computeSourceHash(content: string): string {
|
|
75
|
-
const hash = createHash("sha256").update(content).digest("hex");
|
|
76
|
-
return hash.substring(0, 16); // First 16 hex chars, matching MR API format
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// ── File Discovery ──
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Discover ALL workspace files to sync:
|
|
83
|
-
* - All .md and .txt files in workspace root
|
|
84
|
-
* - All files recursively under memory/ directory
|
|
85
|
-
* - Excludes: hidden files/dirs, node_modules/, files > 1MB, binary files
|
|
86
|
-
*/
|
|
87
|
-
export async function discoverWorkspaceFiles(workspaceDir: string): Promise<string[]> {
|
|
88
|
-
const files: string[] = [];
|
|
89
|
-
|
|
90
|
-
async function walk(dir: string) {
|
|
91
|
-
let entries;
|
|
92
|
-
try {
|
|
93
|
-
entries = await readdir(dir, { withFileTypes: true });
|
|
94
|
-
} catch {
|
|
95
|
-
return; // Directory doesn't exist or not readable
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
for (const entry of entries) {
|
|
99
|
-
// Skip hidden dirs/files and node_modules
|
|
100
|
-
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
101
|
-
|
|
102
|
-
const fullPath = join(dir, entry.name);
|
|
103
|
-
|
|
104
|
-
if (entry.isDirectory()) {
|
|
105
|
-
await walk(fullPath);
|
|
106
|
-
} else if (entry.isFile()) {
|
|
107
|
-
// Only text files we want to index
|
|
108
|
-
const ext = entry.name.toLowerCase().split(".").pop() || "";
|
|
109
|
-
if (!TEXT_EXTENSIONS.has(ext)) continue;
|
|
110
|
-
|
|
111
|
-
// Check file size
|
|
112
|
-
try {
|
|
113
|
-
const stat = await lstat(fullPath);
|
|
114
|
-
if (stat.size > MAX_FILE_SIZE) continue;
|
|
115
|
-
if (stat.size === 0) continue; // Skip empty files
|
|
116
|
-
files.push(relative(workspaceDir, fullPath));
|
|
117
|
-
} catch {
|
|
118
|
-
continue; // Skip unreadable files
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
await walk(workspaceDir);
|
|
125
|
-
return files.sort();
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
// ── API Helpers ──
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* Upload file content with source_hash using isolated chunking.
|
|
132
|
-
* Uses POST /v1/memory/upload-source endpoint.
|
|
133
|
-
*/
|
|
134
|
-
export async function uploadSourceFile(
|
|
135
|
-
endpoint: string,
|
|
136
|
-
memoryKey: string,
|
|
137
|
-
content: string,
|
|
138
|
-
sourceHash: string,
|
|
139
|
-
embeddings?: string,
|
|
140
|
-
): Promise<{ stored: number; chunks: number }> {
|
|
141
|
-
const res = await fetch(`${endpoint}/v1/memory/upload-source`, {
|
|
142
|
-
method: "POST",
|
|
143
|
-
headers: {
|
|
144
|
-
Authorization: `Bearer ${memoryKey}`,
|
|
145
|
-
"Content-Type": "application/json",
|
|
146
|
-
...(embeddings && { "X-Embedding-Model": embeddings }),
|
|
147
|
-
},
|
|
148
|
-
body: JSON.stringify({ content, source_hash: sourceHash }),
|
|
149
|
-
});
|
|
150
|
-
|
|
151
|
-
if (!res.ok) {
|
|
152
|
-
const errText = await res.text().catch(() => "");
|
|
153
|
-
throw new Error(`Upload failed: HTTP ${res.status} — ${errText.slice(0, 200)}`);
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
const data = (await res.json()) as { stored: number; chunks: number; source_hash: string };
|
|
157
|
-
return { stored: data.stored || 0, chunks: data.chunks || data.stored || 0 };
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/**
|
|
161
|
-
* Delete all chunks with a given source_hash.
|
|
162
|
-
* Uses POST /v1/memory/source/delete (POST fallback for clients that can't send DELETE with body).
|
|
163
|
-
*/
|
|
164
|
-
export async function deleteBySourceHash(
|
|
165
|
-
endpoint: string,
|
|
166
|
-
memoryKey: string,
|
|
167
|
-
sourceHash: string,
|
|
168
|
-
): Promise<number> {
|
|
169
|
-
const res = await fetch(`${endpoint}/v1/memory/source/delete`, {
|
|
170
|
-
method: "POST",
|
|
171
|
-
headers: {
|
|
172
|
-
Authorization: `Bearer ${memoryKey}`,
|
|
173
|
-
"Content-Type": "application/json",
|
|
174
|
-
},
|
|
175
|
-
body: JSON.stringify({ source_hash: sourceHash }),
|
|
176
|
-
});
|
|
177
|
-
|
|
178
|
-
if (!res.ok) {
|
|
179
|
-
const errText = await res.text().catch(() => "");
|
|
180
|
-
throw new Error(`Delete failed: HTTP ${res.status} — ${errText.slice(0, 200)}`);
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
const data = (await res.json()) as { deleted: number; source_hash: string };
|
|
184
|
-
return data.deleted || 0;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// ── Main Sync Flow ──
|
|
188
|
-
|
|
189
|
-
/**
|
|
190
|
-
* Sync workspace files to MemoryRouter using source_hash tracking.
|
|
191
|
-
*
|
|
192
|
-
* Algorithm:
|
|
193
|
-
* 1. Load manifest (previous state)
|
|
194
|
-
* 2. Discover current workspace files, compute hashes
|
|
195
|
-
* 3. Diff current vs manifest:
|
|
196
|
-
* - NEW: upload with source_hash
|
|
197
|
-
* - CHANGED: delete old hash, upload new
|
|
198
|
-
* - DELETED: delete from MR
|
|
199
|
-
* - UNCHANGED: skip
|
|
200
|
-
* 4. Save updated manifest
|
|
201
|
-
*/
|
|
202
|
-
export async function syncWorkspaceFiles(options: SyncOptions): Promise<SyncResult> {
|
|
203
|
-
const { workspaceDir, endpoint, memoryKey, embeddings, logging } = options;
|
|
204
|
-
const log = (msg: string) => {
|
|
205
|
-
if (logging) console.log(msg);
|
|
206
|
-
};
|
|
207
|
-
|
|
208
|
-
const result: SyncResult = { uploaded: 0, deleted: 0, unchanged: 0, errors: [] };
|
|
209
|
-
|
|
210
|
-
// 1. Load manifest
|
|
211
|
-
const manifest = await loadManifest();
|
|
212
|
-
log(`memoryrouter: sync starting (${Object.keys(manifest.files).length} files in manifest)`);
|
|
213
|
-
|
|
214
|
-
// 2. Discover workspace files and compute hashes
|
|
215
|
-
const diskFiles = await discoverWorkspaceFiles(workspaceDir);
|
|
216
|
-
log(`memoryrouter: discovered ${diskFiles.length} workspace files`);
|
|
217
|
-
|
|
218
|
-
const diskState = new Map<string, { hash: string; size: number; content: string }>();
|
|
219
|
-
|
|
220
|
-
for (const relPath of diskFiles) {
|
|
221
|
-
const absPath = join(workspaceDir, relPath);
|
|
222
|
-
try {
|
|
223
|
-
const content = await readFile(absPath, "utf-8");
|
|
224
|
-
const trimmed = content.trim();
|
|
225
|
-
if (!trimmed) continue; // Skip empty files
|
|
226
|
-
|
|
227
|
-
const hash = computeSourceHash(content);
|
|
228
|
-
const size = Buffer.byteLength(content, "utf-8");
|
|
229
|
-
diskState.set(relPath, { hash, size, content });
|
|
230
|
-
} catch (err) {
|
|
231
|
-
result.errors.push(`Read error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
// 3a. Process disk files (new + changed)
|
|
236
|
-
for (const [relPath, { hash, size, content }] of diskState) {
|
|
237
|
-
const manifestEntry = manifest.files[relPath];
|
|
238
|
-
|
|
239
|
-
if (!manifestEntry) {
|
|
240
|
-
// NEW FILE — upload
|
|
241
|
-
try {
|
|
242
|
-
const { chunks } = await uploadSourceFile(endpoint, memoryKey, content, hash, embeddings);
|
|
243
|
-
manifest.files[relPath] = {
|
|
244
|
-
source_hash: hash,
|
|
245
|
-
size,
|
|
246
|
-
chunks,
|
|
247
|
-
uploaded_at: Date.now(),
|
|
248
|
-
};
|
|
249
|
-
result.uploaded++;
|
|
250
|
-
log(`memoryrouter: sync uploaded new file ${relPath} (${chunks} chunks)`);
|
|
251
|
-
} catch (err) {
|
|
252
|
-
result.errors.push(`Upload error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
|
|
253
|
-
}
|
|
254
|
-
} else if (manifestEntry.source_hash !== hash) {
|
|
255
|
-
// CHANGED FILE — delete old, upload new
|
|
256
|
-
try {
|
|
257
|
-
const deleted = await deleteBySourceHash(endpoint, memoryKey, manifestEntry.source_hash);
|
|
258
|
-
log(`memoryrouter: sync deleted old version of ${relPath} (${deleted} chunks)`);
|
|
259
|
-
} catch (err) {
|
|
260
|
-
// Log but continue — old chunks may have been deleted manually
|
|
261
|
-
log(`memoryrouter: sync delete warning for ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
try {
|
|
265
|
-
const { chunks } = await uploadSourceFile(endpoint, memoryKey, content, hash, embeddings);
|
|
266
|
-
manifest.files[relPath] = {
|
|
267
|
-
source_hash: hash,
|
|
268
|
-
size,
|
|
269
|
-
chunks,
|
|
270
|
-
uploaded_at: Date.now(),
|
|
271
|
-
};
|
|
272
|
-
result.uploaded++;
|
|
273
|
-
log(`memoryrouter: sync updated file ${relPath} (${chunks} new chunks)`);
|
|
274
|
-
} catch (err) {
|
|
275
|
-
result.errors.push(`Upload error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
|
|
276
|
-
}
|
|
277
|
-
} else {
|
|
278
|
-
// UNCHANGED — skip
|
|
279
|
-
result.unchanged++;
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
// 3b. Handle deleted files (in manifest but not on disk)
|
|
284
|
-
for (const relPath of Object.keys(manifest.files)) {
|
|
285
|
-
if (!diskState.has(relPath)) {
|
|
286
|
-
try {
|
|
287
|
-
const deleted = await deleteBySourceHash(endpoint, memoryKey, manifest.files[relPath].source_hash);
|
|
288
|
-
delete manifest.files[relPath];
|
|
289
|
-
result.deleted++;
|
|
290
|
-
log(`memoryrouter: sync deleted removed file ${relPath} (${deleted} chunks)`);
|
|
291
|
-
} catch (err) {
|
|
292
|
-
result.errors.push(`Delete error: ${relPath} — ${err instanceof Error ? err.message : String(err)}`);
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
// 4. Save updated manifest
|
|
298
|
-
await saveManifest(manifest);
|
|
299
|
-
|
|
300
|
-
log(
|
|
301
|
-
`memoryrouter: sync complete — ${result.uploaded} uploaded, ${result.deleted} deleted, ${result.unchanged} unchanged`,
|
|
302
|
-
);
|
|
303
|
-
|
|
304
|
-
return result;
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
// ── CLI Helper ──
|
|
308
|
-
|
|
309
|
-
/**
|
|
310
|
-
* Run sync from CLI with console output.
|
|
311
|
-
*/
|
|
312
|
-
export async function runSyncCli(options: SyncOptions): Promise<void> {
|
|
313
|
-
const { workspaceDir, embeddings } = options;
|
|
314
|
-
|
|
315
|
-
const modelLabel = embeddings ? `(model: ${embeddings})` : "(model: bge)";
|
|
316
|
-
console.log(`Syncing workspace to MemoryRouter ${modelLabel}...`);
|
|
317
|
-
console.log(` Workspace: ${workspaceDir}`);
|
|
318
|
-
|
|
319
|
-
try {
|
|
320
|
-
const result = await syncWorkspaceFiles({ ...options, logging: true });
|
|
321
|
-
|
|
322
|
-
console.log("\n────────────────────────────────");
|
|
323
|
-
console.log(`✅ Sync complete`);
|
|
324
|
-
console.log(` Uploaded: ${result.uploaded}`);
|
|
325
|
-
console.log(` Deleted: ${result.deleted}`);
|
|
326
|
-
console.log(` Unchanged: ${result.unchanged}`);
|
|
327
|
-
|
|
328
|
-
if (result.errors.length > 0) {
|
|
329
|
-
console.log(`\n⚠️ ${result.errors.length} errors:`);
|
|
330
|
-
for (const err of result.errors.slice(0, 5)) {
|
|
331
|
-
console.log(` • ${err}`);
|
|
332
|
-
}
|
|
333
|
-
if (result.errors.length > 5) {
|
|
334
|
-
console.log(` • ... and ${result.errors.length - 5} more`);
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
} catch (err) {
|
|
338
|
-
console.error(`\n❌ Sync failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
/**
|
|
343
|
-
* Show manifest status from CLI.
|
|
344
|
-
*/
|
|
345
|
-
export async function showManifestStatus(): Promise<void> {
|
|
346
|
-
const manifest = await loadManifest();
|
|
347
|
-
const fileCount = Object.keys(manifest.files).length;
|
|
348
|
-
|
|
349
|
-
console.log("Workspace Sync Manifest");
|
|
350
|
-
console.log("───────────────────────────────");
|
|
351
|
-
console.log(` Path: ${MANIFEST_PATH}`);
|
|
352
|
-
console.log(` Version: ${manifest.version}`);
|
|
353
|
-
console.log(` Last sync: ${manifest.lastSync ? new Date(manifest.lastSync).toLocaleString() : "never"}`);
|
|
354
|
-
console.log(` Files: ${fileCount}`);
|
|
355
|
-
|
|
356
|
-
if (fileCount > 0) {
|
|
357
|
-
console.log("\n Tracked files:");
|
|
358
|
-
const files = Object.entries(manifest.files).sort((a, b) => a[0].localeCompare(b[0]));
|
|
359
|
-
for (const [path, entry] of files.slice(0, 20)) {
|
|
360
|
-
const date = new Date(entry.uploaded_at).toLocaleDateString();
|
|
361
|
-
console.log(` • ${path} (${entry.chunks} chunks, ${date})`);
|
|
362
|
-
}
|
|
363
|
-
if (files.length > 20) {
|
|
364
|
-
console.log(` ... and ${files.length - 20} more`);
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
}
|