mr-memory 3.9.1 → 3.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -0
- package/auto-consolidate.js +292 -0
- package/auto-consolidate.ts +279 -0
- package/import-transport.js +457 -0
- package/import-transport.ts +511 -0
- package/index.js +81 -0
- package/index.ts +82 -0
- package/memoryrouter-api.js +51 -0
- package/memoryrouter-api.ts +59 -0
- package/openclaw.plugin.json +10 -0
- package/package.json +5 -1
|
@@ -0,0 +1,511 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agent-driven historical import: the transport half (OpenClaw lane).
|
|
3
|
+
*
|
|
4
|
+
* Division of labor (locked): the SERVER owns the extraction prompt
|
|
5
|
+
* (GET /v1/memory/imports/instructions, versioned like reflection_contract);
|
|
6
|
+
* the HOST AGENT executes that prompt locally and writes a JSONL extract; THIS
|
|
7
|
+
* MODULE owns transport only: local validation, deterministic identity math
|
|
8
|
+
* (byte-for-byte with workers/src/services/import-identity.ts), create and
|
|
9
|
+
* approve, idempotent batch upload with resume, and the receipt. The agent
|
|
10
|
+
* never touches HTTP for upload and this module never parses history.
|
|
11
|
+
*
|
|
12
|
+
* The Memory Key never appears in the prompt or the extract file; it rides
|
|
13
|
+
* only in the Authorization header.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { createHash } from "node:crypto";
|
|
17
|
+
import { readFileSync } from "node:fs";
|
|
18
|
+
import readline from "node:readline";
|
|
19
|
+
import {
|
|
20
|
+
importInstructions,
|
|
21
|
+
importCreate,
|
|
22
|
+
importApprove,
|
|
23
|
+
importBatch,
|
|
24
|
+
importStatus,
|
|
25
|
+
importReceipt,
|
|
26
|
+
importDismiss,
|
|
27
|
+
} from "./memoryrouter-api.js";
|
|
28
|
+
|
|
29
|
+
export const PLATFORM = "openclaw";
|
|
30
|
+
export const PARSER_VERSION = 1;
|
|
31
|
+
export const CHUNK_VERSION = 1;
|
|
32
|
+
const BATCH_SIZE = 2000;
|
|
33
|
+
const MAX_CONTENT_CHARS = 1000000;
|
|
34
|
+
const MAX_UPLOAD_ATTEMPTS = 5;
|
|
35
|
+
const INITIAL_BACKOFF_MS = 2000;
|
|
36
|
+
const MAX_BACKOFF_MS = 30000;
|
|
37
|
+
|
|
38
|
+
// ── identity math (mirrors the server byte-for-byte) ─────────────────────────
|
|
39
|
+
|
|
40
|
+
function sha256hex(input: string): string {
|
|
41
|
+
return createHash("sha256").update(input, "utf8").digest("hex");
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function sortKeys(value: unknown): unknown {
|
|
45
|
+
if (Array.isArray(value)) return value.map(sortKeys);
|
|
46
|
+
if (value && typeof value === "object") {
|
|
47
|
+
const out: Record<string, unknown> = {};
|
|
48
|
+
for (const key of Object.keys(value as Record<string, unknown>).sort()) {
|
|
49
|
+
out[key] = sortKeys((value as Record<string, unknown>)[key]);
|
|
50
|
+
}
|
|
51
|
+
return out;
|
|
52
|
+
}
|
|
53
|
+
return value;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function canonicalJson(value: unknown): string {
|
|
57
|
+
return JSON.stringify(sortKeys(value));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function normalizeText(input: string): string {
|
|
61
|
+
return input.normalize("NFC").replace(/\r\n?/g, "\n").trim();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function vaultFingerprint(memoryKey: string): string {
|
|
65
|
+
return `mrvault_${sha256hex(memoryKey).slice(0, 16)}`;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function unescapeIdPart(s: string): string {
|
|
69
|
+
return s.replace(/%3A/g, ":").replace(/%25/g, "%");
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function parseExternalId(id: string): { platform: string; conversationId: string; messageId: string; parserVersion: number } | null {
|
|
73
|
+
const parts = id.split(":");
|
|
74
|
+
if (parts.length !== 4) return null;
|
|
75
|
+
const [platform, conv, msg, ver] = parts;
|
|
76
|
+
if (!/^v\d+$/.test(ver)) return null;
|
|
77
|
+
return {
|
|
78
|
+
platform,
|
|
79
|
+
conversationId: unescapeIdPart(conv),
|
|
80
|
+
messageId: unescapeIdPart(msg),
|
|
81
|
+
parserVersion: parseInt(ver.slice(1), 10),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function recordHash(fields: { conversationId: string; messageId: string; role: string; text: string; timestamp: number }): string {
|
|
86
|
+
return sha256hex(
|
|
87
|
+
canonicalJson({
|
|
88
|
+
conversationId: fields.conversationId,
|
|
89
|
+
messageId: fields.messageId,
|
|
90
|
+
role: fields.role,
|
|
91
|
+
text: normalizeText(fields.text),
|
|
92
|
+
timestamp: fields.timestamp,
|
|
93
|
+
v: PARSER_VERSION,
|
|
94
|
+
}),
|
|
95
|
+
).slice(0, 32);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function conversationContentHash(fields: {
|
|
99
|
+
conversationId: string;
|
|
100
|
+
canonicalPolicy: string;
|
|
101
|
+
canonicalLeaf: string | null;
|
|
102
|
+
records: Array<{ externalId: string; recordHash: string; role: string; timestamp: number }>;
|
|
103
|
+
}): string {
|
|
104
|
+
return sha256hex(canonicalJson(fields)).slice(0, 32);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function archiveContentId(conversationContentHashes: string[]): string {
|
|
108
|
+
const canonical = [...conversationContentHashes].sort().join("\n");
|
|
109
|
+
return `arc_${sha256hex(`v${PARSER_VERSION}\n${canonical}`).slice(0, 24)}`;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function deriveImportId(archiveId: string, fingerprint: string): string {
|
|
113
|
+
return `imp_${sha256hex(`${archiveId}|${fingerprint}|p${PARSER_VERSION}|c${CHUNK_VERSION}`).slice(0, 24)}`;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function previewDigest(preview: unknown): string {
|
|
117
|
+
return `pvd_${sha256hex(canonicalJson(preview)).slice(0, 32)}`;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// ── extract validation ───────────────────────────────────────────────────────
|
|
121
|
+
|
|
122
|
+
export interface ExtractRecord {
|
|
123
|
+
externalId: string;
|
|
124
|
+
conversationId: string;
|
|
125
|
+
messageId: string;
|
|
126
|
+
content: string;
|
|
127
|
+
role: "user" | "assistant";
|
|
128
|
+
timestamp: number;
|
|
129
|
+
recordSha256?: string;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export function readExtract(filePath: string, platform: string = PLATFORM): { records?: ExtractRecord[]; error?: string } {
|
|
133
|
+
let raw: string;
|
|
134
|
+
try {
|
|
135
|
+
raw = readFileSync(filePath, "utf8");
|
|
136
|
+
} catch (err) {
|
|
137
|
+
return { error: `could not read ${filePath}: ${err instanceof Error ? err.message : String(err)}` };
|
|
138
|
+
}
|
|
139
|
+
const records: ExtractRecord[] = [];
|
|
140
|
+
const seen = new Set<string>();
|
|
141
|
+
const lines = raw.split("\n");
|
|
142
|
+
for (let i = 0; i < lines.length; i++) {
|
|
143
|
+
const line = lines[i].trim();
|
|
144
|
+
if (!line) continue;
|
|
145
|
+
const lineNo = i + 1;
|
|
146
|
+
let p: Record<string, unknown>;
|
|
147
|
+
try {
|
|
148
|
+
p = JSON.parse(line) as Record<string, unknown>;
|
|
149
|
+
} catch {
|
|
150
|
+
return { error: `line ${lineNo}: not valid JSON` };
|
|
151
|
+
}
|
|
152
|
+
if (!p || typeof p !== "object" || Array.isArray(p)) return { error: `line ${lineNo}: record is not an object` };
|
|
153
|
+
const extras = Object.keys(p).filter((k) => !["external_id", "content", "timestamp", "role"].includes(k));
|
|
154
|
+
if (extras.length) return { error: `line ${lineNo}: unexpected fields: ${extras.join(", ")}` };
|
|
155
|
+
if (typeof p.external_id !== "string" || !p.external_id) return { error: `line ${lineNo}: external_id must be a string` };
|
|
156
|
+
const parsed = parseExternalId(p.external_id);
|
|
157
|
+
if (!parsed) return { error: `line ${lineNo}: malformed external_id (expected ${platform}:<conversation>:<message>:v${PARSER_VERSION})` };
|
|
158
|
+
if (parsed.platform !== platform) return { error: `line ${lineNo}: external_id platform must be ${platform}, got ${parsed.platform}` };
|
|
159
|
+
if (parsed.parserVersion !== PARSER_VERSION) return { error: `line ${lineNo}: external_id version must be v${PARSER_VERSION}` };
|
|
160
|
+
if (typeof p.content !== "string" || p.content.length === 0) return { error: `line ${lineNo}: content must be a non-empty string` };
|
|
161
|
+
if (p.content.length > MAX_CONTENT_CHARS) return { error: `line ${lineNo}: content exceeds ${MAX_CONTENT_CHARS} chars` };
|
|
162
|
+
if (typeof p.timestamp !== "number" || !Number.isFinite(p.timestamp) || p.timestamp <= 0) {
|
|
163
|
+
return { error: `line ${lineNo}: timestamp must be a positive epoch-milliseconds number` };
|
|
164
|
+
}
|
|
165
|
+
if (p.timestamp < 1000000000000) return { error: `line ${lineNo}: timestamp looks like seconds, not milliseconds (${p.timestamp})` };
|
|
166
|
+
if (p.role !== "user" && p.role !== "assistant") return { error: `line ${lineNo}: role must be "user" or "assistant"` };
|
|
167
|
+
if (seen.has(p.external_id)) return { error: `line ${lineNo}: duplicate external_id ${p.external_id}` };
|
|
168
|
+
seen.add(p.external_id);
|
|
169
|
+
records.push({
|
|
170
|
+
externalId: p.external_id,
|
|
171
|
+
conversationId: parsed.conversationId,
|
|
172
|
+
messageId: parsed.messageId,
|
|
173
|
+
content: p.content,
|
|
174
|
+
role: p.role,
|
|
175
|
+
timestamp: p.timestamp,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
if (records.length === 0) return { error: "the extract file contains no records" };
|
|
179
|
+
return { records };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ── identity + preview ───────────────────────────────────────────────────────
|
|
183
|
+
|
|
184
|
+
export function computeIdentity(records: ExtractRecord[], memoryKey: string, platform: string = PLATFORM) {
|
|
185
|
+
const byConversation = new Map<string, ExtractRecord[]>();
|
|
186
|
+
for (const r of records) {
|
|
187
|
+
r.recordSha256 = recordHash({
|
|
188
|
+
conversationId: r.conversationId,
|
|
189
|
+
messageId: r.messageId,
|
|
190
|
+
role: r.role,
|
|
191
|
+
text: r.content,
|
|
192
|
+
timestamp: r.timestamp,
|
|
193
|
+
});
|
|
194
|
+
if (!byConversation.has(r.conversationId)) byConversation.set(r.conversationId, []);
|
|
195
|
+
byConversation.get(r.conversationId)!.push(r);
|
|
196
|
+
}
|
|
197
|
+
const convHashes: string[] = [];
|
|
198
|
+
for (const [conversationId, convRecords] of byConversation) {
|
|
199
|
+
convHashes.push(conversationContentHash({
|
|
200
|
+
conversationId,
|
|
201
|
+
canonicalPolicy: `agent-extract-${platform}`,
|
|
202
|
+
canonicalLeaf: null,
|
|
203
|
+
records: convRecords.map((r) => ({
|
|
204
|
+
externalId: r.externalId,
|
|
205
|
+
recordHash: r.recordSha256!,
|
|
206
|
+
role: r.role,
|
|
207
|
+
timestamp: r.timestamp,
|
|
208
|
+
})),
|
|
209
|
+
}));
|
|
210
|
+
}
|
|
211
|
+
const archiveId = archiveContentId(convHashes);
|
|
212
|
+
const fingerprint = vaultFingerprint(memoryKey);
|
|
213
|
+
const importId = deriveImportId(archiveId, fingerprint);
|
|
214
|
+
return { archiveId, fingerprint, importId, conversations: byConversation.size };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
export function estimateTokens(records: ExtractRecord[]): number {
|
|
218
|
+
let total = 0;
|
|
219
|
+
for (const r of records) total += Math.ceil(r.content.length / 4);
|
|
220
|
+
return total;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export function buildPreviewSummary(records: ExtractRecord[]) {
|
|
224
|
+
let earliest = Number.POSITIVE_INFINITY;
|
|
225
|
+
let latest = 0;
|
|
226
|
+
let user = 0;
|
|
227
|
+
for (const r of records) {
|
|
228
|
+
if (r.role === "user") user++;
|
|
229
|
+
if (r.timestamp < earliest) earliest = r.timestamp;
|
|
230
|
+
if (r.timestamp > latest) latest = r.timestamp;
|
|
231
|
+
}
|
|
232
|
+
return {
|
|
233
|
+
records: records.length,
|
|
234
|
+
userRecords: user,
|
|
235
|
+
assistantRecords: records.length - user,
|
|
236
|
+
earliestMs: Number.isFinite(earliest) ? earliest : 0,
|
|
237
|
+
latestMs: latest,
|
|
238
|
+
estimatedTokens: estimateTokens(records),
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export function toWireRecord(r: ExtractRecord, platform: string = PLATFORM): string {
|
|
243
|
+
return JSON.stringify({
|
|
244
|
+
external_id: r.externalId,
|
|
245
|
+
record_sha256: r.recordSha256,
|
|
246
|
+
content: r.content,
|
|
247
|
+
role: r.role,
|
|
248
|
+
timestamp: r.timestamp,
|
|
249
|
+
source_platform: platform,
|
|
250
|
+
parser_version: PARSER_VERSION,
|
|
251
|
+
chunk_version: CHUNK_VERSION,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// ── flows ────────────────────────────────────────────────────────────────────
|
|
256
|
+
|
|
257
|
+
type Fetchers = {
|
|
258
|
+
importInstructions: typeof importInstructions;
|
|
259
|
+
importCreate: typeof importCreate;
|
|
260
|
+
importApprove: typeof importApprove;
|
|
261
|
+
importBatch: typeof importBatch;
|
|
262
|
+
importStatus: typeof importStatus;
|
|
263
|
+
importReceipt: typeof importReceipt;
|
|
264
|
+
importDismiss: typeof importDismiss;
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
const defaultFetchers: Fetchers = {
|
|
268
|
+
importInstructions,
|
|
269
|
+
importCreate,
|
|
270
|
+
importApprove,
|
|
271
|
+
importBatch,
|
|
272
|
+
importStatus,
|
|
273
|
+
importReceipt,
|
|
274
|
+
importDismiss,
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
export interface ImportOptions {
|
|
278
|
+
endpoint: string;
|
|
279
|
+
key: string;
|
|
280
|
+
platform?: string;
|
|
281
|
+
file?: string;
|
|
282
|
+
yes?: boolean;
|
|
283
|
+
nonInteractive?: boolean;
|
|
284
|
+
batchSize?: number;
|
|
285
|
+
log?: (...args: unknown[]) => void;
|
|
286
|
+
fetchers?: Partial<Fetchers>;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function fmtDate(ms: number): string {
|
|
290
|
+
if (!Number.isFinite(ms) || ms <= 0) return "unknown";
|
|
291
|
+
return new Date(ms).toISOString().slice(0, 10);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function fmtTokens(n: number): string {
|
|
295
|
+
if (n >= 1000000) return `${(n / 1000000).toFixed(1)}M`;
|
|
296
|
+
if (n >= 1000) return `${(n / 1000).toFixed(1)}K`;
|
|
297
|
+
return String(n);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
async function jsonOf(res: Response): Promise<Record<string, any>> {
|
|
301
|
+
try {
|
|
302
|
+
return (await res.json()) as Record<string, any>;
|
|
303
|
+
} catch {
|
|
304
|
+
return {};
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function errorDetail(status: number, body: Record<string, unknown>): string {
|
|
309
|
+
const parts = [body.error, body.message].filter(Boolean);
|
|
310
|
+
return parts.length ? parts.join(": ") : `HTTP ${status}`;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function sleep(ms: number): Promise<void> {
|
|
314
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function ask(question: string): Promise<string> {
|
|
318
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
319
|
+
return new Promise((resolve) => rl.question(question, (answer) => {
|
|
320
|
+
rl.close();
|
|
321
|
+
resolve(answer.trim().toLowerCase());
|
|
322
|
+
}));
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** Fetch and print the server-side extraction prompt for the host agent. */
|
|
326
|
+
export async function runInstructions(options: ImportOptions): Promise<{ version: number }> {
|
|
327
|
+
const f = { ...defaultFetchers, ...(options.fetchers || {}) };
|
|
328
|
+
const log = options.log || ((...args: unknown[]) => console.log(...args));
|
|
329
|
+
const res = await f.importInstructions(options.endpoint, options.key, options.platform || PLATFORM);
|
|
330
|
+
const body = await jsonOf(res);
|
|
331
|
+
if (!res.ok) throw new Error(`could not fetch import instructions (${errorDetail(res.status, body)})`);
|
|
332
|
+
if (!body.prompt) throw new Error("instructions response carried no prompt");
|
|
333
|
+
log(body.prompt);
|
|
334
|
+
return { version: Number(body.import_prompt_version) || 0 };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/** Call the dismiss endpoint so the server-side nudge stops permanently. */
|
|
338
|
+
export async function runDismiss(options: ImportOptions): Promise<void> {
|
|
339
|
+
const f = { ...defaultFetchers, ...(options.fetchers || {}) };
|
|
340
|
+
const log = options.log || ((...args: unknown[]) => console.log(...args));
|
|
341
|
+
const res = await f.importDismiss(options.endpoint, options.key, options.platform || PLATFORM);
|
|
342
|
+
const body = await jsonOf(res);
|
|
343
|
+
if (!res.ok) throw new Error(`dismiss failed (${errorDetail(res.status, body)})`);
|
|
344
|
+
log("Import suggestion dismissed permanently for this vault.");
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** The transport: validate, create, quote, approve, upload, receipt. */
|
|
348
|
+
export async function runImportFile(options: ImportOptions): Promise<{
|
|
349
|
+
aborted?: boolean;
|
|
350
|
+
importId?: string;
|
|
351
|
+
totals?: Record<string, number>;
|
|
352
|
+
complete?: boolean;
|
|
353
|
+
receipt?: unknown;
|
|
354
|
+
}> {
|
|
355
|
+
const f = { ...defaultFetchers, ...(options.fetchers || {}) };
|
|
356
|
+
const log = options.log || ((...args: unknown[]) => console.log(...args));
|
|
357
|
+
const platform = options.platform || PLATFORM;
|
|
358
|
+
if (!options.file) throw new Error("--file <path> is required");
|
|
359
|
+
|
|
360
|
+
const parsed = readExtract(options.file, platform);
|
|
361
|
+
if (parsed.error) throw new Error(`invalid extract: ${parsed.error}`);
|
|
362
|
+
const records = parsed.records!;
|
|
363
|
+
const identity = computeIdentity(records, options.key, platform);
|
|
364
|
+
const preview = buildPreviewSummary(records);
|
|
365
|
+
|
|
366
|
+
const clientPreviewDigest = previewDigest({
|
|
367
|
+
importId: identity.importId,
|
|
368
|
+
archiveId: identity.archiveId,
|
|
369
|
+
fingerprint: identity.fingerprint,
|
|
370
|
+
parserVersion: PARSER_VERSION,
|
|
371
|
+
chunkVersion: CHUNK_VERSION,
|
|
372
|
+
expectedRecords: preview.records,
|
|
373
|
+
expectedChunks: preview.records,
|
|
374
|
+
estimatedTokens: preview.estimatedTokens,
|
|
375
|
+
selection: null,
|
|
376
|
+
categories: [],
|
|
377
|
+
});
|
|
378
|
+
const createRes = await f.importCreate(options.endpoint, options.key, {
|
|
379
|
+
import_id: identity.importId,
|
|
380
|
+
archive_content_id: identity.archiveId,
|
|
381
|
+
archive_sha256: null,
|
|
382
|
+
parser_version: PARSER_VERSION,
|
|
383
|
+
chunk_version: CHUNK_VERSION,
|
|
384
|
+
source_platform: platform,
|
|
385
|
+
vault_fingerprint: identity.fingerprint,
|
|
386
|
+
preview_digest: clientPreviewDigest,
|
|
387
|
+
expected_records: preview.records,
|
|
388
|
+
expected_chunks: preview.records,
|
|
389
|
+
estimated_tokens: preview.estimatedTokens,
|
|
390
|
+
selection: null,
|
|
391
|
+
categories: [],
|
|
392
|
+
});
|
|
393
|
+
const created = await jsonOf(createRes);
|
|
394
|
+
if (!createRes.ok) throw new Error(`import preflight failed (${errorDetail(createRes.status, created)})`);
|
|
395
|
+
if (created.preview_digest && created.preview_digest !== clientPreviewDigest) {
|
|
396
|
+
throw new Error("client/server preview digest mismatch (fail closed)");
|
|
397
|
+
}
|
|
398
|
+
const quote = (created.cost_quote || {}) as Record<string, any>;
|
|
399
|
+
|
|
400
|
+
log("");
|
|
401
|
+
log("Import preview");
|
|
402
|
+
log("----------------------------");
|
|
403
|
+
log(`Platform: ${platform}`);
|
|
404
|
+
log(`Destination vault: ${identity.fingerprint} (opaque fingerprint)`);
|
|
405
|
+
log(`Conversations: ${identity.conversations}`);
|
|
406
|
+
log(`Records: ${preview.records} (user ${preview.userRecords}, assistant ${preview.assistantRecords})`);
|
|
407
|
+
log(`Date range: ${fmtDate(preview.earliestMs)} to ${fmtDate(preview.latestMs)}`);
|
|
408
|
+
log(`Estimated tokens: ${fmtTokens(preview.estimatedTokens)}`);
|
|
409
|
+
if (quote.available) {
|
|
410
|
+
log(`Estimated cost: $${((quote.estimated_cost_cents || 0) / 100).toFixed(2)} (rate $${quote.rate_per_million_usd}/1M paid tokens, free remaining ${fmtTokens(quote.free_tokens_remaining || 0)})`);
|
|
411
|
+
} else {
|
|
412
|
+
log(`Estimated cost: unavailable (${quote.reason || "unknown"}); the server will refuse approval (fail closed)`);
|
|
413
|
+
}
|
|
414
|
+
log(`Import id: ${identity.importId}`);
|
|
415
|
+
log("");
|
|
416
|
+
log("Re-running this command is always safe: committed batches replay for free and duplicates are never re-billed.");
|
|
417
|
+
log("");
|
|
418
|
+
|
|
419
|
+
if (!options.yes) {
|
|
420
|
+
if (options.nonInteractive || !process.stdin.isTTY) {
|
|
421
|
+
throw new Error("approval required: re-run with --yes to confirm the upload");
|
|
422
|
+
}
|
|
423
|
+
const answer = await ask("Proceed with the upload? [y/N]: ");
|
|
424
|
+
if (answer !== "y" && answer !== "yes") {
|
|
425
|
+
log("Aborted. Nothing was uploaded.");
|
|
426
|
+
return { aborted: true };
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
const approveRes = await f.importApprove(options.endpoint, options.key, identity.importId, String(created.approval_digest));
|
|
431
|
+
if (!approveRes.ok) {
|
|
432
|
+
const body = await jsonOf(approveRes);
|
|
433
|
+
// A completed import cannot be re-approved; that is the fully-settled
|
|
434
|
+
// resume case. Print the receipt instead of failing.
|
|
435
|
+
if (body.error === "import_completed") {
|
|
436
|
+
const receiptRes = await f.importReceipt(options.endpoint, options.key, identity.importId);
|
|
437
|
+
let receipt: unknown = null;
|
|
438
|
+
if (receiptRes.ok) {
|
|
439
|
+
const receiptBody = await jsonOf(receiptRes);
|
|
440
|
+
receipt = receiptBody.receipt ?? null;
|
|
441
|
+
}
|
|
442
|
+
log("This exact extract was already fully imported into this vault. Nothing to upload.");
|
|
443
|
+
if (receipt) {
|
|
444
|
+
log("Receipt:");
|
|
445
|
+
log(JSON.stringify(receipt, null, 2));
|
|
446
|
+
}
|
|
447
|
+
return { importId: identity.importId, complete: true, receipt };
|
|
448
|
+
}
|
|
449
|
+
throw new Error(`approval refused (${errorDetail(approveRes.status, body)})`);
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
const batches: ExtractRecord[][] = [];
|
|
453
|
+
const size = options.batchSize || BATCH_SIZE;
|
|
454
|
+
for (let i = 0; i < records.length; i += size) batches.push(records.slice(i, i + size));
|
|
455
|
+
const totals = { stored: 0, duplicate: 0, skipped: 0, failed: 0, conflict: 0 };
|
|
456
|
+
log(`Uploading ${records.length} record(s) in ${batches.length} batch(es)...`);
|
|
457
|
+
for (let i = 0; i < batches.length; i++) {
|
|
458
|
+
const jsonl = batches[i].map((r) => toWireRecord(r, platform)).join("\n");
|
|
459
|
+
const idempotencyKey = `${identity.importId}:b${i}`;
|
|
460
|
+
let res: Response | null = null;
|
|
461
|
+
let backoff = INITIAL_BACKOFF_MS;
|
|
462
|
+
for (let attempt = 1; attempt <= MAX_UPLOAD_ATTEMPTS; attempt++) {
|
|
463
|
+
try {
|
|
464
|
+
res = await f.importBatch(options.endpoint, options.key, identity.importId, i, jsonl, idempotencyKey);
|
|
465
|
+
} catch (err) {
|
|
466
|
+
res = null;
|
|
467
|
+
if (attempt === MAX_UPLOAD_ATTEMPTS) {
|
|
468
|
+
throw new Error(`batch ${i} failed after ${MAX_UPLOAD_ATTEMPTS} attempts: ${err instanceof Error ? err.message : String(err)}`);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
if (res && (res.ok || (res.status < 500 && res.status !== 429))) break;
|
|
472
|
+
if (attempt === MAX_UPLOAD_ATTEMPTS) break;
|
|
473
|
+
await sleep(backoff);
|
|
474
|
+
backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
|
|
475
|
+
}
|
|
476
|
+
if (!res) throw new Error(`batch ${i} received no response`);
|
|
477
|
+
const body = await jsonOf(res);
|
|
478
|
+
if (res.status === 402) {
|
|
479
|
+
throw new Error("payment required: out of tokens/credits. Add credits at https://app.memoryrouter.ai and re-run to resume.");
|
|
480
|
+
}
|
|
481
|
+
if (!res.ok) throw new Error(`batch ${i} failed (${errorDetail(res.status, body)}). Re-run to resume; committed batches replay for free.`);
|
|
482
|
+
const s = (body.summary || {}) as Record<string, number>;
|
|
483
|
+
totals.stored += s.stored || 0;
|
|
484
|
+
totals.duplicate += s.duplicate || 0;
|
|
485
|
+
totals.skipped += s.skipped || 0;
|
|
486
|
+
totals.failed += s.failed || 0;
|
|
487
|
+
totals.conflict += s.conflict || 0;
|
|
488
|
+
log(` batch ${i + 1}/${batches.length}${body.replayed ? " (replayed, free)" : ""}: stored ${s.stored || 0}, duplicate ${s.duplicate || 0}, failed ${s.failed || 0}`);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
const statusRes = await f.importStatus(options.endpoint, options.key, identity.importId);
|
|
492
|
+
const statusBody = await jsonOf(statusRes);
|
|
493
|
+
const complete = Boolean(statusRes.ok && statusBody.complete);
|
|
494
|
+
let receipt: unknown = null;
|
|
495
|
+
if (complete) {
|
|
496
|
+
const receiptRes = await f.importReceipt(options.endpoint, options.key, identity.importId);
|
|
497
|
+
if (receiptRes.ok) {
|
|
498
|
+
const receiptBody = await jsonOf(receiptRes);
|
|
499
|
+
receipt = receiptBody.receipt ?? null;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
log("");
|
|
503
|
+
log(`Done. stored=${totals.stored} duplicate=${totals.duplicate} skipped=${totals.skipped} failed=${totals.failed} conflict=${totals.conflict}`);
|
|
504
|
+
if (receipt) {
|
|
505
|
+
log("Receipt:");
|
|
506
|
+
log(JSON.stringify(receipt, null, 2));
|
|
507
|
+
} else if (!complete) {
|
|
508
|
+
log("The import is not fully settled yet; re-run this command to resume/reconcile.");
|
|
509
|
+
}
|
|
510
|
+
return { importId: identity.importId, totals, complete, receipt };
|
|
511
|
+
}
|
package/index.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { readFile, lstat, writeFile, copyFile, mkdir } from "node:fs/promises";
|
|
11
11
|
import { commitReflections, consolidateCheckout, dateSearch, deleteMemories, deleteMemoryIds, getMemoryStats, ingestMemory, inspectMemory, prepareMemory, searchMemory, warmupMemory } from "./memoryrouter-api.js";
|
|
12
|
+
import { isAutoConsolidationSession, maybeTriggerAutoConsolidate } from "./auto-consolidate.js";
|
|
12
13
|
import { join, resolve, relative, isAbsolute, sep, dirname } from "node:path";
|
|
13
14
|
import os from "node:os";
|
|
14
15
|
const DEFAULT_ENDPOINT = "https://api.memoryrouter.ai";
|
|
@@ -64,6 +65,11 @@ function buildDebtNudgeLine(reflection) {
|
|
|
64
65
|
const kTokens = Math.max(1, Math.round(totalTokens / 1000));
|
|
65
66
|
return `[MemoryRouter] Consolidation suggested: ${kTokens}k unconsolidated tokens. Use the memory_reflect tool to consolidate them now.`;
|
|
66
67
|
}
|
|
68
|
+
function buildImportNudgeLine(importState) {
|
|
69
|
+
if (!importState || !importState.suggested)
|
|
70
|
+
return null;
|
|
71
|
+
return `[MemoryRouter] Historical import suggested: this machine has local conversation history not yet in your vault. Run: openclaw mr import --instructions`;
|
|
72
|
+
}
|
|
67
73
|
// ── memory_get helpers (copied from OpenClaw source — pure filesystem, no embeddings)
|
|
68
74
|
function isFileMissingError(err) {
|
|
69
75
|
return Boolean(err && typeof err === "object" && "code" in err && err.code === "ENOENT");
|
|
@@ -473,8 +479,27 @@ const memoryRouterPlugin = {
|
|
|
473
479
|
const logging = cfg?.logging ?? false;
|
|
474
480
|
const embeddings = cfg?.embeddings; // undefined = bge (default), "qwen" = qwen3-8b
|
|
475
481
|
const agentKeys = cfg?.agentKeys;
|
|
482
|
+
const autoConsolidate = cfg?.autoConsolidate !== false; // default true
|
|
476
483
|
const log = (msg) => { if (logging)
|
|
477
484
|
api.logger.info?.(msg); };
|
|
485
|
+
// Automatic consolidation trigger: deterministic plugin-side automation
|
|
486
|
+
// driving an isolated agent turn via the gateway's subagent runtime.
|
|
487
|
+
// Fire-and-forget; all rails (lock, cooldown, 3-batch bound) live in
|
|
488
|
+
// auto-consolidate.ts. Failures never propagate into the hook path.
|
|
489
|
+
function triggerAutoConsolidation(reflection, ctx, activeKey) {
|
|
490
|
+
try {
|
|
491
|
+
maybeTriggerAutoConsolidate(reflection ?? null, {
|
|
492
|
+
autoConsolidate,
|
|
493
|
+
sessionKey: ctx?.sessionKey,
|
|
494
|
+
runtime: api.runtime?.subagent,
|
|
495
|
+
agentId: ctx?.agentId || "main",
|
|
496
|
+
endpoint,
|
|
497
|
+
key: activeKey,
|
|
498
|
+
embeddings,
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
catch { /* never blocks or crashes the host */ }
|
|
502
|
+
}
|
|
478
503
|
// Resolve the correct memory key for a given agent context.
|
|
479
504
|
// Multi-agent gateways: each agent gets its own key via agentKeys map.
|
|
480
505
|
// Single-agent: falls back to the global key.
|
|
@@ -664,6 +689,11 @@ If setting up MemoryRouter for another agent:
|
|
|
664
689
|
// When PR #24122 merges, OpenClaw will use the returned prependContext.
|
|
665
690
|
// This gives forward compatibility — no plugin update needed.
|
|
666
691
|
api.on("llm_input", async (event, ctx) => {
|
|
692
|
+
// Auto-consolidation runs are single-purpose tool sessions: no memory
|
|
693
|
+
// injection (their prompt must stay exactly the consolidation
|
|
694
|
+
// mission) and no re-trigger loops.
|
|
695
|
+
if (isAutoConsolidationSession(ctx?.sessionKey))
|
|
696
|
+
return;
|
|
667
697
|
// Skip the first call — before_prompt_build already handled it
|
|
668
698
|
// (before_prompt_build includes workspace+tools+skills for accurate billing)
|
|
669
699
|
if (promptBuildFiredThisRun) {
|
|
@@ -735,6 +765,7 @@ If setting up MemoryRouter for another agent:
|
|
|
735
765
|
return;
|
|
736
766
|
}
|
|
737
767
|
const data = (await res.json());
|
|
768
|
+
triggerAutoConsolidation(data.reflection, ctx, activeKey);
|
|
738
769
|
if (data.context) {
|
|
739
770
|
const elapsed = Date.now() - startMs;
|
|
740
771
|
api.logger.info?.(`memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens, ${elapsed}ms) [llm_input]`);
|
|
@@ -744,6 +775,9 @@ If setting up MemoryRouter for another agent:
|
|
|
744
775
|
const nudge = buildDebtNudgeLine(data.reflection);
|
|
745
776
|
if (nudge)
|
|
746
777
|
contextText = `${contextText}\n\n${nudge}`;
|
|
778
|
+
const importNudge = buildImportNudgeLine(data.import);
|
|
779
|
+
if (importNudge)
|
|
780
|
+
contextText = `${contextText}\n\n${importNudge}`;
|
|
747
781
|
const wrapped = wrapForInjection(contextText);
|
|
748
782
|
return { appendSystemContext: wrapped };
|
|
749
783
|
}
|
|
@@ -754,6 +788,8 @@ If setting up MemoryRouter for another agent:
|
|
|
754
788
|
});
|
|
755
789
|
// ── before_prompt_build: fires once per run (primary, includes full billing context)
|
|
756
790
|
api.on("before_prompt_build", async (event, ctx) => {
|
|
791
|
+
if (isAutoConsolidationSession(ctx?.sessionKey))
|
|
792
|
+
return;
|
|
757
793
|
promptBuildFiredThisRun = true;
|
|
758
794
|
const activeKey = resolveKey(ctx);
|
|
759
795
|
if (!activeKey)
|
|
@@ -844,6 +880,7 @@ If setting up MemoryRouter for another agent:
|
|
|
844
880
|
return;
|
|
845
881
|
}
|
|
846
882
|
const data = (await res.json());
|
|
883
|
+
triggerAutoConsolidation(data.reflection, ctx, activeKey);
|
|
847
884
|
if (data.context) {
|
|
848
885
|
const elapsed = Date.now() - startMs;
|
|
849
886
|
api.logger.info?.(`memoryrouter: injected ${data.memories_found || 0} memories (${data.memory_tokens || 0} tokens, ${elapsed}ms)`);
|
|
@@ -853,6 +890,9 @@ If setting up MemoryRouter for another agent:
|
|
|
853
890
|
const nudge = buildDebtNudgeLine(data.reflection);
|
|
854
891
|
if (nudge)
|
|
855
892
|
contextText = `${contextText}\n\n${nudge}`;
|
|
893
|
+
const importNudge = buildImportNudgeLine(data.import);
|
|
894
|
+
if (importNudge)
|
|
895
|
+
contextText = `${contextText}\n\n${importNudge}`;
|
|
856
896
|
const wrapped = wrapForInjection(contextText);
|
|
857
897
|
return { appendSystemContext: wrapped };
|
|
858
898
|
}
|
|
@@ -865,6 +905,9 @@ If setting up MemoryRouter for another agent:
|
|
|
865
905
|
// Core: agent_end — store this turn's conversation
|
|
866
906
|
// ==================================================================
|
|
867
907
|
api.on("agent_end", async (event, ctx) => {
|
|
908
|
+
// Never store the auto-consolidation session's own chatter as memory.
|
|
909
|
+
if (isAutoConsolidationSession(ctx?.sessionKey))
|
|
910
|
+
return;
|
|
868
911
|
const activeKey = resolveKey(ctx);
|
|
869
912
|
if (!activeKey)
|
|
870
913
|
return; // No key for this agent
|
|
@@ -1565,6 +1608,44 @@ If setting up MemoryRouter for another agent:
|
|
|
1565
1608
|
console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1566
1609
|
}
|
|
1567
1610
|
});
|
|
1611
|
+
mr.command("import")
|
|
1612
|
+
.description("Import historical OpenClaw conversations into your vault (agent-driven extraction + CLI transport)")
|
|
1613
|
+
.option("--instructions", "Fetch and print the server's extraction prompt for the host agent to execute")
|
|
1614
|
+
.option("--file <path>", "Upload a JSONL extract: preview, price quote, explicit approval, idempotent batches, resume")
|
|
1615
|
+
.option("--dismiss", "Permanently silence the import suggestion for this vault")
|
|
1616
|
+
.option("--yes", "Skip the interactive approval prompt (required for non-interactive --file runs)")
|
|
1617
|
+
.option("--key <key>", "Override memory key (for subagent vaults)")
|
|
1618
|
+
.action(async (opts) => {
|
|
1619
|
+
const effectiveKey = opts.key || process.env.MEMORYROUTER_API_KEY || memoryKey;
|
|
1620
|
+
if (!effectiveKey) {
|
|
1621
|
+
console.error("Not configured. Run: openclaw mr <key> or pass --key <key>");
|
|
1622
|
+
return;
|
|
1623
|
+
}
|
|
1624
|
+
const modes = [Boolean(opts.instructions), opts.file !== undefined, Boolean(opts.dismiss)].filter(Boolean).length;
|
|
1625
|
+
if (modes > 1) {
|
|
1626
|
+
console.error("Choose one of --instructions, --file <path>, or --dismiss");
|
|
1627
|
+
return;
|
|
1628
|
+
}
|
|
1629
|
+
try {
|
|
1630
|
+
const transport = await import("./import-transport.js");
|
|
1631
|
+
if (opts.dismiss) {
|
|
1632
|
+
await transport.runDismiss({ endpoint, key: effectiveKey });
|
|
1633
|
+
return;
|
|
1634
|
+
}
|
|
1635
|
+
if (opts.file) {
|
|
1636
|
+
await transport.runImportFile({ endpoint, key: effectiveKey, file: opts.file, yes: Boolean(opts.yes) });
|
|
1637
|
+
return;
|
|
1638
|
+
}
|
|
1639
|
+
await transport.runInstructions({ endpoint, key: effectiveKey });
|
|
1640
|
+
console.log("");
|
|
1641
|
+
console.log("Execute the prompt above with your agent, then upload the extract:");
|
|
1642
|
+
console.log(" openclaw mr import --file <path>");
|
|
1643
|
+
}
|
|
1644
|
+
catch (err) {
|
|
1645
|
+
console.error(`Import failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1646
|
+
process.exitCode = 1;
|
|
1647
|
+
}
|
|
1648
|
+
});
|
|
1568
1649
|
mr.command("reflect")
|
|
1569
1650
|
.description("Show reflection debt status (consolidation runs via the agent's memory_reflect tool)")
|
|
1570
1651
|
.option("--key <key>", "Override memory key (for subagent vaults)")
|