mr-memory 3.9.0 → 3.10.0

package/README.md

@@ -40,6 +40,19 @@ openclaw mr upload --workspace <dir>   # Custom workspace directory
 openclaw mr upload --brain <dir>       # Custom state dir (sessions from another agent)
 ```
 
+## Historical Import (agent-driven)
+
+Import existing OpenClaw conversation history with a strict division of labor: the server owns the extraction prompt (versioned, fetched at import time), your agent owns discovery/parsing of the local session files, and the CLI owns transport. The agent never uploads; the CLI never parses history.
+
+```bash
+openclaw mr import --instructions        # print the server's extraction prompt for your agent
+openclaw mr import --file extract.jsonl  # validate, preview, price quote, approve, upload
+openclaw mr import --file extract.jsonl --yes   # non-interactive approval
+openclaw mr import --dismiss             # permanently silence the import nudge
+```
+
+Guarantees: the extract is validated locally (four fields per record, `openclaw:` prefixed ids, epoch-ms timestamps); the CLI shows record count, date range, estimated tokens, and the server price quote and requires explicit approval before any write; the import id is content-derived and every batch carries an Idempotency-Key, so re-running after any interruption replays committed batches for free and resumes. Duplicates are never re-billed. The memory key never appears in the prompt or the extract file. When a vault is new, injected context includes a one-line import suggestion until an import completes or you `--dismiss`.
+
 ## How It Works
 
 When enabled, relay hooks retrieve memory before agent/model input and capture the direct user-to-assistant conversation at `agent_end`. Inference and provider API keys remain inside OpenClaw; the plugin does not route provider calls through MemoryRouter and is not a provider-key proxy.

package/import-transport.js

@@ -0,0 +1,457 @@
+/**
+ * Agent-driven historical import: the transport half (OpenClaw lane).
+ *
+ * Division of labor (locked): the SERVER owns the extraction prompt
+ * (GET /v1/memory/imports/instructions, versioned like reflection_contract);
+ * the HOST AGENT executes that prompt locally and writes a JSONL extract; THIS
+ * MODULE owns transport only: local validation, deterministic identity math
+ * (byte-for-byte with workers/src/services/import-identity.ts), create and
+ * approve, idempotent batch upload with resume, and the receipt. The agent
+ * never touches HTTP for upload and this module never parses history.
+ *
+ * The Memory Key never appears in the prompt or the extract file; it rides
+ * only in the Authorization header.
+ */
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import readline from "node:readline";
+import { importInstructions, importCreate, importApprove, importBatch, importStatus, importReceipt, importDismiss, } from "./memoryrouter-api.js";
+export const PLATFORM = "openclaw";
+export const PARSER_VERSION = 1;
+export const CHUNK_VERSION = 1;
+const BATCH_SIZE = 2000;
+const MAX_CONTENT_CHARS = 1000000;
+const MAX_UPLOAD_ATTEMPTS = 5;
+const INITIAL_BACKOFF_MS = 2000;
+const MAX_BACKOFF_MS = 30000;
+// ── identity math (mirrors the server byte-for-byte) ─────────────────────────
+function sha256hex(input) {
+    return createHash("sha256").update(input, "utf8").digest("hex");
+}
+function sortKeys(value) {
+    if (Array.isArray(value))
+        return value.map(sortKeys);
+    if (value && typeof value === "object") {
+        const out = {};
+        for (const key of Object.keys(value).sort()) {
+            out[key] = sortKeys(value[key]);
+        }
+        return out;
+    }
+    return value;
+}
+export function canonicalJson(value) {
+    return JSON.stringify(sortKeys(value));
+}
+export function normalizeText(input) {
+    return input.normalize("NFC").replace(/\r\n?/g, "\n").trim();
+}
+export function vaultFingerprint(memoryKey) {
+    return `mrvault_${sha256hex(memoryKey).slice(0, 16)}`;
+}
+function unescapeIdPart(s) {
+    return s.replace(/%3A/g, ":").replace(/%25/g, "%");
+}
+export function parseExternalId(id) {
+    const parts = id.split(":");
+    if (parts.length !== 4)
+        return null;
+    const [platform, conv, msg, ver] = parts;
+    if (!/^v\d+$/.test(ver))
+        return null;
+    return {
+        platform,
+        conversationId: unescapeIdPart(conv),
+        messageId: unescapeIdPart(msg),
+        parserVersion: parseInt(ver.slice(1), 10),
+    };
+}
+export function recordHash(fields) {
+    return sha256hex(canonicalJson({
+        conversationId: fields.conversationId,
+        messageId: fields.messageId,
+        role: fields.role,
+        text: normalizeText(fields.text),
+        timestamp: fields.timestamp,
+        v: PARSER_VERSION,
+    })).slice(0, 32);
+}
+export function conversationContentHash(fields) {
+    return sha256hex(canonicalJson(fields)).slice(0, 32);
+}
+export function archiveContentId(conversationContentHashes) {
+    const canonical = [...conversationContentHashes].sort().join("\n");
+    return `arc_${sha256hex(`v${PARSER_VERSION}\n${canonical}`).slice(0, 24)}`;
+}
+export function deriveImportId(archiveId, fingerprint) {
+    return `imp_${sha256hex(`${archiveId}|${fingerprint}|p${PARSER_VERSION}|c${CHUNK_VERSION}`).slice(0, 24)}`;
+}
+export function previewDigest(preview) {
+    return `pvd_${sha256hex(canonicalJson(preview)).slice(0, 32)}`;
+}
+export function readExtract(filePath, platform = PLATFORM) {
+    let raw;
+    try {
+        raw = readFileSync(filePath, "utf8");
+    }
+    catch (err) {
+        return { error: `could not read ${filePath}: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const records = [];
+    const seen = new Set();
+    const lines = raw.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (!line)
+            continue;
+        const lineNo = i + 1;
+        let p;
+        try {
+            p = JSON.parse(line);
+        }
+        catch {
+            return { error: `line ${lineNo}: not valid JSON` };
+        }
+        if (!p || typeof p !== "object" || Array.isArray(p))
+            return { error: `line ${lineNo}: record is not an object` };
+        const extras = Object.keys(p).filter((k) => !["external_id", "content", "timestamp", "role"].includes(k));
+        if (extras.length)
+            return { error: `line ${lineNo}: unexpected fields: ${extras.join(", ")}` };
+        if (typeof p.external_id !== "string" || !p.external_id)
+            return { error: `line ${lineNo}: external_id must be a string` };
+        const parsed = parseExternalId(p.external_id);
+        if (!parsed)
+            return { error: `line ${lineNo}: malformed external_id (expected ${platform}:<conversation>:<message>:v${PARSER_VERSION})` };
+        if (parsed.platform !== platform)
+            return { error: `line ${lineNo}: external_id platform must be ${platform}, got ${parsed.platform}` };
+        if (parsed.parserVersion !== PARSER_VERSION)
+            return { error: `line ${lineNo}: external_id version must be v${PARSER_VERSION}` };
+        if (typeof p.content !== "string" || p.content.length === 0)
+            return { error: `line ${lineNo}: content must be a non-empty string` };
+        if (p.content.length > MAX_CONTENT_CHARS)
+            return { error: `line ${lineNo}: content exceeds ${MAX_CONTENT_CHARS} chars` };
+        if (typeof p.timestamp !== "number" || !Number.isFinite(p.timestamp) || p.timestamp <= 0) {
+            return { error: `line ${lineNo}: timestamp must be a positive epoch-milliseconds number` };
+        }
+        if (p.timestamp < 1000000000000)
+            return { error: `line ${lineNo}: timestamp looks like seconds, not milliseconds (${p.timestamp})` };
+        if (p.role !== "user" && p.role !== "assistant")
+            return { error: `line ${lineNo}: role must be "user" or "assistant"` };
+        if (seen.has(p.external_id))
+            return { error: `line ${lineNo}: duplicate external_id ${p.external_id}` };
+        seen.add(p.external_id);
+        records.push({
+            externalId: p.external_id,
+            conversationId: parsed.conversationId,
+            messageId: parsed.messageId,
+            content: p.content,
+            role: p.role,
+            timestamp: p.timestamp,
+        });
+    }
+    if (records.length === 0)
+        return { error: "the extract file contains no records" };
+    return { records };
+}
+// ── identity + preview ───────────────────────────────────────────────────────
+export function computeIdentity(records, memoryKey, platform = PLATFORM) {
+    const byConversation = new Map();
+    for (const r of records) {
+        r.recordSha256 = recordHash({
+            conversationId: r.conversationId,
+            messageId: r.messageId,
+            role: r.role,
+            text: r.content,
+            timestamp: r.timestamp,
+        });
+        if (!byConversation.has(r.conversationId))
+            byConversation.set(r.conversationId, []);
+        byConversation.get(r.conversationId).push(r);
+    }
+    const convHashes = [];
+    for (const [conversationId, convRecords] of byConversation) {
+        convHashes.push(conversationContentHash({
+            conversationId,
+            canonicalPolicy: `agent-extract-${platform}`,
+            canonicalLeaf: null,
+            records: convRecords.map((r) => ({
+                externalId: r.externalId,
+                recordHash: r.recordSha256,
+                role: r.role,
+                timestamp: r.timestamp,
+            })),
+        }));
+    }
+    const archiveId = archiveContentId(convHashes);
+    const fingerprint = vaultFingerprint(memoryKey);
+    const importId = deriveImportId(archiveId, fingerprint);
+    return { archiveId, fingerprint, importId, conversations: byConversation.size };
+}
+export function estimateTokens(records) {
+    let total = 0;
+    for (const r of records)
+        total += Math.ceil(r.content.length / 4);
+    return total;
+}
+export function buildPreviewSummary(records) {
+    let earliest = Number.POSITIVE_INFINITY;
+    let latest = 0;
+    let user = 0;
+    for (const r of records) {
+        if (r.role === "user")
+            user++;
+        if (r.timestamp < earliest)
+            earliest = r.timestamp;
+        if (r.timestamp > latest)
+            latest = r.timestamp;
+    }
+    return {
+        records: records.length,
+        userRecords: user,
+        assistantRecords: records.length - user,
+        earliestMs: Number.isFinite(earliest) ? earliest : 0,
+        latestMs: latest,
+        estimatedTokens: estimateTokens(records),
+    };
+}
+export function toWireRecord(r, platform = PLATFORM) {
+    return JSON.stringify({
+        external_id: r.externalId,
+        record_sha256: r.recordSha256,
+        content: r.content,
+        role: r.role,
+        timestamp: r.timestamp,
+        source_platform: platform,
+        parser_version: PARSER_VERSION,
+        chunk_version: CHUNK_VERSION,
+    });
+}
+const defaultFetchers = {
+    importInstructions,
+    importCreate,
+    importApprove,
+    importBatch,
+    importStatus,
+    importReceipt,
+    importDismiss,
+};
+function fmtDate(ms) {
+    if (!Number.isFinite(ms) || ms <= 0)
+        return "unknown";
+    return new Date(ms).toISOString().slice(0, 10);
+}
+function fmtTokens(n) {
+    if (n >= 1000000)
+        return `${(n / 1000000).toFixed(1)}M`;
+    if (n >= 1000)
+        return `${(n / 1000).toFixed(1)}K`;
+    return String(n);
+}
+async function jsonOf(res) {
+    try {
+        return (await res.json());
+    }
+    catch {
+        return {};
+    }
+}
+function errorDetail(status, body) {
+    const parts = [body.error, body.message].filter(Boolean);
+    return parts.length ? parts.join(": ") : `HTTP ${status}`;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function ask(question) {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    return new Promise((resolve) => rl.question(question, (answer) => {
+        rl.close();
+        resolve(answer.trim().toLowerCase());
+    }));
+}
+/** Fetch and print the server-side extraction prompt for the host agent. */
+export async function runInstructions(options) {
+    const f = { ...defaultFetchers, ...(options.fetchers || {}) };
+    const log = options.log || ((...args) => console.log(...args));
+    const res = await f.importInstructions(options.endpoint, options.key, options.platform || PLATFORM);
+    const body = await jsonOf(res);
+    if (!res.ok)
+        throw new Error(`could not fetch import instructions (${errorDetail(res.status, body)})`);
+    if (!body.prompt)
+        throw new Error("instructions response carried no prompt");
+    log(body.prompt);
+    return { version: Number(body.import_prompt_version) || 0 };
+}
+/** Call the dismiss endpoint so the server-side nudge stops permanently. */
+export async function runDismiss(options) {
+    const f = { ...defaultFetchers, ...(options.fetchers || {}) };
+    const log = options.log || ((...args) => console.log(...args));
+    const res = await f.importDismiss(options.endpoint, options.key, options.platform || PLATFORM);
+    const body = await jsonOf(res);
+    if (!res.ok)
+        throw new Error(`dismiss failed (${errorDetail(res.status, body)})`);
+    log("Import suggestion dismissed permanently for this vault.");
+}
+/** The transport: validate, create, quote, approve, upload, receipt. */
+export async function runImportFile(options) {
+    const f = { ...defaultFetchers, ...(options.fetchers || {}) };
+    const log = options.log || ((...args) => console.log(...args));
+    const platform = options.platform || PLATFORM;
+    if (!options.file)
+        throw new Error("--file <path> is required");
+    const parsed = readExtract(options.file, platform);
+    if (parsed.error)
+        throw new Error(`invalid extract: ${parsed.error}`);
+    const records = parsed.records;
+    const identity = computeIdentity(records, options.key, platform);
+    const preview = buildPreviewSummary(records);
+    const clientPreviewDigest = previewDigest({
+        importId: identity.importId,
+        archiveId: identity.archiveId,
+        fingerprint: identity.fingerprint,
+        parserVersion: PARSER_VERSION,
+        chunkVersion: CHUNK_VERSION,
+        expectedRecords: preview.records,
+        expectedChunks: preview.records,
+        estimatedTokens: preview.estimatedTokens,
+        selection: null,
+        categories: [],
+    });
+    const createRes = await f.importCreate(options.endpoint, options.key, {
+        import_id: identity.importId,
+        archive_content_id: identity.archiveId,
+        archive_sha256: null,
+        parser_version: PARSER_VERSION,
+        chunk_version: CHUNK_VERSION,
+        source_platform: platform,
+        vault_fingerprint: identity.fingerprint,
+        preview_digest: clientPreviewDigest,
+        expected_records: preview.records,
+        expected_chunks: preview.records,
+        estimated_tokens: preview.estimatedTokens,
+        selection: null,
+        categories: [],
+    });
+    const created = await jsonOf(createRes);
+    if (!createRes.ok)
+        throw new Error(`import preflight failed (${errorDetail(createRes.status, created)})`);
+    if (created.preview_digest && created.preview_digest !== clientPreviewDigest) {
+        throw new Error("client/server preview digest mismatch (fail closed)");
+    }
+    const quote = (created.cost_quote || {});
+    log("");
+    log("Import preview");
+    log("----------------------------");
+    log(`Platform:          ${platform}`);
+    log(`Destination vault: ${identity.fingerprint} (opaque fingerprint)`);
+    log(`Conversations:     ${identity.conversations}`);
+    log(`Records:           ${preview.records} (user ${preview.userRecords}, assistant ${preview.assistantRecords})`);
+    log(`Date range:        ${fmtDate(preview.earliestMs)} to ${fmtDate(preview.latestMs)}`);
+    log(`Estimated tokens:  ${fmtTokens(preview.estimatedTokens)}`);
+    if (quote.available) {
+        log(`Estimated cost:    $${((quote.estimated_cost_cents || 0) / 100).toFixed(2)} (rate $${quote.rate_per_million_usd}/1M paid tokens, free remaining ${fmtTokens(quote.free_tokens_remaining || 0)})`);
+    }
+    else {
+        log(`Estimated cost:    unavailable (${quote.reason || "unknown"}); the server will refuse approval (fail closed)`);
+    }
+    log(`Import id:         ${identity.importId}`);
+    log("");
+    log("Re-running this command is always safe: committed batches replay for free and duplicates are never re-billed.");
+    log("");
+    if (!options.yes) {
+        if (options.nonInteractive || !process.stdin.isTTY) {
+            throw new Error("approval required: re-run with --yes to confirm the upload");
+        }
+        const answer = await ask("Proceed with the upload? [y/N]: ");
+        if (answer !== "y" && answer !== "yes") {
+            log("Aborted. Nothing was uploaded.");
+            return { aborted: true };
+        }
+    }
+    const approveRes = await f.importApprove(options.endpoint, options.key, identity.importId, String(created.approval_digest));
+    if (!approveRes.ok) {
+        const body = await jsonOf(approveRes);
+        // A completed import cannot be re-approved; that is the fully-settled
+        // resume case. Print the receipt instead of failing.
+        if (body.error === "import_completed") {
+            const receiptRes = await f.importReceipt(options.endpoint, options.key, identity.importId);
+            let receipt = null;
+            if (receiptRes.ok) {
+                const receiptBody = await jsonOf(receiptRes);
+                receipt = receiptBody.receipt ?? null;
+            }
+            log("This exact extract was already fully imported into this vault. Nothing to upload.");
+            if (receipt) {
+                log("Receipt:");
+                log(JSON.stringify(receipt, null, 2));
+            }
+            return { importId: identity.importId, complete: true, receipt };
+        }
+        throw new Error(`approval refused (${errorDetail(approveRes.status, body)})`);
+    }
+    const batches = [];
+    const size = options.batchSize || BATCH_SIZE;
+    for (let i = 0; i < records.length; i += size)
+        batches.push(records.slice(i, i + size));
+    const totals = { stored: 0, duplicate: 0, skipped: 0, failed: 0, conflict: 0 };
+    log(`Uploading ${records.length} record(s) in ${batches.length} batch(es)...`);
+    for (let i = 0; i < batches.length; i++) {
+        const jsonl = batches[i].map((r) => toWireRecord(r, platform)).join("\n");
+        const idempotencyKey = `${identity.importId}:b${i}`;
+        let res = null;
+        let backoff = INITIAL_BACKOFF_MS;
+        for (let attempt = 1; attempt <= MAX_UPLOAD_ATTEMPTS; attempt++) {
+            try {
+                res = await f.importBatch(options.endpoint, options.key, identity.importId, i, jsonl, idempotencyKey);
+            }
+            catch (err) {
+                res = null;
+                if (attempt === MAX_UPLOAD_ATTEMPTS) {
+                    throw new Error(`batch ${i} failed after ${MAX_UPLOAD_ATTEMPTS} attempts: ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+            if (res && (res.ok || (res.status < 500 && res.status !== 429)))
+                break;
+            if (attempt === MAX_UPLOAD_ATTEMPTS)
+                break;
+            await sleep(backoff);
+            backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
+        }
+        if (!res)
+            throw new Error(`batch ${i} received no response`);
+        const body = await jsonOf(res);
+        if (res.status === 402) {
+            throw new Error("payment required: out of tokens/credits. Add credits at https://app.memoryrouter.ai and re-run to resume.");
+        }
+        if (!res.ok)
+            throw new Error(`batch ${i} failed (${errorDetail(res.status, body)}). Re-run to resume; committed batches replay for free.`);
+        const s = (body.summary || {});
+        totals.stored += s.stored || 0;
+        totals.duplicate += s.duplicate || 0;
+        totals.skipped += s.skipped || 0;
+        totals.failed += s.failed || 0;
+        totals.conflict += s.conflict || 0;
+        log(`  batch ${i + 1}/${batches.length}${body.replayed ? " (replayed, free)" : ""}: stored ${s.stored || 0}, duplicate ${s.duplicate || 0}, failed ${s.failed || 0}`);
+    }
+    const statusRes = await f.importStatus(options.endpoint, options.key, identity.importId);
+    const statusBody = await jsonOf(statusRes);
+    const complete = Boolean(statusRes.ok && statusBody.complete);
+    let receipt = null;
+    if (complete) {
+        const receiptRes = await f.importReceipt(options.endpoint, options.key, identity.importId);
+        if (receiptRes.ok) {
+            const receiptBody = await jsonOf(receiptRes);
+            receipt = receiptBody.receipt ?? null;
+        }
+    }
+    log("");
+    log(`Done. stored=${totals.stored} duplicate=${totals.duplicate} skipped=${totals.skipped} failed=${totals.failed} conflict=${totals.conflict}`);
+    if (receipt) {
+        log("Receipt:");
+        log(JSON.stringify(receipt, null, 2));
+    }
+    else if (!complete) {
+        log("The import is not fully settled yet; re-run this command to resume/reconcile.");
+    }
+    return { importId: identity.importId, totals, complete, receipt };
+}