@echomem/mcp 1.3.0 → 1.3.1

package/dist/migrate.js

@@ -4,17 +4,18 @@
  *
  * Why this lives in the bridge (client-side): the session logs exist ONLY on the user's machine
  * (~/.codex/sessions, ~/.claude/projects). The bridge discovers each session, assembles it into a
- * `## `-turn transcript, and feeds it to the EXISTING durable import queue (the same one the extension
- * uses): POST /api/extension/import-sessions creates one import_jobs row per session, then the bridge
+ * text-turn-only `## ` transcript, and feeds it to the EXISTING durable import queue
+ * (the same one the extension uses): POST /api/extension/import-sessions creates one import_jobs row per session, then the bridge
  * POSTs each transcript to /api/extension/import-jobs/{id}/run. The web dashboard polls
  * GET /import-sessions/{id} for live progress — the bridge owns local-file access, the server owns the
  * queue + status. (Headless CLI also works; it just shows progress in the terminal.)
  *
  * Idempotency: the queue dedups by (platform, conversationId) → source_hash (a repeat run hits a
  * non-fatal DUPLICATE_SOURCE), and a local ledger (~/.echomem/migrate-ledger.json) skips done+unchanged
- * files WITHOUT a round-trip. Turns are joined with a SINGLE "\n" so the transcript stays byte-stable
- * across runs (guarded by test/migrate.test.mjs). The run is sequential (throttled < 30/min to the /run
- * limit) and stops cleanly if the encrypted vault's key expires mid-run.
+ * files WITHOUT a round-trip. A local metrics log (~/.echomem/migrate-metrics.jsonl) records per-job
+ * timings and sizes, but never transcript text. Turns are joined with a SINGLE "\n" so the transcript
+ * stays byte-stable across runs (guarded by test/migrate.test.mjs). The run is sequential (throttled
+ * < 30/min to the /run limit) and stops cleanly if the encrypted vault's key expires mid-run.
  * NOTE: client-pull — the queue advances only while the bridge runs; there is no server-side worker.
  */
 import fs from "node:fs";
@@ -29,6 +30,8 @@ import { walk, eachLine } from "./report.js";
 const API_BASE = (process.env.ECHO_API_BASE_URL || "https://echo-mem-chrome.vercel.app").replace(/\/$/, "");
 const RATE_MAX = 28; // stay under the import-jobs /run limit of 30 / 60s
 const RATE_WINDOW_MS = 60_000;
+const APPROX_CHARS_PER_TOKEN = 4;
+const ACTIVE_SESSION_GRACE_MS = 5 * 60_000;
 // ---------------------------------------------------------------------------
 // Assemblers — reconstruct a `## `-turn transcript from a raw session log.
 // ---------------------------------------------------------------------------
@@ -42,7 +45,7 @@ function cleanCodexUser(msg) {
     s = s.replace(/<(environment_context|user_instructions|permissions|app-context)>[\s\S]*?<\/\1>/g, "");
     return s.trim();
 }
-/** Codex rollout-*.jsonl → one session. session_meta is TOP-LEVEL `o.type`; turns are payload events. */
+/** Codex rollout-*.jsonl → one session. User/assistant text turns are sent; tool/process events stay local. */
 export function assembleCodex(file) {
     let sessionId = null;
     let cwd = null;
@@ -87,14 +90,14 @@ export function assembleCodex(file) {
         conversationKey: `codex:${sessionId || sha16(file)}`,
         cwd: normalizeCwd(cwd),
         firstTs,
-        title: title || "Codex session",
+        title: title || "Codex text turns",
         rawData: turns.join("\n"),
         turnCount: turns.length,
         size: stat.size,
         mtimeMs: stat.mtimeMs,
     };
 }
-/** Claude Code <uuid>.jsonl → one session. Only user/assistant lines are turns; thinking is dropped. */
+/** Claude Code <uuid>.jsonl → one session. User/assistant text turns are sent; tool calls/results/thinking stay local. */
 export function assembleClaude(file) {
     let sessionId = null;
     let cwd = null;
@@ -113,7 +116,7 @@ export function assembleClaude(file) {
             if (typeof c === "string")
                 body = c.trim();
             else if (Array.isArray(c)) {
-                // tool_result-only carriers are NOT real user prompts → skip; keep any genuine text blocks.
+                // Keep only genuine user text blocks. tool_result carriers stay local.
                 body = c.filter((b) => b && b.type === "text" && b.text).map((b) => b.text).join("\n").trim();
             }
             if (!body)
@@ -132,9 +135,7 @@ export function assembleClaude(file) {
                 for (const b of c) {
                     if (b && b.type === "text" && b.text)
                         parts.push(b.text);
-                    else if (b && b.type === "tool_use")
-                        parts.push(`[tool: ${b.name || "?"} ${truncate(JSON.stringify(b.input || {}), 120)}]`);
-                    // thinking blocks dropped (bulk of tokens, low memory value)
+                    // thinking and tool_use blocks are process/code noise, not assistant output.
                 }
                 body = parts.join("\n").trim();
             }
@@ -147,7 +148,7 @@ export function assembleClaude(file) {
             if (!firstTs)
                 firstTs = ts;
         }
-        // queue-operation / attachment / ai-title / last-prompt / mode / summary / unknown → ignored
+        // tool_result carriers, queue-operation, attachment, ai-title, last-prompt, mode, summary, unknown → ignored
     });
     if (!turns.length)
         return null;
@@ -158,7 +159,7 @@ export function assembleClaude(file) {
         conversationKey: `claude:${sessionId || sha16(file)}`,
         cwd: normalizeCwd(cwd),
         firstTs,
-        title: title || "Claude Code session",
+        title: title || "Claude text turns",
         rawData: turns.join("\n"),
         turnCount: turns.length,
         size: stat.size,
@@ -219,6 +220,67 @@ function saveLedger(l) {
         /* best effort */
     }
 }
+export function defaultMigrationMetricsPath() {
+    return path.join(echoConfigDir(), "migrate-metrics.jsonl");
+}
+function sourceMtimeIso(s) {
+    return s.mtimeMs ? new Date(s.mtimeMs).toISOString() : null;
+}
+function projectName(s) {
+    if (s.cwd)
+        return path.basename(s.cwd);
+    const parent = path.basename(path.dirname(s.filePath));
+    return parent || null;
+}
+export function buildMigrationMetric(args) {
+    const s = args.session;
+    const metric = {
+        schemaVersion: 1,
+        recordedAt: new Date().toISOString(),
+        runId: args.runId,
+        importSessionId: args.importSessionId,
+        jobId: args.jobId,
+        index: args.index,
+        total: args.total,
+        status: args.status,
+        apiBase: API_BASE,
+        source: s.source,
+        conversationKey: s.conversationKey,
+        conversationId: bareId(s),
+        firstTs: s.firstTs,
+        date: s.firstTs ? s.firstTs.slice(0, 10) : null,
+        cwd: s.cwd,
+        project: projectName(s),
+        filePath: s.filePath,
+        rawDataChars: s.rawData.length,
+        approxInputTokens: approxTokens(s.rawData.length),
+        textTurns: s.turnCount,
+        sourceFileBytes: s.size,
+        sourceMtimeMs: s.mtimeMs,
+        sourceMtimeIso: sourceMtimeIso(s),
+    };
+    if (args.durationMs !== undefined)
+        metric.durationMs = args.durationMs;
+    if (args.processingTimeMs !== undefined)
+        metric.processingTimeMs = args.processingTimeMs;
+    if (args.ttfmMs !== undefined)
+        metric.ttfmMs = args.ttfmMs;
+    if (args.memories !== undefined)
+        metric.memories = args.memories;
+    if (args.alreadyDone !== undefined)
+        metric.alreadyDone = args.alreadyDone;
+    if (args.duplicate !== undefined)
+        metric.duplicate = args.duplicate;
+    if (args.error !== undefined)
+        metric.error = truncate(args.error, 240);
+    if (args.selection)
+        metric.selection = args.selection;
+    return metric;
+}
+export function appendMigrationMetric(filePath, metric) {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
+    fs.appendFileSync(filePath, JSON.stringify(metric) + "\n", { mode: 0o600 });
+}
 // ---------------------------------------------------------------------------
 // CLI
 // ---------------------------------------------------------------------------
@@ -227,114 +289,337 @@ const color = (enabled) => {
     return { bold: w("1"), dim: w("2"), red: w("31"), green: w("32"), cyan: w("36"), yellow: w("33") };
 };
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const humanNum = (n) => Math.round(n).toLocaleString();
+function humanDuration(ms) {
+    if (!Number.isFinite(ms) || ms <= 0)
+        return "0s";
+    const sec = Math.round(ms / 1000);
+    if (sec < 90)
+        return `${sec}s`;
+    const min = Math.floor(sec / 60);
+    const rem = sec % 60;
+    if (min < 90)
+        return rem ? `${min}m ${rem}s` : `${min}m`;
+    const hours = Math.floor(min / 60);
+    const mins = min % 60;
+    return mins ? `${hours}h ${mins}m` : `${hours}h`;
+}
+export function formatEta(seconds) {
+    if (!Number.isFinite(seconds) || seconds <= 0)
+        return "complete";
+    if (seconds < 120)
+        return "under 2 minutes";
+    const minutes = Math.ceil(seconds / 60);
+    if (minutes < 60)
+        return `about ${minutes} minutes`;
+    const hours = Math.floor(minutes / 60);
+    const mins = minutes % 60;
+    return mins ? `about ${hours} hr ${mins} min` : `about ${hours} hr`;
+}
+function printMigrationEstimate(e, useJson) {
+    if (useJson) {
+        console.log(JSON.stringify(e, null, 2));
+        return;
+    }
+    console.log("");
+    console.log("Migration estimate (local metadata only; no transcripts uploaded)");
+    console.log(`  Sessions: ${humanNum(e.sessions)} total · ${humanNum(e.pending)} pending · ${humanNum(e.alreadyMigrated)} already processed`);
+    console.log(`  Sources: ${humanNum(e.pendingCodex)} Codex + ${humanNum(e.pendingClaudeCode)} Claude Code pending`);
+    console.log(`  Assembled transcript size: ${humanNum(e.chars.total)} chars ≈ ${humanNum(e.approxInputTokens.total)} input tokens`);
+    console.log(`  Size percentiles: p50 ${humanNum(e.chars.p50)} chars · p90 ${humanNum(e.chars.p90)} · p95 ${humanNum(e.chars.p95)} · max ${humanNum(e.chars.max)}`);
+    console.log(`  Turns: ${humanNum(e.turns.total)} total · p50 ${humanNum(e.turns.p50)} · p90 ${humanNum(e.turns.p90)} · max ${humanNum(e.turns.max)}`);
+    console.log("");
+    console.log("  Buckets:");
+    for (const b of e.buckets) {
+        console.log(`    ${b.label.padEnd(22)} ${String(b.count).padStart(4)} sessions · ${humanNum(b.chars).padStart(12)} chars · ≈${humanNum(b.approxInputTokens)} tokens`);
+    }
+    console.log("");
+    console.log(`  Queue floor from request throttle: ${humanDuration(e.queue.throttleFloorMinutes * 60_000)} (${e.queue.runRequestLimitPerMinute}/min, client-sequential).`);
+    console.log(`  Large-session risk: ${e.queue.largeSessionCount} sessions >120k chars; ${e.queue.hugeSessionCount} >${humanNum(e.queue.needsChunkingAboveChars)} chars should be chunked or sampled before full migration.`);
+    console.log("  For a measured ETA, run a small real sample: echomem-mcp migrate --limit 5 --yes");
+}
 function promptYesNo(question) {
     const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
     return new Promise((resolve) => rl.question(question, (a) => { rl.close(); resolve(!/^n/i.test(a.trim())); }));
 }
-export async function cmdMigrate(flags) {
-    const c = color(process.stdout.isTTY === true && !process.env.NO_COLOR && flags["no-color"] !== true);
-    const store = new KeyStore();
-    // Discover + filter (pure local — discovery and --dry-run need no login).
-    let sessions = discoverSessions();
-    const since = typeof flags.since === "string" ? flags.since : undefined;
-    if (since)
-        sessions = sessions.filter((s) => (s.firstTs || "").slice(0, 10) >= since);
-    const limit = typeof flags.limit === "string" ? parseInt(flags.limit, 10) : undefined;
-    if (limit && limit > 0)
-        sessions = sessions.slice(0, limit);
-    if (!sessions.length) {
-        console.log("No local Codex/Claude Code sessions found to migrate.");
-        return;
-    }
-    const ledger = loadLedger();
-    const isNew = (s) => {
-        const e = ledger[s.conversationKey];
-        return !e || e.size !== s.size; // unseen, or the file grew/changed since last migrate
+function parsePositiveIntFlag(value, name) {
+    if (value == null)
+        return undefined;
+    if (value === true)
+        throw new Error(`${name} requires a number.`);
+    const n = Number(value);
+    if (!Number.isInteger(n) || n <= 0)
+        throw new Error(`${name} must be a positive integer.`);
+    return n;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function responseStatus(e) {
+    const response = e.response;
+    return typeof response?.status === "number" ? response.status : undefined;
+}
+function responseData(e) {
+    const data = e.response?.data;
+    return isRecord(data) ? data : {};
+}
+function responseMessage(e) {
+    const data = responseData(e);
+    const message = data.message || data.error;
+    if (typeof message === "string")
+        return message;
+    return e instanceof Error ? e.message : String(e);
+}
+function codedError(code) {
+    const e = new Error(code);
+    e.code = code;
+    return e;
+}
+const bareId = (s) => s.conversationKey.slice(s.conversationKey.indexOf(":") + 1);
+const mapKey = (platform, conv) => `${platform}:${conv}`;
+const userTimeZone = () => Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC";
+const approxTokens = (chars) => Math.ceil(chars / APPROX_CHARS_PER_TOKEN);
+const jobLockMessage = (message) => /already running|not retryable|job did not start/i.test(message);
+function sum(nums) {
+    return nums.reduce((n, x) => n + x, 0);
+}
+function percentile(nums, p) {
+    if (!nums.length)
+        return 0;
+    const s = [...nums].sort((a, b) => a - b);
+    return s[Math.min(s.length - 1, Math.floor((s.length - 1) * p))] || 0;
+}
+export function estimateMigration(sessions, pending) {
+    const chars = pending.map((s) => s.rawData.length);
+    const turns = pending.map((s) => s.turnCount);
+    const pendingCodex = pending.filter((s) => s.source === "codex").length;
+    const bucketDefs = [
+        { label: "small <=10k chars", min: 0, max: 10_000 },
+        { label: "routine 10k-30k", min: 10_000, max: 30_000 },
+        { label: "medium 30k-60k", min: 30_000, max: 60_000 },
+        { label: "large 60k-120k", min: 60_000, max: 120_000 },
+        { label: "very large 120k-240k", min: 120_000, max: 240_000 },
+        { label: "huge >240k", min: 240_000, max: Number.POSITIVE_INFINITY },
+    ];
+    const buckets = bucketDefs.map((b) => {
+        const xs = pending.filter((s) => s.rawData.length > b.min && s.rawData.length <= b.max);
+        const bucketChars = sum(xs.map((s) => s.rawData.length));
+        return {
+            label: b.label,
+            count: xs.length,
+            chars: bucketChars,
+            approxInputTokens: approxTokens(bucketChars),
+        };
+    });
+    return {
+        sessions: sessions.length,
+        pending: pending.length,
+        alreadyMigrated: sessions.length - pending.length,
+        codex: sessions.filter((s) => s.source === "codex").length,
+        claudeCode: sessions.filter((s) => s.source === "claude-code").length,
+        pendingCodex,
+        pendingClaudeCode: pending.length - pendingCodex,
+        chars: {
+            total: sum(chars),
+            p50: percentile(chars, 0.5),
+            p75: percentile(chars, 0.75),
+            p90: percentile(chars, 0.9),
+            p95: percentile(chars, 0.95),
+            max: percentile(chars, 1),
+        },
+        approxInputTokens: {
+            total: approxTokens(sum(chars)),
+            p50: approxTokens(percentile(chars, 0.5)),
+            p90: approxTokens(percentile(chars, 0.9)),
+            p95: approxTokens(percentile(chars, 0.95)),
+            max: approxTokens(percentile(chars, 1)),
+        },
+        turns: {
+            total: sum(turns),
+            p50: percentile(turns, 0.5),
+            p90: percentile(turns, 0.9),
+            max: percentile(turns, 1),
+        },
+        buckets,
+        queue: {
+            mode: "client-sequential",
+            runRequestLimitPerMinute: RATE_MAX,
+            throttleFloorMinutes: pending.length ? Math.ceil(pending.length / RATE_MAX) : 0,
+            largeSessionCount: pending.filter((s) => s.rawData.length > 120_000).length,
+            hugeSessionCount: pending.filter((s) => s.rawData.length > 240_000).length,
+            needsChunkingAboveChars: 240_000,
+        },
     };
-    const pending = sessions.filter(isNew);
-    const codexN = sessions.filter((s) => s.source === "codex").length;
-    const claudeN = sessions.length - codexN;
-    console.log("");
-    console.log(c.bold(c.cyan("EchoMem migration")) + c.dim(`  (${API_BASE})`));
-    console.log(`Found ${c.bold(String(sessions.length))} sessions (${codexN} Codex + ${claudeN} Claude Code) · ${c.bold(String(pending.length))} new/changed to import` +
-        (sessions.length - pending.length ? c.dim(`, ${sessions.length - pending.length} already migrated`) : ""));
-    if (flags["dry-run"] === true) {
-        console.log(c.dim("\n--dry-run: discovered + assembled only, nothing sent.\n"));
-        for (const s of pending.slice(0, 50)) {
-            console.log(`  ${s.source === "codex" ? "codex " : "claude"} ${(s.firstTs || "").slice(0, 10)}  ${s.turnCount} turns  ${c.dim(truncate(s.title, 60))}`);
-        }
-        if (pending.length > 50)
-            console.log(c.dim(`  … and ${pending.length - 50} more`));
-        return;
+}
+export function estimateMigrationEta(pending, skippedActive = 0) {
+    const bucketDefs = [
+        { key: "small", label: "<=30k chars", min: 0, max: 30_000, secondsPerSession: 10 },
+        { key: "routine", label: "30k-120k", min: 30_000, max: 120_000, secondsPerSession: 16 },
+        { key: "large", label: "120k-350k", min: 120_000, max: 350_000, secondsPerSession: 16 },
+        { key: "veryLarge", label: "350k-1M", min: 350_000, max: 1_000_000, secondsPerSession: 40 },
+        { key: "huge", label: "1M-2M", min: 1_000_000, max: 2_000_000, secondsPerSession: 60 },
+        { key: "massive", label: ">2M", min: 2_000_000, max: Number.POSITIVE_INFINITY, secondsPerSession: 90 },
+    ];
+    const buckets = bucketDefs.map((b) => {
+        const xs = pending.filter((s) => s.rawData.length > b.min && s.rawData.length <= b.max);
+        const chars = sum(xs.map((s) => s.rawData.length));
+        return {
+            key: b.key,
+            label: b.label,
+            count: xs.length,
+            chars,
+            approxInputTokens: approxTokens(chars),
+            secondsPerSession: b.secondsPerSession,
+        };
+    });
+    const unbufferedSeconds = sum(buckets.map((b) => b.count * b.secondsPerSession));
+    const bufferedSeconds = Math.ceil(unbufferedSeconds * 1.25);
+    const throttleFloorSeconds = pending.length ? Math.ceil(pending.length / RATE_MAX) * 60 : 0;
+    const estimatedSeconds = Math.max(bufferedSeconds, throttleFloorSeconds);
+    const totalChars = sum(pending.map((s) => s.rawData.length));
+    return {
+        pending: pending.length,
+        skippedActive,
+        totalChars,
+        approxInputTokens: approxTokens(totalChars),
+        estimatedSeconds,
+        estimatedLabel: formatEta(estimatedSeconds),
+        throttleFloorSeconds,
+        buckets,
+    };
+}
+export function applyMigrationSelection(sessions, opts = {}) {
+    let out = [...sessions];
+    if (typeof opts.minChars === "number") {
+        out = out.filter((s) => s.rawData.length >= opts.minChars);
     }
-    if (!pending.length) {
-        console.log(c.green("\n✓ Everything is already migrated — nothing to do.\n"));
-        return;
+    if (typeof opts.maxChars === "number") {
+        out = out.filter((s) => s.rawData.length <= opts.maxChars);
     }
-    // --- Below here we actually send → require a token + (if encrypted) an unlocked key. ---
-    const token = store.getToken();
-    if (!token) {
-        console.error("Not logged in. Run `echomem-mcp login` first, then re-run migrate.");
-        process.exitCode = 1;
-        return;
+    if (opts.largest) {
+        out.sort((a, b) => b.rawData.length - a.rawData.length || String(b.firstTs || "").localeCompare(String(a.firstTs || "")));
     }
-    const client = axios.create({
+    if (opts.limit && opts.limit > 0) {
+        out = out.slice(0, opts.limit);
+    }
+    return out;
+}
+export function dedupeSessionsByConversation(sessions) {
+    const byKey = new Map();
+    for (const s of sessions) {
+        const prev = byKey.get(s.conversationKey);
+        if (!prev ||
+            s.rawData.length > prev.rawData.length ||
+            (s.rawData.length === prev.rawData.length && s.mtimeMs > prev.mtimeMs)) {
+            byKey.set(s.conversationKey, s);
+        }
+    }
+    return [...byKey.values()].sort((a, b) => String(b.firstTs || "").localeCompare(String(a.firstTs || "")));
+}
+export function isActiveMigrationSession(session, nowMs = Date.now()) {
+    return !!session.mtimeMs && nowMs - session.mtimeMs >= 0 && nowMs - session.mtimeMs < ACTIVE_SESSION_GRACE_MS;
+}
+function authedClient(token) {
+    return axios.create({
         baseURL: API_BASE,
         headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
     });
+}
+export function discoverMigratableSessions(opts = {}) {
+    let sessions = discoverSessions();
+    const since = opts.since;
+    if (since)
+        sessions = sessions.filter((s) => (s.firstTs || "").slice(0, 10) >= since);
+    sessions = dedupeSessionsByConversation(sessions);
+    sessions = applyMigrationSelection(sessions, { minChars: opts.minChars, maxChars: opts.maxChars, largest: opts.largest });
+    const selectableSessions = opts.includeActive ? sessions : sessions.filter((s) => !isActiveMigrationSession(s));
+    const skippedActive = sessions.length - selectableSessions.length;
+    const ledger = loadLedger();
+    const pendingAll = selectableSessions.filter((s) => {
+        const e = ledger[s.conversationKey];
+        return !e || e.size !== s.size;
+    });
+    const pending = opts.limit && opts.limit > 0 ? pendingAll.slice(0, opts.limit) : pendingAll;
+    const codexCount = sessions.filter((s) => s.source === "codex").length;
+    return {
+        sessions,
+        pending,
+        pendingTotal: pendingAll.length,
+        alreadyMigrated: selectableSessions.length - pendingAll.length,
+        skippedActive,
+        limited: pending.length < pendingAll.length,
+        codexCount,
+        claudeCount: sessions.length - codexCount,
+    };
+}
+export async function startMigration(opts) {
+    if (opts.pending.length === 0)
+        throw codedError("NO_PENDING_SESSIONS");
+    const store = new KeyStore();
+    const token = store.getToken();
+    if (!token)
+        throw codedError("NOT_LOGGED_IN");
+    const client = authedClient(token);
     let encKey;
     try {
         const cfg = await fetchEncryptionConfig(client);
         if (cfg.enabled) {
             encKey = store.getKey();
-            if (!encKey) {
-                console.error(store.isKeyExpired()
-                    ? "Vault key expired. Run `echomem-mcp unlock`, then re-run migrate."
-                    : "This account is ENCRYPTED but the vault is locked. Run `echomem-mcp unlock`, then re-run migrate.");
-                process.exitCode = 1;
-                return;
-            }
+            if (!encKey)
+                throw codedError("VAULT_LOCKED");
         }
     }
-    catch {
-        /* config fetch failed → proceed as unencrypted (server enforces 422 if it's actually encrypted) */
-    }
-    // Consent: one keypress (skipped with --yes or when non-interactive).
-    if (flags.yes !== true && process.stdin.isTTY) {
-        const ok = await promptYesNo(`Import ${pending.length} session(s) into your EchoMem memory? [Y/n] `);
-        if (!ok) {
-            console.log("Aborted. Nothing was sent.");
-            return;
-        }
+    catch (e) {
+        if (e?.code === "VAULT_LOCKED")
+            throw e;
+        /* config fetch failed → proceed as unencrypted; server enforces if actually encrypted */
     }
-    const userTz = Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC";
-    const bareId = (s) => s.conversationKey.slice(s.conversationKey.indexOf(":") + 1);
-    const mapKey = (platform, conv) => `${platform}:${conv}`;
-    // 1) Create the import session — descriptors only, NO transcript. The server fans this out to one
-    //    import_jobs row per session, which is what the web dashboard polls (GET /import-sessions/{id}).
-    let toImport = pending;
+    const tz = userTimeZone();
+    let toImport = opts.pending;
+    let capped;
     let session;
     try {
-        session = await createImportSession(client, toImport, bareId, userTz);
+        session = await createImportSession(client, toImport, bareId, tz, opts.signal);
     }
     catch (e) {
-        // Plan caps the batch → import the newest N and tell the user to re-run for the rest.
-        const max = Number(e?.response?.data?.maxConversations);
-        if (e?.response?.status === 422 && e?.response?.data?.error === "IMPORT_LIMIT_EXCEEDED" && max > 0) {
-            console.log(c.yellow(`Your plan imports up to ${max} at a time — importing the newest ${max}; re-run migrate for older sessions.`));
-            toImport = pending.slice(0, max);
-            session = await createImportSession(client, toImport, bareId, userTz);
+        if (responseStatus(e) === 403)
+            throw codedError("FORBIDDEN_SCOPE");
+        const data = responseData(e);
+        const max = Number(data.maxConversations);
+        if (responseStatus(e) === 422 && data.error === "IMPORT_LIMIT_EXCEEDED" && max > 0) {
+            capped = max;
+            toImport = opts.pending.slice(0, max);
+            try {
+                session = await createImportSession(client, toImport, bareId, tz, opts.signal);
+            }
+            catch (retryError) {
+                if (responseStatus(retryError) === 403)
+                    throw codedError("FORBIDDEN_SCOPE");
+                throw retryError;
+            }
         }
         else {
-            console.error(c.red(`Could not start the import: ${e?.response?.data?.message || e?.response?.data?.error || e?.message || e}`));
-            process.exitCode = 1;
-            return;
+            throw e;
         }
     }
     const byConv = new Map(toImport.map((s) => [mapKey(s.source, bareId(s)), s]));
-    console.log(c.dim(`Import session ${session.id} — ${session.jobs.length} jobs queued (the web dashboard can watch this live).`));
-    // 2) Drive each job: the queue lives server-side, but only THIS machine can read the logs, so the
-    //    bridge streams each transcript to /import-jobs/{id}/run. The web just polls. Throttle < 30/min.
+    const runId = crypto.randomUUID();
+    const metricsFile = opts.metricsFile || defaultMigrationMetricsPath();
+    const done = runJobs({
+        client,
+        session,
+        byConv,
+        userTz: tz,
+        encKey,
+        onProgress: opts.onProgress,
+        runId,
+        metricsFile,
+        selection: opts.selection,
+    });
+    return { sessionId: session.id, runId, jobCount: session.jobs.length, metricsFile, ...(capped ? { capped } : {}), done };
+}
+async function runJobs(args) {
+    const ledger = loadLedger();
     const reqTimes = [];
     const throttle = async () => {
         for (;;) {
@@ -349,44 +634,248 @@ export async function cmdMigrate(flags) {
         }
     };
     let migrated = 0, extracted = 0, failed = 0, i = 0;
-    for (const job of session.jobs) {
-        const s = byConv.get(mapKey(job.platform, job.conversation_id));
-        if (!s)
-            continue;
-        i++;
-        const tag = `${c.dim(`[${i}/${session.jobs.length}]`)} ${s.source === "codex" ? "codex " : "claude"} ${(s.firstTs || "").slice(0, 10)}`;
+    const recordMetric = (metric) => {
         try {
-            await throttle();
-            const r = await runImportJob(client, job.id, s, userTz, encKey);
-            extracted += r.memories;
-            migrated++;
-            ledger[s.conversationKey] = { size: s.size, mtimeMs: s.mtimeMs, status: "done", memories: r.memories };
-            saveLedger(ledger);
-            const note = r.alreadyDone || r.duplicate ? c.dim("already imported") : c.green(`+${r.memories} ${r.memories === 1 ? "memory" : "memories"}`);
-            console.log(`${tag}  ${note}  ${c.dim(truncate(s.title, 48))}`);
+            appendMigrationMetric(args.metricsFile, metric);
         }
-        catch (e) {
-            const status = e?.response?.status;
-            const errCode = e?.response?.data?.error;
-            // Vault key expired mid-run (TTL crossed) → stop cleanly; the rest is resumable on re-run.
-            if (status === 422 && errCode === "ENCRYPTION_KEY_REQUIRED") {
-                console.error(c.yellow(`\n⚠ Vault locked mid-run. Run \`echomem-mcp unlock\` and re-run migrate to resume (${migrated} done so far).`));
-                process.exitCode = 1;
-                break;
+        catch {
+            /* best effort: metrics must never block a user migration */
+        }
+    };
+    try {
+        for (const job of args.session.jobs) {
+            const s = args.byConv.get(mapKey(job.platform, job.conversation_id));
+            if (!s)
+                continue;
+            i++;
+            const jobStartedAt = Date.now();
+            try {
+                await throttle();
+                const r = await runImportJob(args.client, job.id, s, args.userTz, args.encKey);
+                extracted += r.memories;
+                migrated++;
+                ledger[s.conversationKey] = { size: s.size, mtimeMs: s.mtimeMs, status: "done", memories: r.memories };
+                saveLedger(ledger);
+                recordMetric(buildMigrationMetric({
+                    runId: args.runId,
+                    importSessionId: args.session.id,
+                    jobId: job.id,
+                    index: i,
+                    total: args.session.jobs.length,
+                    session: s,
+                    status: "completed",
+                    memories: r.memories,
+                    durationMs: r.durationMs,
+                    processingTimeMs: r.processingTimeMs,
+                    ttfmMs: r.ttfmMs,
+                    alreadyDone: r.alreadyDone,
+                    duplicate: r.duplicate,
+                    selection: args.selection,
+                }));
+                args.onProgress?.({
+                    index: i,
+                    total: args.session.jobs.length,
+                    importSessionId: args.session.id,
+                    jobId: job.id,
+                    session: s,
+                    memories: r.memories,
+                    durationMs: r.durationMs,
+                    processingTimeMs: r.processingTimeMs,
+                    ttfmMs: r.ttfmMs,
+                    alreadyDone: r.alreadyDone,
+                    duplicate: r.duplicate,
+                });
+            }
+            catch (e) {
+                const status = responseStatus(e);
+                const errCode = responseData(e).error;
+                const message = responseMessage(e);
+                if (status === 422 && errCode === "ENCRYPTION_KEY_REQUIRED") {
+                    const durationMs = Date.now() - jobStartedAt;
+                    recordMetric(buildMigrationMetric({
+                        runId: args.runId,
+                        importSessionId: args.session.id,
+                        jobId: job.id,
+                        index: i,
+                        total: args.session.jobs.length,
+                        session: s,
+                        status: "stopped",
+                        durationMs,
+                        error: message,
+                        selection: args.selection,
+                    }));
+                    args.onProgress?.({ index: i, total: args.session.jobs.length, importSessionId: args.session.id, jobId: job.id, session: s, error: message, durationMs });
+                    return { migrated, extracted, failed, stoppedReason: "key-expired" };
+                }
+                failed++;
+                const durationMs = Date.now() - jobStartedAt;
+                recordMetric(buildMigrationMetric({
+                    runId: args.runId,
+                    importSessionId: args.session.id,
+                    jobId: job.id,
+                    index: i,
+                    total: args.session.jobs.length,
+                    session: s,
+                    status: "failed",
+                    durationMs,
+                    error: message,
+                    selection: args.selection,
+                }));
+                args.onProgress?.({ index: i, total: args.session.jobs.length, importSessionId: args.session.id, jobId: job.id, session: s, error: message, durationMs });
             }
-            failed++;
-            console.log(`${tag}  ${c.red("✗ failed")} ${c.dim(truncate(String(e?.response?.data?.message || errCode || e?.message || e), 60))}`);
         }
     }
+    catch {
+        failed++;
+    }
+    return { migrated, extracted, failed };
+}
+export async function cmdMigrate(flags) {
+    const c = color(process.stdout.isTTY === true && !process.env.NO_COLOR && flags["no-color"] !== true);
+    // Discover + filter (pure local — discovery and --dry-run need no login).
+    const since = typeof flags.since === "string" ? flags.since : undefined;
+    let limit;
+    let minChars;
+    let maxChars;
+    try {
+        limit = parsePositiveIntFlag(flags.limit, "--limit");
+        minChars = parsePositiveIntFlag(flags["min-chars"], "--min-chars");
+        maxChars = parsePositiveIntFlag(flags["max-chars"], "--max-chars");
+    }
+    catch (e) {
+        console.error(e instanceof Error ? e.message : String(e));
+        process.exitCode = 1;
+        return;
+    }
+    if (minChars && maxChars && minChars > maxChars) {
+        console.error("--min-chars cannot be greater than --max-chars.");
+        process.exitCode = 1;
+        return;
+    }
+    const largest = flags.largest === true;
+    const includeActive = flags["include-active"] === true;
+    const selection = { since, limit, minChars, maxChars, largest, includeActive };
+    const metricsFile = typeof flags["metrics-file"] === "string" ? flags["metrics-file"] : defaultMigrationMetricsPath();
+    const { sessions, pending, pendingTotal, alreadyMigrated, skippedActive, limited, codexCount, claudeCount } = discoverMigratableSessions(selection);
+    const estimateSessions = includeActive ? sessions : sessions.filter((s) => !isActiveMigrationSession(s));
+    if (!sessions.length) {
+        console.log("No local Codex/Claude Code sessions found to migrate.");
+        return;
+    }
+    if (flags.estimate === true && flags.json === true) {
+        printMigrationEstimate(estimateMigration(limit ? pending : estimateSessions, pending), true);
+        return;
+    }
     console.log("");
-    console.log(c.bold("Done.") + `  ${c.green(String(migrated) + " imported")} · ${c.bold(String(extracted))} memories` +
-        (failed ? ` · ${c.red(String(failed) + " failed")}` : "") + ".");
-    if (extracted > 0) {
-        console.log(c.dim("Now ask your agent about a past project — it can recall it from memory.\n"));
+    console.log(c.bold(c.cyan("EchoMem migration")) + c.dim(`  (${API_BASE})`));
+    const pendingText = limited
+        ? `${c.bold(String(pending.length))} of ${c.bold(String(pendingTotal))} new/changed selected`
+        : `${c.bold(String(pending.length))} new/changed to import`;
+    console.log(`Found ${c.bold(String(sessions.length))} sessions (${codexCount} Codex + ${claudeCount} Claude Code) · ${pendingText}` +
+        (alreadyMigrated ? c.dim(`, ${alreadyMigrated} already migrated`) : "") +
+        (skippedActive ? c.dim(`, ${skippedActive} active skipped`) : ""));
+    const filters = [
+        minChars ? `min ${humanNum(minChars)} chars` : "",
+        maxChars ? `max ${humanNum(maxChars)} chars` : "",
+        largest ? "largest first" : "",
+        includeActive ? "include active" : "",
+        limit ? `limit ${humanNum(limit)}` : "",
+    ].filter(Boolean).join(" · ");
+    if (filters)
+        console.log(c.dim(`Selection: ${filters}`));
+    if (flags.estimate === true) {
+        printMigrationEstimate(estimateMigration(limit ? pending : estimateSessions, pending), false);
+        return;
+    }
+    if (flags["dry-run"] === true) {
+        console.log(c.dim("\n--dry-run: discovered + assembled only, nothing sent.\n"));
+        for (const s of pending.slice(0, 50)) {
+            console.log(`  ${s.source === "codex" ? "codex " : "claude"} ${(s.firstTs || "").slice(0, 10)}  ${humanNum(s.rawData.length)} chars  ${s.turnCount} text turns`);
+        }
+        if (pending.length > 50)
+            console.log(c.dim(`  … and ${pending.length - 50} more`));
+        return;
+    }
+    if (!pending.length) {
+        console.log(c.green("\n✓ Everything is already migrated — nothing to do.\n"));
+        return;
+    }
+    // Consent: one keypress (skipped with --yes or when non-interactive).
+    if (flags.yes !== true && process.stdin.isTTY) {
+        const ok = await promptYesNo(`Import ${pending.length} session(s) into your EchoMem memory? [Y/n] `);
+        if (!ok) {
+            console.log("Aborted. Nothing was sent.");
+            return;
+        }
+    }
+    try {
+        const jobDurations = [];
+        const h = await startMigration({
+            pending,
+            metricsFile,
+            selection,
+            onProgress: (ev) => {
+                const tag = `${c.dim(`[${ev.index}/${ev.total}]`)} ${ev.session.source === "codex" ? "codex " : "claude"} ${(ev.session.firstTs || "").slice(0, 10)}`;
+                if (ev.error) {
+                    console.log(`${tag}  ${c.red("✗ failed")} ${c.dim(truncate(ev.error, 60))}` + (ev.durationMs ? c.dim(`  ${humanDuration(ev.durationMs)}`) : ""));
+                    return;
+                }
+                if (ev.durationMs)
+                    jobDurations.push(ev.durationMs);
+                const eta = jobDurations.length > 0
+                    ? humanDuration(percentile(jobDurations, 0.5) * Math.max(0, ev.total - ev.index))
+                    : null;
+                const memories = ev.memories ?? 0;
+                const note = ev.alreadyDone
+                    ? c.dim("already completed")
+                    : memories === 0
+                        ? c.dim("0 memories")
+                        : c.green(`+${memories} ${memories === 1 ? "memory" : "memories"}`);
+                const timing = ev.durationMs
+                    ? c.dim(`  ${humanDuration(ev.durationMs)}${ev.ttfmMs ? ` (first memory ${humanDuration(ev.ttfmMs)})` : ""}${eta ? ` · ETA ${eta}` : ""}`)
+                    : "";
+                console.log(`${tag}  ${note}  ${c.dim(`${humanNum(ev.session.rawData.length)} chars · ${ev.session.turnCount} turns`)}${timing}`);
+            },
+        });
+        if (h.capped)
+            console.log(c.yellow(`Your plan imports up to ${h.capped} at a time — importing the newest ${h.capped}; re-run migrate for older sessions.`));
+        console.log(c.dim(`Import session ${h.sessionId} — ${h.jobCount} jobs queued (the web dashboard can watch this live).`));
+        console.log(c.dim(`Metrics: ${h.metricsFile}`));
+        const r = await h.done;
+        if (r.stoppedReason === "key-expired") {
+            console.error(c.yellow(`\n⚠ Vault locked mid-run. Run \`echomem-mcp unlock\` and re-run migrate to resume (${r.migrated} done so far).`));
+            process.exitCode = 1;
+        }
+        console.log("");
+        console.log(c.bold("Done.") + `  ${c.green(String(r.migrated) + " imported")} · ${c.bold(String(r.extracted))} memories` +
+            (r.failed ? ` · ${c.red(String(r.failed) + " failed")}` : "") + ".");
+        if (r.extracted > 0) {
+            console.log(c.dim("Now ask your agent about a past project — it can recall it from memory.\n"));
+        }
+        if (r.failed)
+            process.exitCode = 1;
+    }
+    catch (e) {
+        const code = e?.code;
+        if (code === "NOT_LOGGED_IN")
+            console.error("Not logged in. Run `echomem-mcp login` first, then re-run migrate.");
+        else if (code === "VAULT_LOCKED") {
+            const store = new KeyStore();
+            console.error(store.isKeyExpired()
+                ? "Vault key expired. Run `echomem-mcp unlock`, then re-run migrate."
+                : "This account is ENCRYPTED but the vault is locked. Run `echomem-mcp unlock`, then re-run migrate.");
+        }
+        else if (code === "FORBIDDEN_SCOPE") {
+            console.error("This device token cannot import history. Re-connect this device with `echomem-mcp setup`.");
+        }
+        else {
+            console.error(c.red(`Could not start the import: ${responseMessage(e)}`));
+        }
+        process.exitCode = 1;
     }
 }
 /** Create an import session (lightweight descriptors only — NO transcript). Returns the session id + jobs. */
-async function createImportSession(client, sessions, bareId, userTz) {
+async function createImportSession(client, sessions, bareId, userTz, signal) {
     const items = sessions.map((s) => ({
         conversationId: bareId(s),
         platform: s.source, // free-text label; also half of the (session, platform, conversation) key
@@ -394,12 +883,13 @@ async function createImportSession(client, sessions, bareId, userTz) {
         sourceDate: s.firstTs,
         userTz,
     }));
-    const res = await client.post("/api/extension/import-sessions", { items });
+    const res = await client.post("/api/extension/import-sessions", { items }, signal ? { signal } : undefined);
     const data = res.data || {};
     return { id: String(data.session?.id || ""), jobs: Array.isArray(data.jobs) ? data.jobs : [] };
 }
-/** Run one queued job: stream its transcript to the server, which extracts it. One retry on 429/5xx. */
+/** Run one queued job: stream its transcript to the server, which extracts it. Retries transient locks/rate limits. */
 async function runImportJob(client, jobId, s, userTz, encKey) {
+    const startedAt = Date.now();
     const body = {
         rawData: s.rawData,
         source: s.source, // NOT containing "mcp" (avoids the route's MCP title truncation)
@@ -415,24 +905,41 @@ async function runImportJob(client, jobId, s, userTz, encKey) {
             const data = res.data || {};
             if (data.claimed === false) {
                 // Already completed on a prior run → idempotent, not an error; otherwise it didn't start.
-                if (data.job?.status === "completed")
-                    return { memories: Number(data.job?.saved_memory_count) || 0, alreadyDone: true, duplicate: false };
+                if (data.job?.status === "completed") {
+                    return {
+                        memories: Number(data.job?.saved_memory_count) || 0,
+                        alreadyDone: true,
+                        duplicate: false,
+                        durationMs: Date.now() - startedAt,
+                    };
+                }
                 throw new Error(data.message || "job did not start");
             }
             const result = data.result || {};
-            return { memories: Number(result.memoriesExtracted) || 0, alreadyDone: false, duplicate: !!result.duplicate };
+            return {
+                memories: Number(result.memoriesExtracted) || 0,
+                alreadyDone: false,
+                duplicate: !!result.duplicate,
+                durationMs: Date.now() - startedAt,
+                processingTimeMs: typeof result.processingTimeMs === "number" ? result.processingTimeMs : undefined,
+                ttfmMs: typeof result.ttfmMs === "number" ? result.ttfmMs : undefined,
+            };
         }
         catch (e) {
             const status = e?.response?.status;
+            const message = e instanceof Error ? e.message : responseMessage(e);
             if (status === 422 && e?.response?.data?.error === "ENCRYPTION_KEY_REQUIRED")
                 throw e; // not retryable
-            if (status === 409)
-                throw e; // claim conflict — don't hammer a job another worker holds
-            const retryable = status === 429 || (status >= 500 && status < 600) || e?.code === "ECONNABORTED" || !status;
-            if (!retryable || attempt >= 2)
+            const lockConflict = status === 409 || jobLockMessage(message);
+            const retryable = lockConflict || status === 429 || (status >= 500 && status < 600) || e?.code === "ECONNABORTED" || !status;
+            const maxAttempts = lockConflict ? 5 : 2;
+            if (!retryable || attempt >= maxAttempts)
                 throw e;
             const retryAfter = Number(e?.response?.headers?.["retry-after"]) || Number(e?.response?.data?.retryAfterSeconds);
-            await sleep((status === 429 && retryAfter > 0 ? retryAfter : Math.pow(2, attempt) * 2) * 1000);
+            const backoffSeconds = lockConflict
+                ? Math.min(30, 5 * (attempt + 1))
+                : Math.pow(2, attempt) * 2;
+            await sleep((status === 429 && retryAfter > 0 ? retryAfter : backoffSeconds) * 1000);
         }
     }
 }