@echomem/mcp 1.3.1 → 1.3.2

package/dist/migrate.js

@@ -30,6 +30,8 @@ import { walk, eachLine } from "./report.js";
 const API_BASE = (process.env.ECHO_API_BASE_URL || "https://echo-mem-chrome.vercel.app").replace(/\/$/, "");
 const RATE_MAX = 28; // stay under the import-jobs /run limit of 30 / 60s
 const RATE_WINDOW_MS = 60_000;
+export const MIGRATE_CONCURRENCY = 6; // import jobs run concurrently; request STARTS are still throttled <RATE_MAX/min
+const FALLBACK_SECONDS_PER_SESSION = 14; // per-session extract time used only until local metrics exist
 const APPROX_CHARS_PER_TOKEN = 4;
 const ACTIVE_SESSION_GRACE_MS = 5 * 60_000;
 // ---------------------------------------------------------------------------
@@ -156,7 +158,7 @@ export function assembleClaude(file) {
     return {
         filePath: file,
         source: "claude-code",
-        conversationKey: `claude:${sessionId || sha16(file)}`,
+        conversationKey: `claude-code:${sessionId || sha16(file)}`,
         cwd: normalizeCwd(cwd),
         firstTs,
         title: title || "Claude text turns",
@@ -200,6 +202,99 @@ export function discoverSessions() {
     out.sort((a, b) => String(b.firstTs || "").localeCompare(String(a.firstTs || "")));
     return out;
 }
+const IMPORT_STATUS_CHUNK_SIZE = 1000;
+function initialJsonObjects(file, maxBytes = 1024 * 1024) {
+    let fd;
+    try {
+        fd = fs.openSync(file, "r");
+    }
+    catch {
+        return [];
+    }
+    try {
+        const buf = Buffer.allocUnsafe(maxBytes);
+        const bytes = fs.readSync(fd, buf, 0, maxBytes, 0);
+        const text = buf.subarray(0, bytes).toString("utf8");
+        return text.split("\n").slice(0, 1000).map((line) => {
+            const s = line.trim();
+            if (!s)
+                return null;
+            try {
+                return JSON.parse(s);
+            }
+            catch {
+                return null;
+            }
+        }).filter(Boolean);
+    }
+    finally {
+        try {
+            fs.closeSync(fd);
+        }
+        catch {
+            /* best effort */
+        }
+    }
+}
+function codexPayload(obj) {
+    return isRecord(obj.payload) ? obj.payload : obj;
+}
+function hasClaudeText(content) {
+    if (typeof content === "string")
+        return content.trim().length > 0;
+    if (!Array.isArray(content))
+        return false;
+    return content.some((block) => isRecord(block) && block.type === "text" && typeof block.text === "string" && block.text.trim().length > 0);
+}
+function fastSessionInfo(file, source) {
+    let conversationKey = `${source === "codex" ? "codex" : "claude-code"}:${sha16(file)}`;
+    let hasTextTurn = false;
+    for (const obj of initialJsonObjects(file)) {
+        if (!isRecord(obj))
+            continue;
+        if (source === "codex" && obj.type === "session_meta") {
+            const payload = isRecord(obj.payload) ? obj.payload : {};
+            if (typeof payload.id === "string" && payload.id)
+                conversationKey = `codex:${payload.id}`;
+            continue;
+        }
+        if (source === "codex") {
+            const p = codexPayload(obj);
+            if (p.type === "user_message" && typeof p.message === "string" && cleanCodexUser(p.message))
+                hasTextTurn = true;
+            if (p.type === "agent_message" && typeof p.message === "string" && p.message.trim())
+                hasTextTurn = true;
+            continue;
+        }
+        if (source === "claude-code" && typeof obj.sessionId === "string" && obj.sessionId) {
+            conversationKey = `claude-code:${obj.sessionId}`;
+        }
+        if (source === "claude-code" && (obj.type === "user" || obj.type === "assistant")) {
+            const message = isRecord(obj.message) ? obj.message : {};
+            if (hasClaudeText(message.content))
+                hasTextTurn = true;
+        }
+    }
+    return { conversationKey, hasTextTurn };
+}
+function fastSessionEntries(opts = {}) {
+    const out = [];
+    const codexRoot = opts.codexRoot ?? path.join(os.homedir(), ".codex", "sessions");
+    for (const filePath of walk(codexRoot, (p) => /rollout-.*\.jsonl$/.test(p), () => false)) {
+        const stat = statSafe(filePath);
+        const info = fastSessionInfo(filePath, "codex");
+        if (info.hasTextTurn)
+            out.push({ filePath, source: "codex", conversationKey: info.conversationKey, size: stat.size, mtimeMs: stat.mtimeMs });
+    }
+    const claudeRoot = opts.claudeRoot ?? path.join(os.homedir(), ".claude", "projects");
+    for (const filePath of walk(claudeRoot, (p) => p.endsWith(".jsonl"), (name) => name === "subagents" || name === "workflows")) {
+        const stat = statSafe(filePath);
+        const info = fastSessionInfo(filePath, "claude-code");
+        if (info.hasTextTurn)
+            out.push({ filePath, source: "claude-code", conversationKey: info.conversationKey, size: stat.size, mtimeMs: stat.mtimeMs });
+    }
+    return out;
+}
 function ledgerPath() {
     return path.join(echoConfigDir(), "migrate-ledger.json");
 }
@@ -223,6 +318,41 @@ function saveLedger(l) {
 export function defaultMigrationMetricsPath() {
     return path.join(echoConfigDir(), "migrate-metrics.jsonl");
 }
+/**
+ * Per-session extraction time (seconds) measured from THIS user's past runs, so the ETA reflects their
+ * real server/network throughput instead of fixed guesses. Reads the local metrics log, keeps recent
+ * genuinely-extracted jobs (completed, not duplicate/already-done), and returns a trimmed mean (drops
+ * the slowest 10% so a rare stalled job doesn't dominate). Returns null when there isn't enough data.
+ */
+export function measuredSecondsPerSession(metricsFile = defaultMigrationMetricsPath()) {
+    let lines;
+    try {
+        lines = fs.readFileSync(metricsFile, "utf8").split("\n");
+    }
+    catch {
+        return null; // never run a real migrate yet
+    }
+    const secs = [];
+    for (const line of lines.slice(-400)) {
+        const s = line.trim();
+        if (!s)
+            continue;
+        try {
+            const m = JSON.parse(s);
+            if (m.status === "completed" && !m.duplicate && !m.alreadyDone && typeof m.durationMs === "number" && m.durationMs > 0) {
+                secs.push(m.durationMs / 1000);
+            }
+        }
+        catch {
+            /* skip malformed line */
+        }
+    }
+    if (secs.length < 8)
+        return null;
+    secs.sort((a, b) => a - b);
+    const kept = secs.slice(0, Math.max(1, Math.floor(secs.length * 0.9)));
+    return kept.reduce((n, x) => n + x, 0) / kept.length;
+}
 function sourceMtimeIso(s) {
     return s.mtimeMs ? new Date(s.mtimeMs).toISOString() : null;
 }
@@ -359,6 +489,11 @@ function responseStatus(e) {
     const response = e.response;
     return typeof response?.status === "number" ? response.status : undefined;
 }
+export function isImportStatusUnsupported(e) {
+    const status = responseStatus(e);
+    const message = e instanceof Error ? e.message : String(e);
+    return status === 404 || status === 405 || /status code (404|405)\b/i.test(message);
+}
 function responseData(e) {
     const data = e.response?.data;
     return isRecord(data) ? data : {};
@@ -452,17 +587,22 @@ export function estimateMigration(sessions, pending) {
     };
 }
 export function estimateMigrationEta(pending, skippedActive = 0) {
+    return estimateMigrationEtaFromLengths(pending.map((s) => s.rawData.length), skippedActive, {
+        secondsPerSession: measuredSecondsPerSession() ?? undefined,
+    });
+}
+export function estimateMigrationEtaFromLengths(lengths, skippedActive = 0, opts = {}) {
     const bucketDefs = [
-        { key: "small", label: "<=30k chars", min: 0, max: 30_000, secondsPerSession: 10 },
-        { key: "routine", label: "30k-120k", min: 30_000, max: 120_000, secondsPerSession: 16 },
-        { key: "large", label: "120k-350k", min: 120_000, max: 350_000, secondsPerSession: 16 },
-        { key: "veryLarge", label: "350k-1M", min: 350_000, max: 1_000_000, secondsPerSession: 40 },
-        { key: "huge", label: "1M-2M", min: 1_000_000, max: 2_000_000, secondsPerSession: 60 },
-        { key: "massive", label: ">2M", min: 2_000_000, max: Number.POSITIVE_INFINITY, secondsPerSession: 90 },
+        { key: "small", label: "Up to 30k chars", min: 0, max: 30_000, secondsPerSession: 10 },
+        { key: "routine", label: "30k-120k chars", min: 30_000, max: 120_000, secondsPerSession: 16 },
+        { key: "large", label: "120k-350k chars", min: 120_000, max: 350_000, secondsPerSession: 16 },
+        { key: "veryLarge", label: "350k-1M chars", min: 350_000, max: 1_000_000, secondsPerSession: 40 },
+        { key: "huge", label: "1M-2M chars", min: 1_000_000, max: 2_000_000, secondsPerSession: 60 },
+        { key: "massive", label: "Over 2M chars", min: 2_000_000, max: Number.POSITIVE_INFINITY, secondsPerSession: 90 },
     ];
     const buckets = bucketDefs.map((b) => {
-        const xs = pending.filter((s) => s.rawData.length > b.min && s.rawData.length <= b.max);
-        const chars = sum(xs.map((s) => s.rawData.length));
+        const xs = lengths.filter((n) => n > b.min && n <= b.max);
+        const chars = sum(xs);
         return {
             key: b.key,
             label: b.label,
@@ -472,13 +612,19 @@ export function estimateMigrationEta(pending, skippedActive = 0) {
             secondsPerSession: b.secondsPerSession,
         };
     });
-    const unbufferedSeconds = sum(buckets.map((b) => b.count * b.secondsPerSession));
-    const bufferedSeconds = Math.ceil(unbufferedSeconds * 1.25);
-    const throttleFloorSeconds = pending.length ? Math.ceil(pending.length / RATE_MAX) * 60 : 0;
-    const estimatedSeconds = Math.max(bufferedSeconds, throttleFloorSeconds);
-    const totalChars = sum(pending.map((s) => s.rawData.length));
+    // Jobs run concurrently (up to MIGRATE_CONCURRENCY in flight), but request STARTS are capped at
+    // RATE_MAX/min. Effective throughput is the lesser of what the worker pool sustains (concurrency /
+    // per-session time) and the server rate cap — so for large batches the rate cap dominates and the
+    // wall-clock is far shorter than the old one-by-one sum. perSession comes from measured metrics.
+    const pending = lengths.length;
+    const perSession = opts.secondsPerSession && opts.secondsPerSession > 0 ? opts.secondsPerSession : FALLBACK_SECONDS_PER_SESSION;
+    const concurrency = opts.concurrency && opts.concurrency > 0 ? opts.concurrency : MIGRATE_CONCURRENCY;
+    const throughputPerSec = pending ? Math.min(concurrency / perSession, RATE_MAX / 60) : 0;
+    const estimatedSeconds = pending && throughputPerSec ? Math.ceil((pending / throughputPerSec + perSession) * 1.15) : 0;
+    const throttleFloorSeconds = pending ? Math.ceil(pending / RATE_MAX) * 60 : 0;
+    const totalChars = sum(lengths);
     return {
-        pending: pending.length,
+        pending: lengths.length,
         skippedActive,
         totalChars,
         approxInputTokens: approxTokens(totalChars),
@@ -488,6 +634,67 @@ export function estimateMigrationEta(pending, skippedActive = 0) {
         buckets,
     };
 }
+export function summarizeFastMigratableDiscovery(discovery) {
+    return {
+        sessions: discovery.sessions.length,
+        pending: discovery.pending.length,
+        pendingTotal: discovery.pendingTotal,
+        alreadyMigrated: discovery.alreadyMigrated,
+        skippedActive: discovery.skippedActive,
+        codexCount: discovery.codexCount,
+        claudeCount: discovery.claudeCount,
+        eta: estimateMigrationEtaFromLengths(discovery.pending.map((s) => s.size), discovery.skippedActive, {
+            secondsPerSession: measuredSecondsPerSession() ?? undefined,
+        }),
+        accountChecked: discovery.accountChecked,
+        accountCheckFailed: discovery.accountCheckFailed,
+        accountCheckUnavailable: discovery.accountCheckUnavailable,
+    };
+}
+export function discoverMigratableFastDiscovery(opts = {}) {
+    const byKey = new Map();
+    const ledger = loadLedger();
+    for (const s of fastSessionEntries(opts)) {
+        const prev = byKey.get(s.conversationKey);
+        const ledgerEntry = ledger[s.conversationKey];
+        const matchesLedger = !!ledgerEntry && ledgerEntry.size === s.size;
+        const prevMatchesLedger = !!prev && !!ledgerEntry && ledgerEntry.size === prev.size;
+        if (!prev ||
+            (matchesLedger && !prevMatchesLedger) ||
+            (matchesLedger === prevMatchesLedger && (s.size > prev.size || (s.size === prev.size && s.mtimeMs > prev.mtimeMs)))) {
+            byKey.set(s.conversationKey, s);
+        }
+    }
+    let sessions = [...byKey.values()];
+    if (opts.since) {
+        const since = opts.since;
+        sessions = sessions.filter((s) => {
+            const iso = sourceMtimeIso({ ...s, rawData: "", turnCount: 0, cwd: null, firstTs: null, title: "" });
+            return !!iso && iso.slice(0, 10) >= since;
+        });
+    }
+    const selectable = opts.includeActive ? sessions : sessions.filter((s) => !isActiveMigrationSession(s, opts.nowMs));
+    const skippedActive = sessions.length - selectable.length;
+    const pendingAll = selectable.filter((s) => {
+        const e = ledger[s.conversationKey];
+        return !e || e.size !== s.size;
+    });
+    const pending = opts.limit && opts.limit > 0 ? pendingAll.slice(0, opts.limit) : pendingAll;
+    const codexCount = sessions.filter((s) => s.source === "codex").length;
+    return {
+        sessions,
+        pending,
+        pendingTotal: pendingAll.length,
+        alreadyMigrated: selectable.length - pendingAll.length,
+        skippedActive,
+        limited: pending.length < pendingAll.length,
+        codexCount,
+        claudeCount: sessions.length - codexCount,
+    };
+}
+export function discoverMigratableSummaryFast(opts = {}) {
+    return summarizeFastMigratableDiscovery(discoverMigratableFastDiscovery(opts));
+}
 export function applyMigrationSelection(sessions, opts = {}) {
     let out = [...sessions];
     if (typeof opts.minChars === "number") {
@@ -552,6 +759,100 @@ export function discoverMigratableSessions(opts = {}) {
         claudeCount: sessions.length - codexCount,
     };
 }
+export function applyAccountImportStatus(discovery, processedKeys, opts = {}) {
+    const selectableSessions = opts.includeActive
+        ? discovery.sessions
+        : discovery.sessions.filter((s) => !isActiveMigrationSession(s, opts.nowMs));
+    const skippedActive = discovery.sessions.length - selectableSessions.length;
+    const pendingAll = selectableSessions.filter((s) => !processedKeys.has(s.conversationKey));
+    const pending = opts.limit && opts.limit > 0 ? pendingAll.slice(0, opts.limit) : pendingAll;
+    return {
+        ...discovery,
+        pending,
+        pendingTotal: pendingAll.length,
+        alreadyMigrated: selectableSessions.length - pendingAll.length,
+        skippedActive,
+        limited: pending.length < pendingAll.length,
+        accountChecked: true,
+        accountCheckFailed: false,
+        accountCheckUnavailable: false,
+    };
+}
+export function applyFastAccountImportStatus(discovery, processedKeys, opts = {}) {
+    const selectableSessions = opts.includeActive
+        ? discovery.sessions
+        : discovery.sessions.filter((s) => !isActiveMigrationSession(s, opts.nowMs));
+    const skippedActive = discovery.sessions.length - selectableSessions.length;
+    const pendingAll = selectableSessions.filter((s) => !processedKeys.has(s.conversationKey));
+    const pending = opts.limit && opts.limit > 0 ? pendingAll.slice(0, opts.limit) : pendingAll;
+    return {
+        ...discovery,
+        pending,
+        pendingTotal: pendingAll.length,
+        alreadyMigrated: selectableSessions.length - pendingAll.length,
+        skippedActive,
+        limited: pending.length < pendingAll.length,
+        accountChecked: true,
+        accountCheckFailed: false,
+        accountCheckUnavailable: false,
+    };
+}
+export function markAccountImportStatusFailed(discovery) {
+    return {
+        ...discovery,
+        accountChecked: false,
+        accountCheckFailed: true,
+        accountCheckUnavailable: false,
+    };
+}
+export function markAccountImportStatusUnavailable(discovery) {
+    return {
+        ...discovery,
+        accountChecked: false,
+        accountCheckFailed: false,
+        accountCheckUnavailable: true,
+    };
+}
+export function markFastAccountImportStatusUnavailable(discovery) {
+    return {
+        ...discovery,
+        accountChecked: false,
+        accountCheckFailed: false,
+        accountCheckUnavailable: true,
+    };
+}
+export async function fetchProcessedImportKeys(token, sessions, signal) {
+    const client = authedClient(token);
+    const processedKeys = new Set();
+    for (let i = 0; i < sessions.length; i += IMPORT_STATUS_CHUNK_SIZE) {
+        const chunk = sessions.slice(i, i + IMPORT_STATUS_CHUNK_SIZE);
+        const items = chunk.map((s) => ({
+            conversationId: bareId(s),
+            platform: s.source,
+            sourceDate: s.firstTs,
+        }));
+        const res = await client.post("/api/extension/import-sessions/status", { items }, signal ? { signal } : undefined);
+        const data = (res.data && typeof res.data === "object" ? res.data : {});
+        const statusItems = Array.isArray(data.items) ? data.items : [];
+        for (const item of statusItems) {
+            if (item.processed === true &&
+                typeof item.platform === "string" &&
+                typeof item.conversationId === "string" &&
+                item.conversationId) {
+                processedKeys.add(mapKey(item.platform, item.conversationId));
+            }
+        }
+    }
+    return processedKeys;
+}
+export async function reconcileMigratableWithAccount(token, discovery, opts = {}) {
+    const processedKeys = await fetchProcessedImportKeys(token, discovery.sessions, opts.signal);
+    return applyAccountImportStatus(discovery, processedKeys, opts);
+}
+export async function reconcileFastMigratableWithAccount(token, discovery, opts = {}) {
+    const processedKeys = await fetchProcessedImportKeys(token, discovery.sessions, opts.signal);
+    return applyFastAccountImportStatus(discovery, processedKeys, opts);
+}
 export async function startMigration(opts) {
     if (opts.pending.length === 0)
         throw codedError("NO_PENDING_SESSIONS");
@@ -572,7 +873,10 @@ export async function startMigration(opts) {
     catch (e) {
         if (e?.code === "VAULT_LOCKED")
             throw e;
-        /* config fetch failed → proceed as unencrypted; server enforces if actually encrypted */
+        // Config fetch failed (e.g., a transient DNS/network timeout). If a verified key is stored, send it
+        // anyway — the server uses it iff the account is encrypted and ignores it otherwise. Dropping it here
+        // made an encrypted account fail every job with ENCRYPTION_KEY_REQUIRED on a single flaky request.
+        encKey = store.getKey() || undefined;
     }
     const tz = userTimeZone();
     let toImport = opts.pending;
@@ -633,7 +937,9 @@ async function runJobs(args) {
             await sleep(RATE_WINDOW_MS - (now - reqTimes[0]) + 100);
         }
     };
-    let migrated = 0, extracted = 0, failed = 0, i = 0;
+    let migrated = 0, extracted = 0, failed = 0, done = 0;
+    let stoppedReason;
+    const total = args.session.jobs.length;
     const recordMetric = (metric) => {
         try {
             appendMigrationMetric(args.metricsFile, metric);
@@ -642,93 +948,80 @@ async function runJobs(args) {
             /* best effort: metrics must never block a user migration */
         }
     };
-    try {
-        for (const job of args.session.jobs) {
-            const s = args.byConv.get(mapKey(job.platform, job.conversation_id));
-            if (!s)
-                continue;
-            i++;
-            const jobStartedAt = Date.now();
-            try {
-                await throttle();
-                const r = await runImportJob(args.client, job.id, s, args.userTz, args.encKey);
-                extracted += r.memories;
-                migrated++;
-                ledger[s.conversationKey] = { size: s.size, mtimeMs: s.mtimeMs, status: "done", memories: r.memories };
-                saveLedger(ledger);
-                recordMetric(buildMigrationMetric({
-                    runId: args.runId,
-                    importSessionId: args.session.id,
-                    jobId: job.id,
-                    index: i,
-                    total: args.session.jobs.length,
-                    session: s,
-                    status: "completed",
-                    memories: r.memories,
-                    durationMs: r.durationMs,
-                    processingTimeMs: r.processingTimeMs,
-                    ttfmMs: r.ttfmMs,
-                    alreadyDone: r.alreadyDone,
-                    duplicate: r.duplicate,
-                    selection: args.selection,
-                }));
-                args.onProgress?.({
-                    index: i,
-                    total: args.session.jobs.length,
-                    importSessionId: args.session.id,
-                    jobId: job.id,
-                    session: s,
-                    memories: r.memories,
-                    durationMs: r.durationMs,
-                    processingTimeMs: r.processingTimeMs,
-                    ttfmMs: r.ttfmMs,
-                    alreadyDone: r.alreadyDone,
-                    duplicate: r.duplicate,
-                });
-            }
-            catch (e) {
-                const status = responseStatus(e);
-                const errCode = responseData(e).error;
-                const message = responseMessage(e);
-                if (status === 422 && errCode === "ENCRYPTION_KEY_REQUIRED") {
-                    const durationMs = Date.now() - jobStartedAt;
-                    recordMetric(buildMigrationMetric({
-                        runId: args.runId,
-                        importSessionId: args.session.id,
-                        jobId: job.id,
-                        index: i,
-                        total: args.session.jobs.length,
-                        session: s,
-                        status: "stopped",
-                        durationMs,
-                        error: message,
-                        selection: args.selection,
-                    }));
-                    args.onProgress?.({ index: i, total: args.session.jobs.length, importSessionId: args.session.id, jobId: job.id, session: s, error: message, durationMs });
-                    return { migrated, extracted, failed, stoppedReason: "key-expired" };
-                }
-                failed++;
-                const durationMs = Date.now() - jobStartedAt;
+    // Run a single job: throttle the request START, extract, then record the outcome. Counters are only
+    // mutated between awaits, so single-threaded interleaving keeps them consistent without locks.
+    const runOne = async (job, s) => {
+        const jobStartedAt = Date.now();
+        try {
+            await throttle();
+            if (stoppedReason)
+                return; // an earlier job hit an unrecoverable stop — don't start more work
+            const r = await runImportJob(args.client, job.id, s, args.userTz, args.encKey);
+            extracted += r.memories;
+            migrated++;
+            done++;
+            ledger[s.conversationKey] = { size: s.size, mtimeMs: s.mtimeMs, status: "done", memories: r.memories };
+            saveLedger(ledger);
+            recordMetric(buildMigrationMetric({
+                runId: args.runId, importSessionId: args.session.id, jobId: job.id, index: done, total,
+                session: s, status: "completed", memories: r.memories, durationMs: r.durationMs,
+                processingTimeMs: r.processingTimeMs, ttfmMs: r.ttfmMs, alreadyDone: r.alreadyDone,
+                duplicate: r.duplicate, selection: args.selection,
+            }));
+            args.onProgress?.({
+                index: done, total, importSessionId: args.session.id, jobId: job.id, session: s,
+                memories: r.memories, durationMs: r.durationMs, processingTimeMs: r.processingTimeMs,
+                ttfmMs: r.ttfmMs, alreadyDone: r.alreadyDone, duplicate: r.duplicate,
+            });
+        }
+        catch (e) {
+            const status = responseStatus(e);
+            const errCode = responseData(e).error;
+            const message = responseMessage(e);
+            const durationMs = Date.now() - jobStartedAt;
+            if (status === 422 && errCode === "ENCRYPTION_KEY_REQUIRED") {
+                stoppedReason = "key-expired"; // the vault key expired mid-run — signal workers to stop pulling
+                done++;
                 recordMetric(buildMigrationMetric({
-                    runId: args.runId,
-                    importSessionId: args.session.id,
-                    jobId: job.id,
-                    index: i,
-                    total: args.session.jobs.length,
-                    session: s,
-                    status: "failed",
-                    durationMs,
-                    error: message,
-                    selection: args.selection,
+                    runId: args.runId, importSessionId: args.session.id, jobId: job.id, index: done, total,
+                    session: s, status: "stopped", durationMs, error: message, selection: args.selection,
                 }));
-                args.onProgress?.({ index: i, total: args.session.jobs.length, importSessionId: args.session.id, jobId: job.id, session: s, error: message, durationMs });
+                args.onProgress?.({ index: done, total, importSessionId: args.session.id, jobId: job.id, session: s, error: message, durationMs });
+                return;
             }
+            failed++;
+            done++;
+            recordMetric(buildMigrationMetric({
+                runId: args.runId, importSessionId: args.session.id, jobId: job.id, index: done, total,
+                session: s, status: "failed", durationMs, error: message, selection: args.selection,
+            }));
+            args.onProgress?.({ index: done, total, importSessionId: args.session.id, jobId: job.id, session: s, error: message, durationMs });
         }
+    };
+    // Worker pool: each worker pulls the next job until the list drains or a stop is signalled. The
+    // shared throttle keeps request STARTS under RATE_MAX/min regardless of how many run concurrently.
+    let next = 0;
+    const worker = async () => {
+        for (;;) {
+            if (stoppedReason)
+                return;
+            const myIdx = next++;
+            if (myIdx >= total)
+                return;
+            const job = args.session.jobs[myIdx];
+            const s = args.byConv.get(mapKey(job.platform, job.conversation_id));
+            if (!s)
+                continue;
+            await runOne(job, s);
+        }
+    };
+    try {
+        await Promise.all(Array.from({ length: Math.min(MIGRATE_CONCURRENCY, Math.max(1, total)) }, () => worker()));
     }
     catch {
         failed++;
     }
-    return { migrated, extracted, failed };
+    return { migrated, extracted, failed, ...(stoppedReason ? { stoppedReason } : {}) };
 }
 export async function cmdMigrate(flags) {
     const c = color(process.stdout.isTTY === true && !process.env.NO_COLOR && flags["no-color"] !== true);
@@ -756,7 +1049,25 @@ export async function cmdMigrate(flags) {
     const includeActive = flags["include-active"] === true;
     const selection = { since, limit, minChars, maxChars, largest, includeActive };
     const metricsFile = typeof flags["metrics-file"] === "string" ? flags["metrics-file"] : defaultMigrationMetricsPath();
-    const { sessions, pending, pendingTotal, alreadyMigrated, skippedActive, limited, codexCount, claudeCount } = discoverMigratableSessions(selection);
+    let discovery = discoverMigratableSessions(selection);
+    if (flags.estimate !== true && flags["dry-run"] !== true) {
+        const token = new KeyStore().getToken();
+        if (token) {
+            try {
+                discovery = await reconcileMigratableWithAccount(token, discovery, selection);
+            }
+            catch (e) {
+                if (isImportStatusUnsupported(e)) {
+                    discovery = markAccountImportStatusUnavailable(discovery);
+                }
+                else {
+                    discovery = markAccountImportStatusFailed(discovery);
+                    console.warn(`Could not check this EchoMem account's import status; using local migration ledger. ${responseMessage(e)}`);
+                }
+            }
+        }
+    }
+    const { sessions, pending, pendingTotal, alreadyMigrated, skippedActive, limited, codexCount, claudeCount, accountChecked, accountCheckFailed, accountCheckUnavailable } = discovery;
     const estimateSessions = includeActive ? sessions : sessions.filter((s) => !isActiveMigrationSession(s));
     if (!sessions.length) {
         console.log("No local Codex/Claude Code sessions found to migrate.");
@@ -768,12 +1079,19 @@ export async function cmdMigrate(flags) {
     }
     console.log("");
     console.log(c.bold(c.cyan("EchoMem migration")) + c.dim(`  (${API_BASE})`));
+    const pendingName = accountChecked ? "unprocessed" : "new/changed";
     const pendingText = limited
-        ? `${c.bold(String(pending.length))} of ${c.bold(String(pendingTotal))} new/changed selected`
-        : `${c.bold(String(pending.length))} new/changed to import`;
+        ? `${c.bold(String(pending.length))} of ${c.bold(String(pendingTotal))} ${pendingName} selected`
+        : `${c.bold(String(pending.length))} ${pendingName} to import`;
     console.log(`Found ${c.bold(String(sessions.length))} sessions (${codexCount} Codex + ${claudeCount} Claude Code) · ${pendingText}` +
         (alreadyMigrated ? c.dim(`, ${alreadyMigrated} already migrated`) : "") +
         (skippedActive ? c.dim(`, ${skippedActive} active skipped`) : ""));
+    if (accountChecked)
+        console.log(c.dim("Checked this EchoMem account; already processed conversations are skipped."));
+    else if (accountCheckFailed)
+        console.log(c.dim("Using local migration ledger because the account status check failed."));
+    else if (accountCheckUnavailable)
+        console.log(c.dim("Using local migration ledger; this API does not expose the optional account status precheck yet."));
     const filters = [
         minChars ? `min ${humanNum(minChars)} chars` : "",
         maxChars ? `max ${humanNum(maxChars)} chars` : "",

package/dist/report.js

@@ -510,6 +510,7 @@ export async function buildStatsPayload(stats, inject) {
     return {
         schemaVersion: 1,
         ...(inject?.partial ? { partial: true } : {}),
+        ...(inject?.discovery ? { discovery: inject.discovery } : {}),
         generatedFrom: ["~/.codex/sessions", "~/.claude/projects"],
         llmCallsUsed: 0,
         transcriptsUploaded: false,