onadit 0.0.1 → 1.0.0

package/dist/src/cli/commands/backfill.js

@@ -0,0 +1,1379 @@
+/**
+ * backfill CLI command — staged historical import.
+ *
+ * Imports web conversation exports from a dedicated staging area.
+ * The founder moves/copies exports into batch folders and runs:
+ *
+ *   onadit backfill --batch data/imports/exports-batch-1 --source chatgpt_web
+ *
+ * Slice 4 introduces a preflight-first workflow. Every run:
+ *   - refreshes `manifest.json` (catalog of batch files)
+ *   - refreshes `preflight.json` (junk + duplicate classification)
+ *
+ * The default operator flow is:
+ *   1. onadit backfill --batch ... --source ... --dry-run     (preflight only)
+ *   2. inspect manifest.json / preflight.json
+ *   3. onadit backfill --batch ... --source ...                (ingest)
+ *
+ * Reuses the existing POST /ingest pipeline (archive security,
+ * parsers, extraction, quarantine). Emits a per-batch report.
+ *
+ * Batch rerun is idempotent — the ingest pipeline's idempotency_key
+ * prevents duplicate conversations.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { Command } from "commander";
+import { loadConfig } from "../../config/loader.js";
+import { createHttpClient, readToken, resolveBaseUrl, CliError, requireExistingConfigPath, } from "../http-client.js";
+import { pollJob } from "../job-poller.js";
+import { isJobFailure } from "../output.js";
+import { isSourceSupported, INGEST_SOURCES, SUPPORTED_INGEST_SOURCES, } from "../../ingestion/parsers/types.js";
+import { stageBatch, SOURCE_EXTENSIONS, } from "../../ingestion/staging.js";
+import { extractArchive } from "../../security/archive.js";
+import { parseChatGPTWeb } from "../../ingestion/parsers/chatgpt-web.js";
+import { parseClaudeWeb } from "../../ingestion/parsers/claude-web.js";
+import { parseGeminiWeb } from "../../ingestion/parsers/gemini-web.js";
+import { parseGrokWeb } from "../../ingestion/parsers/grok-web.js";
+import { addToReview, reviewQueueSize } from "../../capture/review-queue.js";
+import { loadHistoricalCampaignState, writeHistoricalCampaignState, ensureHistoricalCampaignBatch, startHistoricalCampaignBatch, listTerminalHistoricalItemIds, listTerminalHistoricalFiles, isFileQuarantineCandidate, collectHistoricalBatchQuarantineReasons, markHistoricalFile, markHistoricalItem, finalizeHistoricalFiles, addHistoricalRunBudgetUsage, evaluateHistoricalBatch, getHistoricalCampaignBudgetWindow, recomputeHistoricalCampaignSummary, historicalCampaignReportPath, summarizeHistoricalBatchState, snapshotHistoricalCampaignReport, snapshotHistoricalNoOpReport, writeHistoricalCampaignReport, formatHistoricalCampaignSummary, reportHistoricalCampaignBlockers, } from "../../ingestion/historical-campaign.js";
+import { HISTORICAL_BACKFILL_CAPABILITY_HEADER, INTERNAL_HISTORICAL_INGEST_CODEX_ROUTE, INTERNAL_HISTORICAL_TRIAGE_CODEX_ROUTE, effectiveTriageProviderName, isOperatorOverrideProvider, readHistoricalBackfillCapability, triageMeteringModeForProvider, triageMeteringModeForProviderName, } from "../../providers/operator-overrides.js";
+const HISTORICAL_PARSERS = {
+    chatgpt_web: parseChatGPTWeb,
+    claude_web: parseClaudeWeb,
+    gemini_web: parseGeminiWeb,
+    grok_web: parseGrokWeb,
+};
+export function makeBackfillCommand() {
+    return new Command("backfill")
+        .description("Stage and import web conversation exports from a batch folder")
+        .requiredOption("--batch <path>", "Path to batch folder (e.g., data/imports/exports-batch-1)")
+        .requiredOption("--source <source>", "Source type (currently supported: chatgpt_web, claude_web, gemini_web, grok_web)")
+        .option("--dry-run", "Preflight only — refresh manifest/preflight and exit without ingesting")
+        .option("--force-unstaged", "Ingest files even if they are flagged as junk in preflight")
+        .option("--provider <provider>", "Backfill-only override for both triage and extraction (supported: codex-cli)")
+        .option("--triage-provider <provider>", "Backfill-only triage override (supported: codex-cli)")
+        .option("--extraction-provider <provider>", "Backfill-only extraction override (supported: codex-cli)")
+        .option("--limit <count>", "Cap this run to the first N unresolved files for smoke testing", parsePositiveInteger)
+        .action(async (opts, cmd) => {
+        const globalOpts = cmd.optsWithGlobals();
+        try {
+            // Validate source
+            const allSources = INGEST_SOURCES;
+            if (!allSources.includes(opts.source)) {
+                process.stderr.write(`Error: Unknown source "${opts.source}". ` +
+                    `Known sources: ${INGEST_SOURCES.join(", ")}\n`);
+                process.exitCode = 1;
+                return;
+            }
+            if (!isSourceSupported(opts.source)) {
+                process.stderr.write(`Error: "${opts.source}" is not yet supported — no real export fixture exists to validate the parser against.\n` +
+                    `To enable: add a real export sample to tests/fixtures/exports/${opts.source}/ and implement the parser.\n` +
+                    `Currently supported: ${SUPPORTED_INGEST_SOURCES.join(", ")}\n`);
+                process.exitCode = 1;
+                return;
+            }
+            // Validate batch directory
+            const batchDir = path.resolve(opts.batch);
+            if (!fs.existsSync(batchDir) ||
+                !fs.statSync(batchDir).isDirectory()) {
+                process.stderr.write(`Error: Not a directory: ${batchDir}\n`);
+                process.exitCode = 1;
+                return;
+            }
+            const source = opts.source;
+            const providerOverride = resolveBackfillProviderOverride(opts.provider, "provider");
+            const triageProviderOverride = resolveBackfillProviderOverride(opts.triageProvider ?? providerOverride, "triage-provider");
+            const extractionProviderOverride = resolveBackfillProviderOverride(opts.extractionProvider ?? providerOverride, "extraction-provider");
+            const requestedTriageMeteringMode = triageMeteringModeForProvider(triageProviderOverride);
+            const explicitLimit = typeof opts.limit === "number" && Number.isFinite(opts.limit)
+                ? opts.limit
+                : undefined;
+            // --- Stage the batch: manifest + preflight, persisted to disk. ---
+            const { manifest, preflight } = stageBatch({ batchDir, source });
+            const batchName = manifest.batch_name;
+            writePreflightSummary({
+                batchDir,
+                manifest,
+                preflight,
+                stderr: process.stderr,
+            });
+            if (opts.dryRun) {
+                if (globalOpts.json) {
+                    process.stdout.write(JSON.stringify({
+                        mode: "preflight",
+                        batch: batchName,
+                        source,
+                        manifest_path: path.join(batchDir, "manifest.json"),
+                        preflight_path: path.join(batchDir, "preflight.json"),
+                        preflight,
+                    }, null, 2) + "\n");
+                }
+                process.stderr.write("\n(dry-run — no changes made beyond manifest.json / preflight.json)\n");
+                return;
+            }
+            // Per A-2.2: route through requireExistingConfigPath so missing
+            // config produces the canonical CliError, not a raw ENOENT.
+            const configPath = requireExistingConfigPath(globalOpts);
+            const config = loadConfig(configPath);
+            const dataDir = config.data_paths.data_dir;
+            const now = new Date();
+            const nowIso = now.toISOString();
+            const reportPath = historicalCampaignReportPath(batchDir);
+            const campaignState = loadHistoricalCampaignState(dataDir);
+            const campaignBatch = ensureHistoricalCampaignBatch(campaignState, batchName, source, nowIso);
+            const reviewBacklogBefore = reviewQueueSize(dataDir);
+            const terminalItemIds = listTerminalHistoricalItemIds(campaignBatch);
+            const terminalFilesBefore = listTerminalHistoricalFiles(campaignBatch);
+            const rerunnableFiles = preflight.files
+                .filter((file) => !terminalFilesBefore.has(file.name))
+                .sort((left, right) => left.name.localeCompare(right.name));
+            for (const file of rerunnableFiles) {
+                ensureHistoricalPendingFile(campaignBatch, file.name, nowIso);
+            }
+            if (rerunnableFiles.length === 0) {
+                const noOpNextAction = "Batch is already terminal. Prepare the next staged batch.";
+                const terminalReport = snapshotHistoricalNoOpReport({
+                    state: campaignState,
+                    batch: campaignBatch,
+                    preflight,
+                    now,
+                    reviewBacklogNow: reviewBacklogBefore,
+                    nextAction: noOpNextAction,
+                });
+                campaignBatch.next_action = noOpNextAction;
+                campaignBatch.report_path = reportPath;
+                campaignState.last_run_at = nowIso;
+                await writeHistoricalCampaignReport(reportPath, terminalReport);
+                await writeHistoricalCampaignState(dataDir, campaignState);
+                if (globalOpts.json) {
+                    process.stdout.write(JSON.stringify(terminalReport, null, 2) + "\n");
+                }
+                else {
+                    process.stdout.write(formatHistoricalCampaignSummary(terminalReport));
+                }
+                return;
+            }
+            const budgetWindow = getHistoricalCampaignBudgetWindow(campaignState, now);
+            const runFileCount = Math.min(rerunnableFiles.length, budgetWindow.remaining_daily_files, budgetWindow.remaining_weekly_files, explicitLimit ?? Number.POSITIVE_INFINITY);
+            const filesForRun = rerunnableFiles.slice(0, runFileCount);
+            const deferredFiles = rerunnableFiles.slice(runFileCount);
+            const limitedByOperator = explicitLimit !== undefined &&
+                runFileCount === explicitLimit &&
+                rerunnableFiles.length > explicitLimit;
+            const preflightQuarantined = filesForRun.filter(isFileQuarantineCandidate);
+            const preflightQuarantineReasons = collectHistoricalBatchQuarantineReasons({
+                preflight,
+                failedFilesRate: 0,
+            });
+            const ingestable = filesForRun.filter((file) => {
+                if (preflightQuarantined.some((entry) => entry.name === file.name))
+                    return false;
+                if (!file.supported)
+                    return false;
+                if (file.junk_markers.length === 0)
+                    return true;
+                return Boolean(opts.forceUnstaged);
+            });
+            const skipped = filesForRun.filter((file) => !ingestable.includes(file) &&
+                !preflightQuarantined.some((entry) => entry.name === file.name));
+            const triageCandidates = ingestable
+                .flatMap((file) => buildHistoricalTriageItem(file, source))
+                .filter((item) => !terminalItemIds.has(item.id));
+            const triageCandidateById = new Map(triageCandidates.map((item) => [item.id, item]));
+            const cachedTriageById = new Map();
+            const requestedTriageProviderName = effectiveTriageProviderName(triageProviderOverride);
+            for (const candidate of triageCandidates) {
+                const existingItem = campaignBatch.items[candidate.id];
+                if (existingItem?.terminal || !existingItem?.triage)
+                    continue;
+                if (existingItem.triage.provider !== requestedTriageProviderName) {
+                    continue;
+                }
+                cachedTriageById.set(candidate.id, {
+                    item_id: candidate.id,
+                    file: candidate.file.name,
+                    kind: candidate.kind,
+                    ...(candidate.conversationId
+                        ? { conversation_id: candidate.conversationId }
+                        : {}),
+                    ...(candidate.title ? { title: candidate.title } : {}),
+                    ...(candidate.createdAt ? { created_at: candidate.createdAt } : {}),
+                    conversation_count: candidate.conversationCount,
+                    triage: existingItem.triage,
+                });
+            }
+            const triageCandidatesNeedingWork = triageCandidates.filter((candidate) => !cachedTriageById.has(candidate.id));
+            const blockers = reportHistoricalCampaignBlockers({
+                state: campaignState,
+                now,
+                reviewBacklogBefore,
+                meteringMode: requestedTriageMeteringMode,
+                triageBatchCostCapUsd: config.openai.max_cost_per_batch_usd,
+                triageCandidates: triageCandidatesNeedingWork.length,
+            });
+            if (runFileCount === 0 || blockers.length > 0) {
+                const nextAction = blockers.length > 0
+                    ? blockers.join("; ")
+                    : `Daily or weekly file budget is exhausted. Resume the remaining ${rerunnableFiles.length} file(s) in the next budget window.`;
+                const blockedStatus = blockers.some((reason) => reason.includes("triage spend") || reason.includes("review backlog"))
+                    ? "failed"
+                    : "running";
+                const previousBatchStatus = campaignBatch.status;
+                const preserveTerminalOutcome = previousBatchStatus === "failed" ||
+                    previousBatchStatus === "quarantined";
+                const finalBlockedStatus = preserveTerminalOutcome ? previousBatchStatus : blockedStatus;
+                startHistoricalCampaignBatch(campaignBatch, {
+                    totalFiles: preflight.total_files,
+                    runFiles: 0,
+                    triageCandidates: triageCandidates.length,
+                    reviewBacklogBefore,
+                    meteringMode: requestedTriageMeteringMode,
+                    nowIso,
+                });
+                campaignBatch.last_run_at = nowIso;
+                campaignBatch.status = finalBlockedStatus;
+                campaignBatch.review_backlog_after = reviewBacklogBefore;
+                campaignBatch.next_action = nextAction;
+                campaignBatch.report_path = reportPath;
+                const blockedReport = snapshotHistoricalCampaignReport({
+                    state: campaignState,
+                    batch: campaignBatch,
+                    preflight,
+                    now,
+                    runFileCount: 0,
+                    reviewBacklogAfter: reviewBacklogBefore,
+                    triageSpendUsd: 0,
+                    processedFiles: [],
+                    status: finalBlockedStatus,
+                    nextAction,
+                    results: [],
+                });
+                recomputeHistoricalCampaignSummary(campaignState);
+                campaignState.last_run_at = nowIso;
+                await writeHistoricalCampaignReport(reportPath, blockedReport);
+                await writeHistoricalCampaignState(dataDir, campaignState);
+                if (globalOpts.json) {
+                    process.stdout.write(JSON.stringify(blockedReport, null, 2) + "\n");
+                }
+                else {
+                    process.stdout.write(formatHistoricalCampaignSummary(blockedReport));
+                }
+                if (blockedStatus === "failed") {
+                    process.exitCode = 1;
+                }
+                return;
+            }
+            startHistoricalCampaignBatch(campaignBatch, {
+                totalFiles: preflight.total_files,
+                runFiles: runFileCount,
+                triageCandidates: triageCandidates.length,
+                reviewBacklogBefore,
+                meteringMode: requestedTriageMeteringMode,
+                nowIso,
+            });
+            for (const file of filesForRun) {
+                markHistoricalFile(campaignBatch, file.name, {
+                    terminal: false,
+                    lastResult: "running",
+                    nowIso,
+                });
+            }
+            const results = [];
+            let triageSpendUsd = 0;
+            let reviewBacklogAfter = reviewBacklogBefore;
+            const touchedFilesThisRun = new Set();
+            let stopReason = deferredFiles.length > 0
+                ? limitedByOperator
+                    ? `Processed ${runFileCount} file(s) in this smoke run. Resume the remaining ${deferredFiles.length} file(s) when ready.`
+                    : `Processed ${runFileCount} file(s) in this run. Resume the remaining ${deferredFiles.length} file(s) in the next daily or weekly budget window.`
+                : null;
+            let reviewBacklogStopped = false;
+            let budgetUsageCommitted = false;
+            await persistHistoricalCampaignCheckpoint({
+                dataDir,
+                state: campaignState,
+                batch: campaignBatch,
+                preflight,
+                now,
+                reportPath,
+                runFileCount,
+                reviewBacklogAfter,
+                triageSpendUsd,
+                processedFiles: filesForRun.map((file) => file.name),
+                results,
+                status: "running",
+                nextAction: stopReason ??
+                    `Running historical batch ${batchName}. Resume only unresolved files if interrupted.`,
+            });
+            try {
+                for (const file of preflightQuarantined) {
+                    touchedFilesThisRun.add(file.name);
+                    markHistoricalFile(campaignBatch, file.name, {
+                        terminal: true,
+                        terminalStatus: "quarantined",
+                        lastResult: "quarantined",
+                        nowIso,
+                    });
+                    results.push({
+                        file: file.name,
+                        status: "quarantined",
+                        error: file.junk_markers.join(",") ||
+                            file.notes.join(",") ||
+                            "quarantined_by_policy",
+                    });
+                }
+                await persistHistoricalCampaignCheckpoint({
+                    dataDir,
+                    state: campaignState,
+                    batch: campaignBatch,
+                    preflight,
+                    now,
+                    reportPath,
+                    runFileCount,
+                    reviewBacklogAfter,
+                    triageSpendUsd,
+                    processedFiles: filesForRun.map((file) => file.name),
+                    results,
+                    status: "running",
+                    nextAction: stopReason ??
+                        `Running historical batch ${batchName}. Resume only unresolved files if interrupted.`,
+                });
+                if (preflightQuarantineReasons.length > 0) {
+                    for (const file of filesForRun.filter((candidate) => !preflightQuarantined.some((entry) => entry.name === candidate.name))) {
+                        touchedFilesThisRun.add(file.name);
+                        markHistoricalFile(campaignBatch, file.name, {
+                            terminal: true,
+                            terminalStatus: "quarantined",
+                            lastResult: "quarantined",
+                            nowIso,
+                        });
+                        results.push({
+                            file: file.name,
+                            status: "quarantined",
+                            error: `batch_quarantined:${preflightQuarantineReasons.join(";")}`,
+                        });
+                    }
+                    finalizeHistoricalFiles(campaignBatch, nowIso);
+                    addHistoricalRunBudgetUsage(campaignState, {
+                        weekKey: budgetWindow.week_key,
+                        dayKey: budgetWindow.day_key,
+                        filesProcessed: touchedFilesThisRun.size,
+                        triageSpendUsd,
+                    });
+                    budgetUsageCommitted = true;
+                    const evaluation = evaluateHistoricalBatch({
+                        state: campaignState,
+                        batch: campaignBatch,
+                        preflight,
+                        now,
+                        runFileCount,
+                        reviewBacklogAfter,
+                        triageSpendUsd,
+                        reportPath,
+                        processedFiles: filesForRun.map((file) => file.name),
+                        results,
+                    });
+                    campaignBatch.report_path = reportPath;
+                    campaignState.last_run_at = nowIso;
+                    await writeHistoricalCampaignReport(reportPath, evaluation.report);
+                    await writeHistoricalCampaignState(dataDir, campaignState);
+                    if (globalOpts.json) {
+                        process.stdout.write(JSON.stringify(evaluation.report, null, 2) + "\n");
+                    }
+                    else {
+                        process.stdout.write(formatHistoricalCampaignSummary(evaluation.report));
+                    }
+                    process.exitCode = 1;
+                    return;
+                }
+                for (const file of skipped) {
+                    touchedFilesThisRun.add(file.name);
+                    markHistoricalFile(campaignBatch, file.name, {
+                        terminal: false,
+                        lastResult: "skipped",
+                        nowIso,
+                    });
+                    results.push({
+                        file: file.name,
+                        status: "skipped",
+                        error: file.junk_markers.join(",") || "skipped_by_preflight",
+                    });
+                }
+                await persistHistoricalCampaignCheckpoint({
+                    dataDir,
+                    state: campaignState,
+                    batch: campaignBatch,
+                    preflight,
+                    now,
+                    reportPath,
+                    runFileCount,
+                    reviewBacklogAfter,
+                    triageSpendUsd,
+                    processedFiles: filesForRun.map((file) => file.name),
+                    results,
+                    status: "running",
+                    nextAction: stopReason ??
+                        `Running historical batch ${batchName}. Resume only unresolved files if interrupted.`,
+                });
+                const triageById = new Map(cachedTriageById);
+                let client = null;
+                if (triageCandidates.length > 0) {
+                    const baseUrl = resolveBaseUrl(globalOpts.config);
+                    const token = await readToken();
+                    if (!token) {
+                        throw new Error("No auth token. Run: onadit token issue");
+                    }
+                    client = createHttpClient({ baseUrl, token });
+                }
+                if (triageCandidatesNeedingWork.length > 0) {
+                    const triageOutput = await runHistoricalTriage({
+                        batchDir,
+                        client: client,
+                        dataDir,
+                        items: triageCandidatesNeedingWork,
+                        ...(triageProviderOverride
+                            ? { providerOverride: triageProviderOverride }
+                            : {}),
+                    });
+                    triageSpendUsd = triageOutput.spent_usd ?? 0;
+                    if (triageOutput.metering_mode) {
+                        campaignBatch.metering_mode = triageOutput.metering_mode;
+                    }
+                    for (const item of triageOutput.items) {
+                        triageById.set(item.item_id, item);
+                    }
+                    for (const candidate of triageCandidatesNeedingWork) {
+                        const triageItem = triageById.get(candidate.id);
+                        if (!triageItem?.triage)
+                            continue;
+                        touchedFilesThisRun.add(candidate.file.name);
+                        markHistoricalItem(campaignBatch, {
+                            itemId: candidate.id,
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversationId: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            lastResult: "triaged",
+                            terminal: false,
+                            triage: triageItem.triage,
+                            nowIso,
+                        });
+                    }
+                    await persistHistoricalCampaignCheckpoint({
+                        dataDir,
+                        state: campaignState,
+                        batch: campaignBatch,
+                        preflight,
+                        now,
+                        reportPath,
+                        runFileCount,
+                        reviewBacklogAfter,
+                        triageSpendUsd,
+                        processedFiles: filesForRun.map((file) => file.name),
+                        results,
+                        status: "running",
+                        nextAction: stopReason ??
+                            `Running historical batch ${batchName}. Resume only unresolved files if interrupted.`,
+                    });
+                }
+                const actualTriageProviderName = resolveHistoricalRunTriageProviderName(triageCandidates, triageById);
+                if (actualTriageProviderName) {
+                    campaignBatch.metering_mode = triageMeteringModeForProviderName(actualTriageProviderName);
+                }
+                for (const candidate of triageCandidates) {
+                    const label = formatHistoricalLabel(candidate);
+                    const triageItem = triageById.get(candidate.id);
+                    if (!triageItem) {
+                        touchedFilesThisRun.add(candidate.file.name);
+                        markHistoricalItem(campaignBatch, {
+                            itemId: candidate.id,
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversationId: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            lastResult: "error",
+                            terminal: false,
+                            nowIso,
+                        });
+                        results.push({
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversation_id: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            status: "error",
+                            error: "missing_triage_result",
+                        });
+                        process.stderr.write(`  ${label}: missing triage result\n`);
+                        continue;
+                    }
+                    if (triageItem.error) {
+                        touchedFilesThisRun.add(candidate.file.name);
+                        markHistoricalItem(campaignBatch, {
+                            itemId: candidate.id,
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversationId: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            lastResult: "error",
+                            terminal: false,
+                            nowIso,
+                        });
+                        results.push({
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversation_id: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            status: "error",
+                            error: triageItem.error,
+                        });
+                        process.stderr.write(`  ${label}: triage error: ${triageItem.error}\n`);
+                        continue;
+                    }
+                    if (!triageItem.triage) {
+                        touchedFilesThisRun.add(candidate.file.name);
+                        markHistoricalItem(campaignBatch, {
+                            itemId: candidate.id,
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversationId: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            lastResult: "error",
+                            terminal: false,
+                            nowIso,
+                        });
+                        results.push({
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversation_id: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            status: "error",
+                            error: "triage_result_missing",
+                        });
+                        process.stderr.write(`  ${label}: triage result missing\n`);
+                        continue;
+                    }
+                    const triage = triageItem.triage;
+                    process.stderr.write(`  ${label}: ${triage.lane} (score=${triage.score.toFixed(3)}, decision=${triage.decision})\n`);
+                    if (triage.lane === "archive_only") {
+                        touchedFilesThisRun.add(candidate.file.name);
+                        markHistoricalItem(campaignBatch, {
+                            itemId: candidate.id,
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversationId: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            lastResult: "archive_only",
+                            terminal: true,
+                            terminalStatus: "archive_only",
+                            triage,
+                            nowIso,
+                        });
+                        results.push({
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversation_id: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            status: "archive_only",
+                            lane: triage.lane,
+                        });
+                        continue;
+                    }
+                    if (triage.lane === "review") {
+                        touchedFilesThisRun.add(candidate.file.name);
+                        await addToReview(dataDir, {
+                            source,
+                            conversationId: candidate.kind === "conversation" && candidate.conversationId
+                                ? historicalConversationKey(source, candidate.conversationId)
+                                : `backfill:${batchName}:${candidate.file.name}`,
+                            summary: triage.summary ||
+                                candidate.title ||
+                                candidate.file.name,
+                            content: triageCandidateById.get(candidate.id)?.reviewContent ??
+                                candidate.file.name,
+                            triage,
+                        });
+                        markHistoricalItem(campaignBatch, {
+                            itemId: candidate.id,
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversationId: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            lastResult: "review",
+                            terminal: true,
+                            terminalStatus: "review",
+                            triage,
+                            nowIso,
+                        });
+                        results.push({
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversation_id: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            status: "review",
+                            lane: triage.lane,
+                        });
+                        reviewBacklogAfter = reviewQueueSize(dataDir);
+                        if (reviewBacklogAfter >
+                            400) {
+                            stopReason =
+                                "Stop historical import. Review backlog exceeds 400 open items.";
+                            reviewBacklogStopped = true;
+                            process.stderr.write(`  ${label}: stopping batch because review backlog is now ${reviewBacklogAfter}\n`);
+                            break;
+                        }
+                        continue;
+                    }
+                    const upload = candidate.upload;
+                    if (!upload) {
+                        touchedFilesThisRun.add(candidate.file.name);
+                        const error = candidate.kind === "file_fallback"
+                            ? "parser_zero_conversations"
+                            : "missing_promoted_payload";
+                        markHistoricalItem(campaignBatch, {
+                            itemId: candidate.id,
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversationId: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            lastResult: "error",
+                            terminal: false,
+                            triage,
+                            nowIso,
+                        });
+                        results.push({
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversation_id: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            status: "error",
+                            lane: triage.lane,
+                            error,
+                        });
+                        process.stderr.write(`    Error: ${error} for ${label}\n`);
+                        continue;
+                    }
+                    process.stderr.write(`    Promoting ${label} into /ingest\n`);
+                    try {
+                        if (!client) {
+                            throw new Error("Missing HTTP client for historical promotion");
+                        }
+                        const form = new FormData();
+                        form.append("source", source);
+                        // Fastify's multipart parser validates fields available at the
+                        // moment it encounters the file part. Keep the historical
+                        // idempotency key ahead of the file so larger uploads don't
+                        // lose it and fail the internal override route validation.
+                        form.append("idempotency_key", candidate.kind === "conversation" && candidate.conversationId
+                            ? historicalConversationKey(source, candidate.conversationId)
+                            : `backfill:${batchName}:${candidate.file.name}`);
+                        form.append("file", new Blob([new Uint8Array(upload.fileData)]), upload.filename);
+                        const { data } = await client.postMultipart(historicalIngestEndpointPath(extractionProviderOverride), form, historicalInternalRouteOptions(dataDir, extractionProviderOverride));
+                        const resp = data;
+                        if (resp.duplicate) {
+                            touchedFilesThisRun.add(candidate.file.name);
+                            markHistoricalItem(campaignBatch, {
+                                itemId: candidate.id,
+                                file: candidate.file.name,
+                                ...(candidate.conversationId
+                                    ? { conversationId: candidate.conversationId }
+                                    : {}),
+                                ...(candidate.title ? { title: candidate.title } : {}),
+                                lastResult: "duplicate",
+                                terminal: true,
+                                terminalStatus: "promote_to_extract",
+                                triage,
+                                nowIso,
+                            });
+                            process.stderr.write("    Duplicate (already ingested)\n");
+                            results.push({
+                                file: candidate.file.name,
+                                ...(candidate.conversationId
+                                    ? { conversation_id: candidate.conversationId }
+                                    : {}),
+                                ...(candidate.title ? { title: candidate.title } : {}),
+                                status: "duplicate",
+                                lane: triage.lane,
+                            });
+                            continue;
+                        }
+                        const result = await pollJob({
+                            get: client.get.bind(client),
+                            jobId: resp.job_id,
+                        });
+                        if (isJobFailure(result.job, result.timedOut)) {
+                            touchedFilesThisRun.add(candidate.file.name);
+                            const error = result.job.error ??
+                                (result.timedOut ? "timed out" : "unknown");
+                            markHistoricalItem(campaignBatch, {
+                                itemId: candidate.id,
+                                file: candidate.file.name,
+                                ...(candidate.conversationId
+                                    ? { conversationId: candidate.conversationId }
+                                    : {}),
+                                ...(candidate.title ? { title: candidate.title } : {}),
+                                lastResult: "failed",
+                                terminal: false,
+                                triage,
+                                nowIso,
+                            });
+                            results.push({
+                                file: candidate.file.name,
+                                ...(candidate.conversationId
+                                    ? { conversation_id: candidate.conversationId }
+                                    : {}),
+                                ...(candidate.title ? { title: candidate.title } : {}),
+                                status: "failed",
+                                lane: triage.lane,
+                                error,
+                            });
+                            process.stderr.write(`    Failed: ${error}\n`);
+                        }
+                        else {
+                            touchedFilesThisRun.add(candidate.file.name);
+                            markHistoricalItem(campaignBatch, {
+                                itemId: candidate.id,
+                                file: candidate.file.name,
+                                ...(candidate.conversationId
+                                    ? { conversationId: candidate.conversationId }
+                                    : {}),
+                                ...(candidate.title ? { title: candidate.title } : {}),
+                                lastResult: "promote_to_extract",
+                                terminal: true,
+                                terminalStatus: "promote_to_extract",
+                                triage,
+                                nowIso,
+                            });
+                            results.push({
+                                file: candidate.file.name,
+                                ...(candidate.conversationId
+                                    ? { conversation_id: candidate.conversationId }
+                                    : {}),
+                                ...(candidate.title ? { title: candidate.title } : {}),
+                                status: "completed",
+                                lane: triage.lane,
+                            });
+                            process.stderr.write("    Completed\n");
+                        }
+                    }
+                    catch (err) {
+                        touchedFilesThisRun.add(candidate.file.name);
+                        const msg = err instanceof CliError
+                            ? formatCliErrorForHistoricalBackfill(err)
+                            : err instanceof Error
+                                ? err.message
+                                : String(err);
+                        markHistoricalItem(campaignBatch, {
+                            itemId: candidate.id,
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversationId: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            lastResult: "error",
+                            terminal: false,
+                            triage,
+                            nowIso,
+                        });
+                        results.push({
+                            file: candidate.file.name,
+                            ...(candidate.conversationId
+                                ? { conversation_id: candidate.conversationId }
+                                : {}),
+                            ...(candidate.title ? { title: candidate.title } : {}),
+                            status: "error",
+                            lane: triage.lane,
+                            error: msg,
+                        });
+                        process.stderr.write(`    Error: ${msg}\n`);
+                    }
+                }
+                finalizeHistoricalFiles(campaignBatch, nowIso);
+                reviewBacklogAfter = reviewQueueSize(dataDir);
+                addHistoricalRunBudgetUsage(campaignState, {
+                    weekKey: budgetWindow.week_key,
+                    dayKey: budgetWindow.day_key,
+                    filesProcessed: touchedFilesThisRun.size,
+                    triageSpendUsd,
+                });
+                budgetUsageCommitted = true;
+                const hasProcessingErrors = results.some((result) => result.status === "failed" || result.status === "error");
+                const operatorLimitedCompletion = limitedByOperator && !hasProcessingErrors;
+                const evaluation = evaluateHistoricalBatch({
+                    state: campaignState,
+                    batch: campaignBatch,
+                    preflight,
+                    now,
+                    runFileCount,
+                    reviewBacklogAfter,
+                    triageSpendUsd,
+                    reportPath,
+                    processedFiles: filesForRun.map((file) => file.name),
+                    results,
+                    ...(!operatorLimitedCompletion &&
+                        !reviewBacklogStopped &&
+                        stopReason &&
+                        !hasProcessingErrors
+                        ? { incompleteStatus: "running" }
+                        : {}),
+                });
+                const isResumableIncompleteStatus = evaluation.report.status === "running";
+                if (operatorLimitedCompletion &&
+                    isResumableIncompleteStatus &&
+                    evaluation.report.status === "running") {
+                    evaluation.report.status = "completed_with_warnings";
+                    evaluation.report.next_action = stopReason;
+                    campaignBatch.status = "completed_with_warnings";
+                    campaignBatch.next_action = stopReason;
+                    recomputeHistoricalCampaignSummary(campaignState);
+                }
+                else if (reviewBacklogStopped &&
+                    !hasProcessingErrors &&
+                    isResumableIncompleteStatus &&
+                    evaluation.report.status === "running") {
+                    evaluation.report.status = "completed_with_warnings";
+                    evaluation.report.next_action = stopReason;
+                    campaignBatch.status = "completed_with_warnings";
+                    campaignBatch.next_action = stopReason;
+                    recomputeHistoricalCampaignSummary(campaignState);
+                }
+                else if (stopReason && !hasProcessingErrors && isResumableIncompleteStatus) {
+                    evaluation.report.status = "running";
+                    evaluation.report.next_action = stopReason;
+                    campaignBatch.status = "running";
+                    campaignBatch.next_action = stopReason;
+                    recomputeHistoricalCampaignSummary(campaignState);
+                }
+                campaignBatch.report_path = reportPath;
+                campaignState.last_run_at = nowIso;
+                await writeHistoricalCampaignReport(reportPath, evaluation.report);
+                await writeHistoricalCampaignState(dataDir, campaignState);
+                if (globalOpts.json) {
+                    process.stdout.write(JSON.stringify(evaluation.report, null, 2) + "\n");
+                }
+                else {
+                    process.stdout.write(formatHistoricalCampaignSummary(evaluation.report));
+                }
+                if (campaignBatch.status === "failed" || campaignBatch.status === "quarantined") {
+                    process.exitCode = 1;
+                }
+            }
+            catch (runErr) {
+                finalizeHistoricalFiles(campaignBatch, new Date().toISOString());
+                reviewBacklogAfter = reviewQueueSize(dataDir);
+                if (!budgetUsageCommitted && (triageSpendUsd > 0 || results.length > 0)) {
+                    addHistoricalRunBudgetUsage(campaignState, {
+                        weekKey: budgetWindow.week_key,
+                        dayKey: budgetWindow.day_key,
+                        filesProcessed: touchedFilesThisRun.size,
+                        triageSpendUsd,
+                    });
+                    budgetUsageCommitted = true;
+                }
+                await persistHistoricalCampaignCheckpoint({
+                    dataDir,
+                    state: campaignState,
+                    batch: campaignBatch,
+                    preflight,
+                    now,
+                    reportPath,
+                    runFileCount,
+                    reviewBacklogAfter,
+                    triageSpendUsd,
+                    processedFiles: filesForRun.map((file) => file.name),
+                    results,
+                    status: "failed",
+                    nextAction: "Batch failed mid-run. Resume only unresolved files after fixing the error.",
+                    incompleteStatus: "failed",
+                });
+                throw runErr;
+            }
+        }
+        catch (err) {
+            process.stderr.write(`Error: ${err instanceof CliError ? err.message : err instanceof Error ? err.message : String(err)}\n`);
+            process.exitCode = 1;
+        }
+    });
+}
+function ensureHistoricalPendingFile(batch, fileName, nowIso) {
+    batch.files[fileName] ??= {
+        file: fileName,
+        terminal: false,
+        item_ids: [],
+        last_result: "pending",
+        attempts: 0,
+        updated_at: nowIso,
+    };
+}
+async function persistHistoricalCampaignCheckpoint(params) {
+    let report;
+    params.batch.report_path = params.reportPath;
+    params.batch.next_action = params.nextAction;
+    if (params.status === "running") {
+        params.batch.status = "running";
+        report = snapshotHistoricalCampaignReport({
+            state: params.state,
+            batch: params.batch,
+            preflight: params.preflight,
+            now: params.now,
+            runFileCount: params.runFileCount,
+            reviewBacklogAfter: params.reviewBacklogAfter,
+            triageSpendUsd: params.triageSpendUsd,
+            processedFiles: params.processedFiles,
+            status: "running",
+            nextAction: params.nextAction,
+            nextBatchSize: currentHistoricalBatchLevel(params.batch.total_files),
+            results: params.results,
+        });
+        params.batch.counts = summarizeHistoricalBatchState(params.batch, params.preflight).counts;
+        params.batch.review_backlog_after = params.reviewBacklogAfter;
+    }
+    else {
+        const evaluation = evaluateHistoricalBatch({
+            state: params.state,
+            batch: params.batch,
+            preflight: params.preflight,
+            now: params.now,
+            runFileCount: params.runFileCount,
+            reviewBacklogAfter: params.reviewBacklogAfter,
+            triageSpendUsd: params.triageSpendUsd,
+            reportPath: params.reportPath,
+            processedFiles: params.processedFiles,
+            results: params.results,
+            ...(params.incompleteStatus
+                ? { incompleteStatus: params.incompleteStatus }
+                : {}),
+        });
+        report = evaluation.report;
+        report.status = params.status;
+        report.next_action = params.nextAction;
+        report.results = params.results;
+        params.batch.status = params.status;
+        params.batch.next_action = params.nextAction;
+    }
+    recomputeHistoricalCampaignSummary(params.state);
+    params.state.last_run_at = params.now.toISOString();
+    await writeHistoricalCampaignReport(params.reportPath, report);
+    await writeHistoricalCampaignState(params.dataDir, params.state);
+    return report;
+}
+function currentHistoricalBatchLevel(totalFiles) {
+    if (totalFiles <= 100)
+        return 100;
+    if (totalFiles <= 250)
+        return 250;
+    return 500;
+}
+function buildHistoricalTriageItem(file, source) {
+    const fileBuffer = fs.readFileSync(file.path);
+    const extracted = extractArchive(fileBuffer, file.name);
+    const parser = HISTORICAL_PARSERS[source];
+    if (!parser) {
+        throw new Error(`No historical triage parser for source: ${source}`);
+    }
+    const parsed = parser(extracted.files);
+    if (parsed.length === 0) {
+        const fallback = fileBuffer.toString("utf-8").slice(0, 12_000);
+        return [{
+                id: file.name,
+                kind: "file_fallback",
+                source,
+                file,
+                triageContent: fallback || `File: ${file.name}`,
+                reviewContent: fallback || `File: ${file.name}`,
+                conversationCount: 0,
+            }];
+    }
+    const rawByConversationId = collectRawConversations(source, extracted.files);
+    return parsed.map((conversation) => {
+        const reviewContent = buildConversationReviewContent({
+            source,
+            fileName: file.name,
+            conversationId: conversation.id,
+            title: conversation.title,
+            createdAt: conversation.createdAt,
+            content: conversation.content,
+        });
+        return {
+            id: historicalTriageItemId(file.name, conversation.id),
+            kind: "conversation",
+            source,
+            file,
+            conversationId: conversation.id,
+            title: conversation.title,
+            createdAt: conversation.createdAt,
+            triageContent: reviewContent.slice(0, 12_000),
+            reviewContent,
+            conversationCount: 1,
+            ...(rawByConversationId.has(conversation.id)
+                ? {
+                    upload: buildSyntheticConversationUpload(source, conversation.id, rawByConversationId.get(conversation.id)),
+                }
+                : {}),
+        };
+    });
+}
+async function runHistoricalTriage(params) {
+    const endpointPath = historicalTriageEndpointPath(params.providerOverride);
+    const body = {
+        items: params.items.map((item) => ({
+            id: item.id,
+            content: item.triageContent,
+            source: item.source,
+        })),
+        idempotency_key: historicalTriageIdempotencyKey(params.batchDir, params.items, params.providerOverride),
+    };
+    const { data } = await params.client.post(endpointPath, body, historicalInternalRouteOptions(params.dataDir, params.providerOverride));
+    const resp = data;
+    const triageJob = await pollJob({
+        get: params.client.get.bind(params.client),
+        jobId: resp.job_id,
+    });
+    if (isJobFailure(triageJob.job, triageJob.timedOut)) {
+        throw new Error(triageJob.job.error ??
+            (triageJob.timedOut ? "triage timed out" : "triage failed"));
+    }
+    const resultsPath = path.join(params.dataDir, "triage-results", `${resp.job_id}.json`);
+    if (!fs.existsSync(resultsPath)) {
+        throw new Error(`Missing triage results file: ${resultsPath}`);
+    }
+    const output = JSON.parse(fs.readFileSync(resultsPath, "utf-8"));
+    const artifact = {
+        batch: path.basename(params.batchDir),
+        source: params.items[0]?.source,
+        generated_at: new Date().toISOString(),
+        ...(output.metering_mode ? { metering_mode: output.metering_mode } : {}),
+        ...(output.spend_note ? { spend_note: output.spend_note } : {}),
+        ...(typeof output.spent_usd === "number"
+            ? { spent_usd: output.spent_usd }
+            : {}),
+        items: params.items.map((item) => {
+            const result = output.items.find((entry) => entry.id === item.id);
+            return {
+                item_id: item.id,
+                file: item.file.name,
+                kind: item.kind,
+                ...(item.conversationId
+                    ? { conversation_id: item.conversationId }
+                    : {}),
+                ...(item.title ? { title: item.title } : {}),
+                ...(item.createdAt ? { created_at: item.createdAt } : {}),
+                conversation_count: item.conversationCount,
+                ...(result?.triage ? { triage: result.triage } : {}),
+                ...(result?.error ? { error: result.error } : {}),
+            };
+        }),
+    };
+    fs.writeFileSync(path.join(params.batchDir, "triage.json"), JSON.stringify(artifact, null, 2) + "\n", "utf-8");
+    return artifact;
+}
+function writePreflightSummary(params) {
+    const { batchDir, manifest, preflight, stderr } = params;
+    const allowed = SOURCE_EXTENSIONS[manifest.source].join(", ");
+    stderr.write(`Preflight: ${manifest.batch_name}\n` +
+        `  Path:              ${batchDir}\n` +
+        `  Source:            ${manifest.source} (allowed extensions: ${allowed})\n` +
+        `  Total files:       ${preflight.total_files}\n` +
+        `  Supported:         ${preflight.supported_files}\n` +
+        `  Unsupported:       ${preflight.unsupported_files}\n` +
+        `  Junk candidates:   ${preflight.junk_candidates}\n` +
+        `  Duplicate hashes:  ${preflight.duplicate_candidates}\n` +
+        `  Size (min/med/max/total bytes): ${preflight.size_distribution.min}/${preflight.size_distribution.median}/${preflight.size_distribution.max}/${preflight.size_distribution.total}\n` +
+        `  Manifest:          ${path.join(batchDir, "manifest.json")}\n` +
+        `  Preflight:         ${path.join(batchDir, "preflight.json")}\n`);
+    const flagged = preflight.files.filter((f) => f.junk_markers.length > 0);
+    if (flagged.length > 0) {
+        stderr.write(`\nFlagged files:\n`);
+        for (const f of flagged) {
+            stderr.write(`  - ${f.name}: ${f.junk_markers.join(", ")}\n`);
+        }
+    }
+}
+function historicalTriageItemId(fileName, conversationId) {
+    return `${fileName}::${conversationId}`;
+}
+function historicalConversationKey(source, conversationId) {
+    return `historical:${source}:${conversationId}`;
+}
+function formatHistoricalLabel(item) {
+    if (item.kind === "conversation" && item.conversationId) {
+        return `${item.file.name}#${item.conversationId}`;
+    }
+    return item.file.name;
+}
+function buildConversationReviewContent(params) {
+    return [
+        `Source: ${params.source}`,
+        `File: ${params.fileName}`,
+        `Conversation ID: ${params.conversationId}`,
+        `Title: ${params.title}`,
+        `Created: ${params.createdAt}`,
+        "",
+        params.content,
+    ].join("\n");
+}
+function collectRawConversations(source, files) {
+    switch (source) {
+        case "chatgpt_web":
+            return collectRawChatGptConversations(files);
+        case "claude_web":
+            return collectRawClaudeConversations(files);
+        case "gemini_web":
+            return collectRawGeminiConversations(files);
+        case "grok_web":
+            return collectRawGrokConversations(files);
+        default:
+            return new Map();
+    }
+}
+function collectRawChatGptConversations(files) {
+    const raw = new Map();
+    for (const file of files) {
+        if (!file.path.toLowerCase().endsWith(".json"))
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(file.data.toString("utf-8"));
+        }
+        catch {
+            continue;
+        }
+        if (!Array.isArray(parsed))
+            continue;
+        for (const entry of parsed) {
+            if (!entry || typeof entry !== "object")
+                continue;
+            const conversation = entry;
+            const id = typeof conversation.conversation_id === "string"
+                ? conversation.conversation_id
+                : typeof conversation.id === "string"
+                    ? conversation.id
+                    : null;
+            if (!id || raw.has(id))
+                continue;
+            raw.set(id, entry);
+        }
+    }
+    return raw;
+}
+function collectRawClaudeConversations(files) {
+    const raw = new Map();
+    for (const file of files) {
+        const lower = file.path.toLowerCase();
+        if (!lower.endsWith(".json"))
+            continue;
+        const basename = lower.split("/").pop() ?? "";
+        if (basename !== "conversations.json" &&
+            !basename.startsWith("conversations")) {
+            continue;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(file.data.toString("utf-8"));
+        }
+        catch {
+            continue;
+        }
+        if (!Array.isArray(parsed))
+            continue;
+        for (const entry of parsed) {
+            if (!entry || typeof entry !== "object")
+                continue;
+            const conversation = entry;
+            if (typeof conversation.uuid !== "string" || raw.has(conversation.uuid)) {
+                continue;
+            }
+            raw.set(conversation.uuid, entry);
+        }
+    }
+    return raw;
+}
+function collectRawGeminiConversations(files) {
+    const raw = new Map();
+    for (const file of files) {
+        if (!file.path.toLowerCase().endsWith(".json"))
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(file.data.toString("utf-8"));
+        }
+        catch {
+            continue;
+        }
+        if (!Array.isArray(parsed))
+            continue;
+        for (const entry of parsed) {
+            if (!entry || typeof entry !== "object")
+                continue;
+            const conversation = entry;
+            if (typeof conversation.conversation_id !== "string" ||
+                conversation.source !== "gemini_web" ||
+                conversation.operative_unit !== "gemini_activity_card" ||
+                raw.has(conversation.conversation_id)) {
+                continue;
+            }
+            raw.set(conversation.conversation_id, entry);
+        }
+    }
+    return raw;
+}
+function collectRawGrokConversations(files) {
+    const raw = new Map();
+    for (const file of files) {
+        if (!file.path.toLowerCase().endsWith(".json"))
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(file.data.toString("utf-8"));
+        }
+        catch {
+            continue;
+        }
+        const conversations = parsed?.conversations;
+        if (!Array.isArray(conversations))
+            continue;
+        for (const wrapper of conversations) {
+            if (!wrapper || typeof wrapper !== "object")
+                continue;
+            const id = wrapper.conversation?.id;
+            if (typeof id !== "string" || raw.has(id))
+                continue;
+            raw.set(id, wrapper);
+        }
+    }
+    return raw;
+}
+function buildSyntheticConversationUpload(source, conversationId, rawConversation) {
+    // Grok's parser expects the top-level export shape
+    // `{ conversations: [...], projects, tasks, media_posts }`
+    // rather than a bare array of conversations, so the synthetic
+    // single-row upload has to preserve that wrapper.
+    const payload = source === "grok_web"
+        ? { conversations: [rawConversation], projects: [], tasks: [], media_posts: [] }
+        : [rawConversation];
+    return {
+        filename: syntheticConversationFilename(source, conversationId),
+        fileData: Buffer.from(JSON.stringify(payload, null, 2) + "\n", "utf-8"),
+    };
+}
+function syntheticConversationFilename(source, conversationId) {
+    const safeConversationId = conversationId.replace(/[^a-zA-Z0-9_-]/g, "_");
+    if (source === "claude_web") {
+        return `conversations-${safeConversationId}.json`;
+    }
+    return `conversation-${safeConversationId}.json`;
+}
+function historicalTriageIdempotencyKey(batchDir, items, providerOverride) {
+    const digest = crypto
+        .createHash("sha256")
+        .update(JSON.stringify({
+        batch_dir: path.resolve(batchDir),
+        source: items[0]?.source,
+        triage_provider: effectiveTriageProviderName(providerOverride),
+        items: items
+            .map((item) => ({
+            id: item.id,
+            file: item.file.name,
+            file_path: item.file.path,
+            content_hash: item.file.content_hash,
+            conversation_id: item.conversationId,
+        }))
+            .sort((a, b) => a.id.localeCompare(b.id)),
+    }))
+        .digest("hex")
+        .slice(0, 24);
+    return `backfill-triage:${digest}`;
+}
+function resolveBackfillProviderOverride(value, flagName) {
+    if (value === undefined)
+        return undefined;
+    if (!isOperatorOverrideProvider(value)) {
+        throw new Error(`Unsupported ${flagName} "${String(value)}". Supported overrides: codex-cli`);
+    }
+    return value;
+}
+function historicalTriageEndpointPath(providerOverride) {
+    return providerOverride === "codex-cli"
+        ? INTERNAL_HISTORICAL_TRIAGE_CODEX_ROUTE
+        : "/triage";
+}
+function historicalIngestEndpointPath(providerOverride) {
+    return providerOverride === "codex-cli"
+        ? INTERNAL_HISTORICAL_INGEST_CODEX_ROUTE
+        : "/ingest";
+}
+function historicalInternalRouteOptions(dataDir, providerOverride) {
+    if (providerOverride !== "codex-cli") {
+        return undefined;
+    }
+    return {
+        headers: {
+            [HISTORICAL_BACKFILL_CAPABILITY_HEADER]: readRequiredHistoricalCapability(dataDir),
+        },
+    };
+}
+function readRequiredHistoricalCapability(dataDir) {
+    const capability = readHistoricalBackfillCapability(dataDir);
+    if (!capability) {
+        throw new Error("Historical Codex override is not initialized on this machine. Start the daemon first so it can create the local backfill capability.");
+    }
+    return capability;
+}
+function resolveHistoricalRunTriageProviderName(candidates, triageById) {
+    const providers = new Set(candidates
+        .map((candidate) => triageById.get(candidate.id)?.triage?.provider)
+        .filter((provider) => typeof provider === "string"));
+    return providers.size === 1 ? [...providers][0] : undefined;
+}
+function formatCliErrorForHistoricalBackfill(err) {
+    const bodyMessage = extractCliErrorBodyMessage(err.body);
+    return bodyMessage ? `${err.message}: ${bodyMessage}` : err.message;
+}
+function extractCliErrorBodyMessage(body) {
+    if (!body || typeof body !== "object")
+        return undefined;
+    const record = body;
+    if (typeof record.message === "string" && record.message.trim()) {
+        return record.message.trim();
+    }
+    if (Array.isArray(record.details)) {
+        const detailMessages = record.details
+            .map((detail) => {
+            if (!detail || typeof detail !== "object")
+                return null;
+            const message = detail.message;
+            return typeof message === "string" && message.trim()
+                ? message.trim()
+                : null;
+        })
+            .filter((message) => message !== null);
+        if (detailMessages.length > 0) {
+            return detailMessages.join("; ");
+        }
+    }
+    return typeof record.error === "string" && record.error.trim()
+        ? record.error.trim()
+        : undefined;
+}
+function parsePositiveInteger(value) {
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isFinite(parsed) || parsed <= 0) {
+        throw new Error(`Expected a positive integer, got "${value}"`);
+    }
+    return parsed;
+}
+//# sourceMappingURL=backfill.js.map