@getrift/rift 0.0.0 → 0.1.0-beta.0

package/dist/src/ingestion/historical-campaign.js

@@ -0,0 +1,1010 @@
+import fs from "node:fs";
+import path from "node:path";
+import { atomicWrite } from "../storage/atomic.js";
+import { ARCHIVE_ONLY_THRESHOLD, PROMOTE_TO_EXTRACT_THRESHOLD, } from "../capture/triage-lane.js";
+export const HISTORICAL_IMPORT_POLICY = {
+    batch_sizes: {
+        initial: 100,
+        step_up: 250,
+        max_production: 500,
+    },
+    clean_batch: {
+        archive_only_sample_size: 100,
+        max_false_discard_rate: 0.05,
+        max_failed_files_rate: 0.02,
+        max_unsupported_or_junk_rate: 0.10,
+        max_review_queue_net_growth: 25,
+    },
+    scale_up: {
+        clean_batches_required_per_level: 2,
+    },
+    weekly_budget: {
+        max_weekly_files_processed: 1500,
+        max_daily_files_processed: 300,
+        max_weekly_triage_spend_usd: 25,
+    },
+    stop_rules: {
+        review_backlog_open_items_gt: 400,
+        review_lane_rate_gt: 0.40,
+        archive_only_lane_rate_gt: 0.85,
+        promote_to_extract_lane_rate_lt: 0.05,
+        promote_to_extract_low_streak_batches: 2,
+    },
+    lane_rules: {
+        archive_only_score_lt: ARCHIVE_ONLY_THRESHOLD,
+        review_score_gte: ARCHIVE_ONLY_THRESHOLD,
+        review_score_lte: PROMOTE_TO_EXTRACT_THRESHOLD,
+        promote_to_extract_score_gt: PROMOTE_TO_EXTRACT_THRESHOLD,
+        decision_aware_routing: true,
+        routing_source: "scoreHistoricalTriage",
+        routing_notes: [
+            'Explicit "review" decisions always stay in the review lane.',
+            'Explicit "discard" decisions only reach archive_only below the archive threshold; otherwise they stay in review.',
+            'Explicit "save" decisions only reach promote_to_extract above the promote threshold; otherwise they stay in review.',
+        ],
+    },
+    batch_quarantine: {
+        failed_files_rate_gt: 0.05,
+        duplicate_candidates_rate_gt: 0.20,
+        unsupported_format_rate_gt: 0.15,
+        non_utf8_or_corrupt_rate_gt: 0.05,
+    },
+    file_quarantine_rules: [
+        "parseability fails",
+        "format is unsupported",
+        "file is corrupt or unreadable",
+        "required source markers are missing",
+    ],
+    resume: {
+        terminal_statuses: [
+            "archive_only",
+            "review",
+            "promote_to_extract",
+            "quarantined",
+        ],
+    },
+};
+function makeSummary() {
+    return {
+        total_batches_seen: 0,
+        batches_completed: 0,
+        batches_completed_with_warnings: 0,
+        batches_failed: 0,
+        batches_quarantined: 0,
+        files_processed: 0,
+        lane_counts: {
+            archive_only: 0,
+            review: 0,
+            promote_to_extract: 0,
+            quarantined: 0,
+        },
+    };
+}
+export function historicalCampaignStatePath(dataDir) {
+    return path.join(dataDir, "imports", "campaign-state.json");
+}
+export function createHistoricalCampaignState() {
+    return {
+        version: 1,
+        policy: HISTORICAL_IMPORT_POLICY,
+        last_run_at: null,
+        weeks: {},
+        summary: makeSummary(),
+        batches: {},
+    };
+}
+export function loadHistoricalCampaignState(dataDir) {
+    const filePath = historicalCampaignStatePath(dataDir);
+    if (!fs.existsSync(filePath))
+        return createHistoricalCampaignState();
+    const parsed = JSON.parse(fs.readFileSync(filePath, "utf-8"));
+    const merged = {
+        ...createHistoricalCampaignState(),
+        ...parsed,
+        policy: HISTORICAL_IMPORT_POLICY,
+        summary: {
+            ...makeSummary(),
+            ...parsed.summary,
+            lane_counts: {
+                ...makeSummary().lane_counts,
+                ...parsed.summary?.lane_counts,
+            },
+        },
+    };
+    if (parsed.baseline_discontinuities) {
+        merged.baseline_discontinuities = parsed.baseline_discontinuities;
+    }
+    return merged;
+}
+export async function writeHistoricalCampaignState(dataDir, state) {
+    await atomicWrite(historicalCampaignStatePath(dataDir), JSON.stringify(state, null, 2) + "\n");
+}
+export function ensureHistoricalCampaignBatch(state, batchName, source, nowIso) {
+    const existing = state.batches[batchName];
+    if (existing) {
+        existing.source = source;
+        return existing;
+    }
+    const batch = {
+        batch: batchName,
+        source,
+        status: "pending",
+        metering_mode: "openai_usd",
+        first_seen_at: nowIso,
+        last_started_at: null,
+        last_run_at: null,
+        run_count: 0,
+        total_files: 0,
+        last_run_file_count: 0,
+        triage_candidates: 0,
+        counts: {
+            archive_only: 0,
+            review: 0,
+            promote_to_extract: 0,
+            duplicate: 0,
+            failed: 0,
+            skipped: 0,
+            quarantined: 0,
+            unsupported_or_junk: 0,
+        },
+        review_backlog_before: 0,
+        review_backlog_after: 0,
+        triage_spend_usd: 0,
+        archive_only_sample: {
+            required_sample_size: HISTORICAL_IMPORT_POLICY.clean_batch.archive_only_sample_size,
+            reviewed_count: 0,
+            false_discards: 0,
+            false_discard_rate: null,
+            passed: false,
+        },
+        clean_checks: {
+            archive_only_sample: {
+                required_sample_size: HISTORICAL_IMPORT_POLICY.clean_batch.archive_only_sample_size,
+                reviewed_count: 0,
+                false_discards: 0,
+                false_discard_rate: null,
+                passed: false,
+            },
+            failed_files_rate: 0,
+            unsupported_or_junk_rate: 0,
+            review_queue_net_growth: 0,
+            review_queue_net_growth_passed: true,
+            clean_batch: false,
+            reasons: [],
+        },
+        policy_flags: {
+            stop_due_weekly_spend: false,
+            stop_due_review_backlog: false,
+            review_lane_rate: 0,
+            review_lane_gt_40_percent: false,
+            archive_only_lane_rate: 0,
+            archive_only_lane_gt_85_percent: false,
+            promote_to_extract_lane_rate: 0,
+            promote_to_extract_lt_5_percent_streak: 0,
+            duplicate_candidates_rate: 0,
+            unsupported_format_rate: 0,
+            non_utf8_or_corrupt_rate: 0,
+            batch_quarantine_reasons: [],
+        },
+        next_batch_size: HISTORICAL_IMPORT_POLICY.batch_sizes.initial,
+        next_action: `Start with ${HISTORICAL_IMPORT_POLICY.batch_sizes.initial} files.`,
+        files: {},
+        items: {},
+    };
+    state.batches[batchName] = batch;
+    return batch;
+}
+export function startHistoricalCampaignBatch(batch, params) {
+    batch.status = "running";
+    batch.metering_mode = params.meteringMode;
+    batch.last_started_at = params.nowIso;
+    batch.total_files = params.totalFiles;
+    batch.last_run_file_count = params.runFiles;
+    batch.triage_candidates = params.triageCandidates;
+    batch.review_backlog_before = params.reviewBacklogBefore;
+    batch.run_count += 1;
+}
+export function listTerminalHistoricalItemIds(batch) {
+    return new Set(Object.values(batch.items)
+        .filter((item) => item.terminal)
+        .map((item) => item.item_id));
+}
+export function listTerminalHistoricalFiles(batch) {
+    return new Set(Object.values(batch.files)
+        .filter((file) => file.terminal)
+        .map((file) => file.file));
+}
+export function isFileQuarantineCandidate(file) {
+    if (!file.supported)
+        return true;
+    if (file.junk_markers.includes("unsupported_format"))
+        return true;
+    if (file.junk_markers.includes("non_utf8"))
+        return true;
+    if (file.junk_markers.includes("no_assistant_turns"))
+        return true;
+    if (file.notes.includes("unparseable"))
+        return true;
+    if (file.notes.includes("unreadable"))
+        return true;
+    if (file.notes.includes("unvalidated_too_large"))
+        return true;
+    if (file.size_bytes === 0)
+        return true;
+    if (file.size_bytes < 512)
+        return true;
+    return false;
+}
+export function markHistoricalFile(batch, fileName, params) {
+    const existing = batch.files[fileName];
+    batch.files[fileName] = {
+        file: fileName,
+        terminal: params.terminal,
+        ...(params.terminalStatus ? { terminal_status: params.terminalStatus } : {}),
+        item_ids: params.itemIds ?? existing?.item_ids ?? [],
+        last_result: params.lastResult,
+        attempts: (existing?.attempts ?? 0) + 1,
+        updated_at: params.nowIso,
+    };
+}
+export function markHistoricalItem(batch, params) {
+    batch.items[params.itemId] = {
+        item_id: params.itemId,
+        file: params.file,
+        ...(params.conversationId ? { conversation_id: params.conversationId } : {}),
+        ...(params.title ? { title: params.title } : {}),
+        last_result: params.lastResult,
+        terminal: params.terminal,
+        ...(params.terminalStatus ? { terminal_status: params.terminalStatus } : {}),
+        ...(params.triage ? { triage: params.triage } : {}),
+        updated_at: params.nowIso,
+    };
+    const fileState = batch.files[params.file];
+    const itemIds = new Set(fileState?.item_ids ?? []);
+    itemIds.add(params.itemId);
+    batch.files[params.file] = {
+        file: params.file,
+        terminal: fileState?.terminal ?? false,
+        ...(fileState?.terminal_status
+            ? { terminal_status: fileState.terminal_status }
+            : {}),
+        item_ids: [...itemIds].sort(),
+        last_result: fileState?.last_result ?? "running",
+        attempts: fileState?.attempts ?? 0,
+        updated_at: params.nowIso,
+    };
+}
+export function finalizeHistoricalFiles(batch, nowIso) {
+    for (const fileState of Object.values(batch.files)) {
+        if (fileState.terminal_status === "quarantined") {
+            fileState.terminal = true;
+            fileState.last_result = "quarantined";
+            fileState.updated_at = nowIso;
+            continue;
+        }
+        if (fileState.item_ids.length === 0) {
+            fileState.updated_at = nowIso;
+            continue;
+        }
+        const items = fileState.item_ids
+            .map((itemId) => batch.items[itemId])
+            .filter((item) => Boolean(item));
+        if (items.length === 0) {
+            fileState.updated_at = nowIso;
+            continue;
+        }
+        const unresolved = items.filter((item) => !item.terminal);
+        if (unresolved.length > 0) {
+            fileState.terminal = false;
+            const last = unresolved[unresolved.length - 1];
+            fileState.last_result = last.last_result;
+            delete fileState.terminal_status;
+            fileState.updated_at = nowIso;
+            continue;
+        }
+        fileState.terminal = true;
+        const terminalStatuses = new Set(items
+            .map((item) => item.terminal_status)
+            .filter((status) => Boolean(status)));
+        if (terminalStatuses.size === 1) {
+            const [onlyStatus] = [...terminalStatuses];
+            if (onlyStatus) {
+                fileState.terminal_status = onlyStatus;
+            }
+        }
+        else {
+            delete fileState.terminal_status;
+        }
+        const itemResults = new Set(items.map((item) => item.last_result));
+        fileState.last_result =
+            itemResults.size === 1
+                ? [...itemResults][0]
+                : "promote_to_extract";
+        fileState.updated_at = nowIso;
+    }
+}
+export function currentHistoricalWeekKey(date) {
+    const target = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
+    const dayNr = (target.getUTCDay() + 6) % 7;
+    target.setUTCDate(target.getUTCDate() - dayNr + 3);
+    const firstThursday = new Date(Date.UTC(target.getUTCFullYear(), 0, 4));
+    const diff = target.getTime() - firstThursday.getTime();
+    const week = 1 + Math.round(diff / 604_800_000);
+    return `${target.getUTCFullYear()}-W${String(week).padStart(2, "0")}`;
+}
+export function currentHistoricalDayKey(date) {
+    return date.toISOString().slice(0, 10);
+}
+export function getHistoricalWeekState(state, weekKey) {
+    state.weeks[weekKey] ??= {
+        files_processed: 0,
+        triage_spend_usd: 0,
+        days: {},
+    };
+    return state.weeks[weekKey];
+}
+export function getHistoricalDayFilesProcessed(state, weekKey, dayKey) {
+    return state.weeks[weekKey]?.days[dayKey]?.files_processed ?? 0;
+}
+export function addHistoricalRunBudgetUsage(state, params) {
+    const week = getHistoricalWeekState(state, params.weekKey);
+    week.files_processed += params.filesProcessed;
+    week.triage_spend_usd = roundUsd(week.triage_spend_usd + params.triageSpendUsd);
+    week.days[params.dayKey] ??= { files_processed: 0 };
+    week.days[params.dayKey].files_processed += params.filesProcessed;
+}
+export function getHistoricalCampaignBudgetWindow(state, now) {
+    const weekKey = currentHistoricalWeekKey(now);
+    const dayKey = currentHistoricalDayKey(now);
+    const week = getHistoricalWeekState(state, weekKey);
+    const dailyFilesProcessed = getHistoricalDayFilesProcessed(state, weekKey, dayKey);
+    return {
+        week_key: weekKey,
+        day_key: dayKey,
+        weekly_files_processed: week.files_processed,
+        daily_files_processed: dailyFilesProcessed,
+        remaining_weekly_files: Math.max(HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_files_processed -
+            week.files_processed, 0),
+        remaining_daily_files: Math.max(HISTORICAL_IMPORT_POLICY.weekly_budget.max_daily_files_processed -
+            dailyFilesProcessed, 0),
+        weekly_triage_spend_usd: roundUsd(week.triage_spend_usd),
+        remaining_weekly_triage_spend_usd: Math.max(HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_triage_spend_usd -
+            roundUsd(week.triage_spend_usd), 0),
+    };
+}
+export function countUnsupportedOrJunkFiles(preflight) {
+    return preflight.files.filter((file) => !file.supported || file.junk_markers.length > 0).length;
+}
+export function countUnsupportedFormatFiles(preflight) {
+    return preflight.files.filter((file) => file.junk_markers.includes("unsupported_format")).length;
+}
+export function countNonUtf8OrCorruptFiles(preflight) {
+    return preflight.files.filter((file) => file.junk_markers.includes("non_utf8") ||
+        file.notes.includes("unparseable") ||
+        file.notes.includes("unreadable")).length;
+}
+export function collectHistoricalBatchQuarantineReasons(params) {
+    const total = Math.max(params.preflight.total_files, 1);
+    const duplicateRate = params.preflight.duplicate_candidates / total;
+    const unsupportedRate = countUnsupportedFormatFiles(params.preflight) / total;
+    const corruptRate = countNonUtf8OrCorruptFiles(params.preflight) / total;
+    const reasons = [];
+    if (params.failedFilesRate > HISTORICAL_IMPORT_POLICY.batch_quarantine.failed_files_rate_gt) {
+        reasons.push(`failed files ${formatPercent(params.failedFilesRate)} > ${formatPercent(HISTORICAL_IMPORT_POLICY.batch_quarantine.failed_files_rate_gt)}`);
+    }
+    if (duplicateRate > HISTORICAL_IMPORT_POLICY.batch_quarantine.duplicate_candidates_rate_gt) {
+        reasons.push(`duplicate candidates ${formatPercent(duplicateRate)} > ${formatPercent(HISTORICAL_IMPORT_POLICY.batch_quarantine.duplicate_candidates_rate_gt)}`);
+    }
+    if (unsupportedRate > HISTORICAL_IMPORT_POLICY.batch_quarantine.unsupported_format_rate_gt) {
+        reasons.push(`unsupported format ${formatPercent(unsupportedRate)} > ${formatPercent(HISTORICAL_IMPORT_POLICY.batch_quarantine.unsupported_format_rate_gt)}`);
+    }
+    if (corruptRate > HISTORICAL_IMPORT_POLICY.batch_quarantine.non_utf8_or_corrupt_rate_gt) {
+        reasons.push(`non-UTF-8 or corrupt ${formatPercent(corruptRate)} > ${formatPercent(HISTORICAL_IMPORT_POLICY.batch_quarantine.non_utf8_or_corrupt_rate_gt)}`);
+    }
+    return reasons;
+}
+export function summarizeHistoricalBatchState(batch, preflight) {
+    const counts = {
+        archive_only: 0,
+        review: 0,
+        promote_to_extract: 0,
+        duplicate: 0,
+        failed: 0,
+        skipped: 0,
+        quarantined: 0,
+        unsupported_or_junk: countUnsupportedOrJunkFiles(preflight),
+    };
+    for (const fileState of Object.values(batch.files)) {
+        if (fileState.terminal_status === "quarantined")
+            counts.quarantined += 1;
+        else if (fileState.last_result === "failed" || fileState.last_result === "error")
+            counts.failed += 1;
+        else if (fileState.last_result === "skipped")
+            counts.skipped += 1;
+    }
+    for (const item of Object.values(batch.items)) {
+        if (item.last_result === "archive_only")
+            counts.archive_only += 1;
+        else if (item.last_result === "review")
+            counts.review += 1;
+        else if (item.last_result === "promote_to_extract" || item.last_result === "duplicate") {
+            counts.promote_to_extract += 1;
+        }
+        if (item.last_result === "duplicate")
+            counts.duplicate += 1;
+    }
+    const unresolvedFiles = Object.values(batch.files)
+        .filter((file) => !file.terminal)
+        .map((file) => file.file)
+        .sort();
+    const terminalFiles = Object.values(batch.files)
+        .filter((file) => file.terminal)
+        .map((file) => file.file)
+        .sort();
+    return { counts, unresolvedFiles, terminalFiles };
+}
+function summarizeHistoricalRunReport(params) {
+    const counts = {
+        archive_only: 0,
+        review: 0,
+        promote_to_extract: 0,
+        duplicate: 0,
+        failed: 0,
+        skipped: 0,
+        quarantined: 0,
+        unsupported_or_junk: 0,
+    };
+    const processedFiles = [...new Set(params.processedFiles)].sort();
+    const processedFileSet = new Set(processedFiles);
+    for (const result of params.results) {
+        switch (result.status) {
+            case "archive_only":
+                counts.archive_only += 1;
+                break;
+            case "review":
+                counts.review += 1;
+                break;
+            case "completed":
+                counts.promote_to_extract += 1;
+                break;
+            case "duplicate":
+                counts.promote_to_extract += 1;
+                counts.duplicate += 1;
+                break;
+            case "failed":
+            case "error":
+                counts.failed += 1;
+                break;
+            case "skipped":
+                counts.skipped += 1;
+                break;
+            case "quarantined":
+                counts.quarantined += 1;
+                break;
+            default:
+                break;
+        }
+    }
+    for (const file of params.preflight.files) {
+        if (!processedFileSet.has(file.name))
+            continue;
+        if (!file.supported || file.junk_markers.length > 0) {
+            counts.unsupported_or_junk += 1;
+        }
+    }
+    return { counts, processedFiles };
+}
+export function snapshotHistoricalCampaignReport(params) {
+    const budget = getHistoricalCampaignBudgetWindow(params.state, params.now);
+    const { counts: cumulativeCounts, unresolvedFiles, terminalFiles, } = summarizeHistoricalBatchState(params.batch, params.preflight);
+    const { counts: runCounts, processedFiles } = summarizeHistoricalRunReport({
+        preflight: params.preflight,
+        processedFiles: params.processedFiles ?? [],
+        results: params.results ?? [],
+    });
+    return {
+        version: 1,
+        batch: params.batch.batch,
+        source: params.batch.source,
+        status: params.status ?? params.batch.status,
+        metering_mode: params.batch.metering_mode,
+        generated_at: params.now.toISOString(),
+        last_started_at: params.batch.last_started_at,
+        last_run_at: params.batch.last_run_at,
+        run_count: params.batch.run_count,
+        total_files: params.batch.total_files,
+        run_files: params.runFileCount,
+        triage_candidates: params.batch.triage_candidates,
+        counts_scope: "last_run",
+        counts: runCounts,
+        cumulative_counts: cumulativeCounts,
+        duplicate_candidates: params.preflight.duplicate_candidates,
+        processed_files_scope: "last_run",
+        review_backlog_before: params.batch.review_backlog_before,
+        review_backlog_after: params.reviewBacklogAfter,
+        review_backlog_net_growth: params.reviewBacklogAfter - params.batch.review_backlog_before,
+        triage_spend_usd: roundUsd(params.triageSpendUsd),
+        triage_spend_note: describeHistoricalTriageSpend(params.batch.metering_mode),
+        weekly_files_processed: budget.weekly_files_processed,
+        daily_files_processed: budget.daily_files_processed,
+        weekly_triage_spend_usd: budget.weekly_triage_spend_usd,
+        clean_checks: params.batch.clean_checks,
+        policy_flags: params.batch.policy_flags,
+        next_batch_size: params.nextBatchSize ?? params.batch.next_batch_size,
+        next_action: params.nextAction ?? params.batch.next_action,
+        processed_files: processedFiles,
+        terminal_files: terminalFiles,
+        unresolved_files: unresolvedFiles,
+        results: params.results ?? [],
+    };
+}
+function recoverHistoricalProcessedFiles(batch) {
+    return Object.values(batch.files)
+        .filter((file) => file.terminal || file.attempts > 0 || file.item_ids.length > 0)
+        .map((file) => file.file)
+        .sort((left, right) => left.localeCompare(right));
+}
+function recoverHistoricalResults(batch) {
+    const results = [];
+    const files = Object.values(batch.files).sort((left, right) => left.file.localeCompare(right.file));
+    for (const fileState of files) {
+        if (fileState.terminal_status === "quarantined") {
+            results.push({
+                file: fileState.file,
+                status: "quarantined",
+            });
+            continue;
+        }
+        const items = fileState.item_ids
+            .map((itemId) => batch.items[itemId])
+            .filter((item) => Boolean(item))
+            .sort((left, right) => left.item_id.localeCompare(right.item_id));
+        if (items.length === 0) {
+            if (fileState.last_result === "failed" ||
+                fileState.last_result === "error" ||
+                fileState.last_result === "skipped") {
+                results.push({
+                    file: fileState.file,
+                    status: fileState.last_result,
+                });
+            }
+            continue;
+        }
+        for (const item of items) {
+            const base = {
+                file: item.file,
+                ...(item.conversation_id ? { conversation_id: item.conversation_id } : {}),
+                ...(item.title ? { title: item.title } : {}),
+            };
+            switch (item.last_result) {
+                case "archive_only":
+                    results.push({ ...base, status: "archive_only", lane: "archive_only" });
+                    break;
+                case "review":
+                    results.push({ ...base, status: "review", lane: "review" });
+                    break;
+                case "promote_to_extract":
+                    results.push({
+                        ...base,
+                        status: "completed",
+                        lane: "promote_to_extract",
+                    });
+                    break;
+                case "duplicate":
+                    results.push({
+                        ...base,
+                        status: "duplicate",
+                        lane: "promote_to_extract",
+                    });
+                    break;
+                case "failed":
+                case "error":
+                case "quarantined":
+                    results.push({
+                        ...base,
+                        status: item.last_result,
+                        ...(item.triage?.lane ? { lane: item.triage.lane } : {}),
+                    });
+                    break;
+                default:
+                    break;
+            }
+        }
+    }
+    return results;
+}
+export function snapshotHistoricalNoOpReport(params) {
+    const budget = getHistoricalCampaignBudgetWindow(params.state, params.now);
+    const { counts: cumulativeCounts, unresolvedFiles, terminalFiles, } = summarizeHistoricalBatchState(params.batch, params.preflight);
+    const results = recoverHistoricalResults(params.batch);
+    // No-op rerun of an already-terminal batch: the visible report IS the
+    // batch's cumulative state. Emit cumulative values in `counts` /
+    // `processed_files` and flip the scope fields so readers know this is
+    // not a last-run slice. Run-level numbers (run_files, triage_spend_usd)
+    // stay zero because no new work happened.
+    return {
+        version: 1,
+        batch: params.batch.batch,
+        source: params.batch.source,
+        status: params.batch.status,
+        metering_mode: params.batch.metering_mode,
+        generated_at: params.now.toISOString(),
+        last_started_at: params.batch.last_started_at,
+        last_run_at: params.batch.last_run_at,
+        run_count: params.batch.run_count,
+        total_files: params.batch.total_files,
+        run_files: 0,
+        triage_candidates: 0,
+        counts_scope: "cumulative",
+        counts: cumulativeCounts,
+        cumulative_counts: cumulativeCounts,
+        duplicate_candidates: params.preflight.duplicate_candidates,
+        processed_files_scope: "cumulative",
+        review_backlog_before: params.reviewBacklogNow,
+        review_backlog_after: params.reviewBacklogNow,
+        review_backlog_net_growth: 0,
+        triage_spend_usd: 0,
+        triage_spend_note: describeHistoricalTriageSpend(params.batch.metering_mode),
+        weekly_files_processed: budget.weekly_files_processed,
+        daily_files_processed: budget.daily_files_processed,
+        weekly_triage_spend_usd: budget.weekly_triage_spend_usd,
+        clean_checks: params.batch.clean_checks,
+        policy_flags: params.batch.policy_flags,
+        next_batch_size: params.batch.next_batch_size,
+        next_action: params.nextAction,
+        processed_files: terminalFiles,
+        terminal_files: terminalFiles,
+        unresolved_files: unresolvedFiles,
+        results,
+    };
+}
+export function evaluateHistoricalBatch(params) {
+    const budget = getHistoricalCampaignBudgetWindow(params.state, params.now);
+    const { counts, unresolvedFiles, terminalFiles } = summarizeHistoricalBatchState(params.batch, params.preflight);
+    params.batch.counts = counts;
+    params.batch.review_backlog_after = params.reviewBacklogAfter;
+    params.batch.triage_spend_usd = roundUsd(params.batch.triage_spend_usd + params.triageSpendUsd);
+    params.batch.report_path = params.reportPath;
+    const totalFiles = Math.max(params.batch.total_files, 1);
+    const failedFilesRate = counts.failed / totalFiles;
+    const unsupportedOrJunkRate = counts.unsupported_or_junk / totalFiles;
+    const reviewQueueNetGrowth = params.reviewBacklogAfter - params.batch.review_backlog_before;
+    const archiveOnlySample = {
+        ...params.batch.archive_only_sample,
+        false_discard_rate: params.batch.archive_only_sample.reviewed_count > 0
+            ? params.batch.archive_only_sample.false_discards /
+                params.batch.archive_only_sample.reviewed_count
+            : null,
+    };
+    archiveOnlySample.passed =
+        archiveOnlySample.reviewed_count <= counts.archive_only &&
+            archiveOnlySample.reviewed_count >= archiveOnlySample.required_sample_size &&
+            archiveOnlySample.false_discard_rate !== null &&
+            archiveOnlySample.false_discard_rate <=
+                HISTORICAL_IMPORT_POLICY.clean_batch.max_false_discard_rate;
+    params.batch.archive_only_sample = archiveOnlySample;
+    const cleanReasons = [];
+    const archiveOnlySampleInvalid = archiveOnlySample.reviewed_count > counts.archive_only;
+    if (archiveOnlySampleInvalid) {
+        cleanReasons.push(`archive_only sample invalid (${archiveOnlySample.reviewed_count} reviewed > ${counts.archive_only} available archive_only items)`);
+    }
+    else if (!archiveOnlySample.passed) {
+        cleanReasons.push(archiveOnlySample.reviewed_count >= archiveOnlySample.required_sample_size
+            ? `archive_only false-discard rate ${formatPercent(archiveOnlySample.false_discard_rate ?? 0)} > ${formatPercent(HISTORICAL_IMPORT_POLICY.clean_batch.max_false_discard_rate)}`
+            : `archive_only sample pending (${archiveOnlySample.reviewed_count}/${archiveOnlySample.required_sample_size} reviewed)`);
+    }
+    if (failedFilesRate > HISTORICAL_IMPORT_POLICY.clean_batch.max_failed_files_rate) {
+        cleanReasons.push(`failed files ${formatPercent(failedFilesRate)} > ${formatPercent(HISTORICAL_IMPORT_POLICY.clean_batch.max_failed_files_rate)}`);
+    }
+    if (unsupportedOrJunkRate > HISTORICAL_IMPORT_POLICY.clean_batch.max_unsupported_or_junk_rate) {
+        cleanReasons.push(`unsupported or junk files ${formatPercent(unsupportedOrJunkRate)} > ${formatPercent(HISTORICAL_IMPORT_POLICY.clean_batch.max_unsupported_or_junk_rate)}`);
+    }
+    if (reviewQueueNetGrowth > HISTORICAL_IMPORT_POLICY.clean_batch.max_review_queue_net_growth) {
+        cleanReasons.push(`review queue net growth ${reviewQueueNetGrowth} > ${HISTORICAL_IMPORT_POLICY.clean_batch.max_review_queue_net_growth}`);
+    }
+    const laneDenominator = Math.max(counts.archive_only + counts.review + counts.promote_to_extract, 1);
+    const reviewLaneRate = counts.review / laneDenominator;
+    const archiveOnlyLaneRate = counts.archive_only / laneDenominator;
+    const promoteLaneRate = counts.promote_to_extract / laneDenominator;
+    const promoteLowStreak = countLowPromoteStreak(params.state, params.batch.batch, promoteLaneRate);
+    const batchQuarantineReasons = collectHistoricalBatchQuarantineReasons({
+        preflight: params.preflight,
+        failedFilesRate,
+    });
+    const weeklyFilesProcessed = budget.weekly_files_processed;
+    const dailyFilesProcessed = budget.daily_files_processed;
+    const weeklyTriageSpendUsd = budget.weekly_triage_spend_usd;
+    const stopDueWeeklySpend = params.batch.metering_mode === "openai_usd" &&
+        weeklyTriageSpendUsd >
+            HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_triage_spend_usd;
+    const stopDueReviewBacklog = params.reviewBacklogAfter > HISTORICAL_IMPORT_POLICY.stop_rules.review_backlog_open_items_gt;
+    params.batch.clean_checks = {
+        archive_only_sample: archiveOnlySample,
+        failed_files_rate: failedFilesRate,
+        unsupported_or_junk_rate: unsupportedOrJunkRate,
+        review_queue_net_growth: reviewQueueNetGrowth,
+        review_queue_net_growth_passed: reviewQueueNetGrowth <=
+            HISTORICAL_IMPORT_POLICY.clean_batch.max_review_queue_net_growth,
+        clean_batch: cleanReasons.length === 0,
+        reasons: cleanReasons,
+    };
+    params.batch.policy_flags = {
+        stop_due_weekly_spend: stopDueWeeklySpend,
+        stop_due_review_backlog: stopDueReviewBacklog,
+        review_lane_rate: reviewLaneRate,
+        review_lane_gt_40_percent: reviewLaneRate > HISTORICAL_IMPORT_POLICY.stop_rules.review_lane_rate_gt,
+        archive_only_lane_rate: archiveOnlyLaneRate,
+        archive_only_lane_gt_85_percent: archiveOnlyLaneRate > HISTORICAL_IMPORT_POLICY.stop_rules.archive_only_lane_rate_gt,
+        promote_to_extract_lane_rate: promoteLaneRate,
+        promote_to_extract_lt_5_percent_streak: promoteLowStreak,
+        duplicate_candidates_rate: params.preflight.duplicate_candidates / totalFiles,
+        unsupported_format_rate: countUnsupportedFormatFiles(params.preflight) / totalFiles,
+        non_utf8_or_corrupt_rate: countNonUtf8OrCorruptFiles(params.preflight) / totalFiles,
+        batch_quarantine_reasons: batchQuarantineReasons,
+    };
+    if (batchQuarantineReasons.length > 0) {
+        params.batch.status = "quarantined";
+    }
+    else if (unresolvedFiles.length > 0) {
+        params.batch.status =
+            params.incompleteStatus ??
+                (counts.failed > 0 ? "failed" : "running");
+    }
+    else if (!params.batch.clean_checks.clean_batch ||
+        stopDueWeeklySpend ||
+        stopDueReviewBacklog ||
+        params.batch.policy_flags.review_lane_gt_40_percent ||
+        params.batch.policy_flags.archive_only_lane_gt_85_percent ||
+        params.batch.policy_flags.promote_to_extract_lt_5_percent_streak >=
+            HISTORICAL_IMPORT_POLICY.stop_rules.promote_to_extract_low_streak_batches) {
+        params.batch.status = "completed_with_warnings";
+    }
+    else {
+        params.batch.status = "completed";
+    }
+    params.batch.next_batch_size =
+        params.batch.status === "running"
+            ? normalizeBatchLevel(params.batch.total_files)
+            : recommendNextBatchSize(params.state, params.batch);
+    params.batch.next_action = recommendNextAction(params.batch, params.preflight);
+    params.batch.last_run_at = params.now.toISOString();
+    recomputeHistoricalCampaignSummary(params.state);
+    return {
+        report: snapshotHistoricalCampaignReport({
+            state: params.state,
+            batch: params.batch,
+            preflight: params.preflight,
+            now: params.now,
+            runFileCount: params.runFileCount,
+            reviewBacklogAfter: params.reviewBacklogAfter,
+            triageSpendUsd: params.triageSpendUsd,
+            ...(params.processedFiles
+                ? { processedFiles: params.processedFiles }
+                : {}),
+            ...(params.results ? { results: params.results } : {}),
+        }),
+        weeklyFilesProcessed,
+        dailyFilesProcessed,
+        weeklyTriageSpendUsd,
+    };
+}
+export function recomputeHistoricalCampaignSummary(state) {
+    const summary = makeSummary();
+    const batches = Object.values(state.batches);
+    summary.total_batches_seen = batches.length;
+    summary.files_processed = Object.values(state.weeks).reduce((total, week) => total + week.files_processed, 0);
+    for (const batch of batches) {
+        if (batch.status === "completed")
+            summary.batches_completed += 1;
+        if (batch.status === "completed_with_warnings")
+            summary.batches_completed_with_warnings += 1;
+        if (batch.status === "failed")
+            summary.batches_failed += 1;
+        if (batch.status === "quarantined")
+            summary.batches_quarantined += 1;
+        summary.lane_counts.archive_only += batch.counts.archive_only;
+        summary.lane_counts.review += batch.counts.review;
+        summary.lane_counts.promote_to_extract += batch.counts.promote_to_extract;
+        summary.lane_counts.quarantined += batch.counts.quarantined;
+    }
+    state.summary = summary;
+}
+export async function writeHistoricalCampaignReport(reportPath, report) {
+    await atomicWrite(reportPath, JSON.stringify(report, null, 2) + "\n");
+}
+export function historicalCampaignReportPath(batchDir) {
+    return path.join(batchDir, "campaign-report.json");
+}
+export function formatHistoricalCampaignSummary(report) {
+    const triageSpendLine = report.metering_mode === "unmetered_codex_cli"
+        ? "unmetered (Codex CLI override)"
+        : `$${report.triage_spend_usd.toFixed(4)}`;
+    const weeklySpendLine = report.metering_mode === "unmetered_codex_cli"
+        ? `OpenAI-only $${report.weekly_triage_spend_usd.toFixed(4)}/$${HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_triage_spend_usd.toFixed(2)}`
+        : `$${report.weekly_triage_spend_usd.toFixed(4)}/$${HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_triage_spend_usd.toFixed(2)}`;
+    return (`\nHistorical import campaign: ${report.batch}\n` +
+        `  Status:                 ${report.status}\n` +
+        `  Metering mode:          ${report.metering_mode}\n` +
+        `  Files in batch:         ${report.total_files}\n` +
+        `  Files touched this run: ${report.run_files}\n` +
+        `  Triage candidates:      ${report.triage_candidates}\n` +
+        `  archive_only (run):     ${report.counts.archive_only}\n` +
+        `  review (run):           ${report.counts.review}\n` +
+        `  promote_to_extract(run): ${report.counts.promote_to_extract}\n` +
+        `  quarantined (run):      ${report.counts.quarantined}\n` +
+        `  failed (run):           ${report.counts.failed}\n` +
+        `  duplicates (run):       ${report.counts.duplicate}\n` +
+        `  archive_only (batch):   ${report.cumulative_counts.archive_only}\n` +
+        `  review (batch):         ${report.cumulative_counts.review}\n` +
+        `  promote_to_extract(batch): ${report.cumulative_counts.promote_to_extract}\n` +
+        `  quarantined (batch):    ${report.cumulative_counts.quarantined}\n` +
+        `  failed (batch):         ${report.cumulative_counts.failed}\n` +
+        `  duplicates (batch):     ${report.cumulative_counts.duplicate}\n` +
+        `  review backlog now:     ${report.review_backlog_after} (net ${formatSigned(report.review_backlog_net_growth)})\n` +
+        `  triage spend this run:  ${triageSpendLine}\n` +
+        `  weekly files:           ${report.weekly_files_processed}/${HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_files_processed}\n` +
+        `  daily files:            ${report.daily_files_processed}/${HISTORICAL_IMPORT_POLICY.weekly_budget.max_daily_files_processed}\n` +
+        `  weekly triage spend:    ${weeklySpendLine}\n` +
+        `  spend note:             ${report.triage_spend_note}\n` +
+        `  Clean batch:            ${report.clean_checks.clean_batch ? "yes" : "no"}\n` +
+        `  Next batch size:        ${report.next_batch_size}\n` +
+        `  Next action:            ${report.next_action}\n`);
+}
+export function reportHistoricalCampaignBlockers(params) {
+    const budget = getHistoricalCampaignBudgetWindow(params.state, params.now);
+    const blockers = [];
+    if (params.meteringMode === "openai_usd") {
+        if (budget.weekly_triage_spend_usd >
+            HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_triage_spend_usd) {
+            blockers.push(`weekly triage spend already exceeds $${HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_triage_spend_usd.toFixed(2)}`);
+        }
+    }
+    if (params.reviewBacklogBefore > HISTORICAL_IMPORT_POLICY.stop_rules.review_backlog_open_items_gt) {
+        blockers.push(`review backlog ${params.reviewBacklogBefore} exceeds ${HISTORICAL_IMPORT_POLICY.stop_rules.review_backlog_open_items_gt}`);
+    }
+    if (budget.remaining_daily_files <= 0) {
+        blockers.push(`daily file budget ${budget.daily_files_processed}/${HISTORICAL_IMPORT_POLICY.weekly_budget.max_daily_files_processed} is already exhausted`);
+    }
+    if (budget.remaining_weekly_files <= 0) {
+        blockers.push(`weekly file budget ${budget.weekly_files_processed}/${HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_files_processed} is already exhausted`);
+    }
+    if (params.meteringMode === "openai_usd" &&
+        (params.triageCandidates ?? 0) > 0 &&
+        typeof params.triageBatchCostCapUsd === "number" &&
+        params.triageBatchCostCapUsd > budget.remaining_weekly_triage_spend_usd) {
+        blockers.push(`remaining weekly triage spend $${budget.remaining_weekly_triage_spend_usd.toFixed(4)} is below the configured per-run triage cap $${params.triageBatchCostCapUsd.toFixed(4)}`);
+    }
+    return blockers;
+}
+function recommendNextBatchSize(state, batch) {
+    const currentLevel = normalizeBatchLevel(batch.total_files);
+    if (!batch.clean_checks.clean_batch) {
+        if (currentLevel === HISTORICAL_IMPORT_POLICY.batch_sizes.max_production) {
+            return HISTORICAL_IMPORT_POLICY.batch_sizes.step_up;
+        }
+        return HISTORICAL_IMPORT_POLICY.batch_sizes.initial;
+    }
+    if (isScaleUpBlocked(batch)) {
+        return currentLevel;
+    }
+    if (currentLevel === HISTORICAL_IMPORT_POLICY.batch_sizes.max_production) {
+        return HISTORICAL_IMPORT_POLICY.batch_sizes.max_production;
+    }
+    const streak = countCleanLevelStreak(state, batch.batch, currentLevel);
+    if (currentLevel === HISTORICAL_IMPORT_POLICY.batch_sizes.initial &&
+        streak >= HISTORICAL_IMPORT_POLICY.scale_up.clean_batches_required_per_level) {
+        return HISTORICAL_IMPORT_POLICY.batch_sizes.step_up;
+    }
+    if (currentLevel === HISTORICAL_IMPORT_POLICY.batch_sizes.step_up &&
+        streak >= HISTORICAL_IMPORT_POLICY.scale_up.clean_batches_required_per_level) {
+        return HISTORICAL_IMPORT_POLICY.batch_sizes.max_production;
+    }
+    return currentLevel;
+}
+function recommendNextAction(batch, preflight) {
+    if (batch.policy_flags.batch_quarantine_reasons.length > 0) {
+        return `Quarantine this batch for manual inspection: ${batch.policy_flags.batch_quarantine_reasons.join("; ")}.`;
+    }
+    if (batch.policy_flags.stop_due_weekly_spend) {
+        return `Stop historical import this week. Weekly triage spend exceeded $${HISTORICAL_IMPORT_POLICY.weekly_budget.max_weekly_triage_spend_usd.toFixed(2)}.`;
+    }
+    if (batch.policy_flags.stop_due_review_backlog) {
+        return `Stop historical import. Review backlog exceeds ${HISTORICAL_IMPORT_POLICY.stop_rules.review_backlog_open_items_gt} open items.`;
+    }
+    const unresolved = Object.values(batch.files).filter((file) => !file.terminal).length;
+    if (unresolved > 0) {
+        return `Resume only unresolved files. ${unresolved} file(s) still need terminal outcomes.`;
+    }
+    if (!batch.clean_checks.archive_only_sample.passed) {
+        if (batch.archive_only_sample.reviewed_count > batch.counts.archive_only) {
+            return `Fix archive_only audit state in campaign-state.json. ${batch.archive_only_sample.reviewed_count} reviewed exceeds ${batch.counts.archive_only} available archive_only items for this batch.`;
+        }
+        if (batch.counts.archive_only < HISTORICAL_IMPORT_POLICY.clean_batch.archive_only_sample_size) {
+            return `Only ${batch.counts.archive_only} archive_only item(s) exist in this batch. Review all ${batch.counts.archive_only} now and do not scale up until the 100-item audit gate is satisfied by campaign policy.`;
+        }
+        return `Audit ${HISTORICAL_IMPORT_POLICY.clean_batch.archive_only_sample_size} archive_only items and update campaign-state.json before scaling up.`;
+    }
+    if (batch.policy_flags.review_lane_gt_40_percent) {
+        return "Hold current batch size. Review lane exceeded 40% of the batch.";
+    }
+    if (batch.policy_flags.archive_only_lane_gt_85_percent) {
+        return "Inspect source quality before continuing. archive_only exceeded 85% of the batch.";
+    }
+    if (batch.policy_flags.promote_to_extract_lt_5_percent_streak >=
+        HISTORICAL_IMPORT_POLICY.stop_rules.promote_to_extract_low_streak_batches) {
+        return "Tighten staging or source selection before continuing. promote_to_extract stayed below 5% for two consecutive batches.";
+    }
+    if (batch.next_batch_size > normalizeBatchLevel(batch.total_files)) {
+        return `Scale the next staged batch to ${batch.next_batch_size} files.`;
+    }
+    if (preflight.total_files > HISTORICAL_IMPORT_POLICY.batch_sizes.max_production) {
+        return `Future staged batches must be split to ${HISTORICAL_IMPORT_POLICY.batch_sizes.max_production} files or fewer.`;
+    }
+    return `Continue at ${batch.next_batch_size} files until two consecutive clean batches are recorded at that level.`;
+}
+function describeHistoricalTriageSpend(meteringMode) {
+    return meteringMode === "unmetered_codex_cli"
+        ? "Codex CLI historical backfill override is unmetered by OpenAI USD."
+        : "OpenAI USD metering.";
+}
+function countCleanLevelStreak(state, currentBatchName, level) {
+    const batches = Object.values(state.batches)
+        .filter((batch) => batch.last_run_at)
+        .sort((a, b) => (b.last_run_at ?? "").localeCompare(a.last_run_at ?? ""));
+    let streak = 0;
+    for (const batch of batches) {
+        if (normalizeBatchLevel(batch.total_files) !== level) {
+            if (batch.batch === currentBatchName)
+                continue;
+            break;
+        }
+        if (!batch.clean_checks.clean_batch || isScaleUpBlocked(batch))
+            break;
+        streak += 1;
+    }
+    return streak;
+}
+function isScaleUpBlocked(batch) {
+    return (batch.policy_flags.stop_due_weekly_spend ||
+        batch.policy_flags.stop_due_review_backlog ||
+        batch.policy_flags.review_lane_gt_40_percent ||
+        batch.policy_flags.archive_only_lane_gt_85_percent ||
+        batch.policy_flags.promote_to_extract_lt_5_percent_streak >=
+            HISTORICAL_IMPORT_POLICY.stop_rules.promote_to_extract_low_streak_batches);
+}
+function countLowPromoteStreak(state, currentBatchName, currentPromoteRate) {
+    const batches = Object.values(state.batches)
+        .filter((batch) => batch.last_run_at)
+        .sort((a, b) => (b.last_run_at ?? "").localeCompare(a.last_run_at ?? ""));
+    let streak = 0;
+    for (const batch of batches) {
+        const rate = batch.batch === currentBatchName
+            ? currentPromoteRate
+            : batch.policy_flags.promote_to_extract_lane_rate;
+        if (rate < HISTORICAL_IMPORT_POLICY.stop_rules.promote_to_extract_lane_rate_lt) {
+            streak += 1;
+            continue;
+        }
+        break;
+    }
+    return streak;
+}
+function normalizeBatchLevel(totalFiles) {
+    if (totalFiles <= HISTORICAL_IMPORT_POLICY.batch_sizes.initial) {
+        return HISTORICAL_IMPORT_POLICY.batch_sizes.initial;
+    }
+    if (totalFiles <= HISTORICAL_IMPORT_POLICY.batch_sizes.step_up) {
+        return HISTORICAL_IMPORT_POLICY.batch_sizes.step_up;
+    }
+    return HISTORICAL_IMPORT_POLICY.batch_sizes.max_production;
+}
+function formatPercent(value) {
+    return `${(value * 100).toFixed(1)}%`;
+}
+function formatSigned(value) {
+    return value >= 0 ? `+${value}` : String(value);
+}
+function roundUsd(value) {
+    return Number(value.toFixed(4));
+}
+//# sourceMappingURL=historical-campaign.js.map