@bli-cockpit/cli 0.2.49 → 0.2.51

package/dist/commands/backfill-output.js

@@ -0,0 +1,75 @@
+/**
+ * What a person sees and is asked. Every line a human reads from
+ * `cockpit backfill` is written here: the dry-run review, the `--all`
+ * confirmation prompt, the final PASS/BLOCKED lines, and the skip/reason table
+ * that goes under all of them. A blocked or partial run always names its
+ * reason and prints the retry command; none of these paths may go silent.
+ */
+import { uploadableCandidates } from "./backfill-batches.js";
+export function writeDryRunSummary(io, options) {
+    const uploadable = uploadableCandidates(options.candidates);
+    writeLine(io.stdout, `${options.dryRunOnly ? "DRY-RUN" : "Review"}: ${options.candidates.length} session(s), ${uploadable.length} uploadable, ${options.candidates.length - uploadable.length} skipped.`);
+    writeReasonTable(io, options.reasonCounts);
+    if (options.dryRunOnly) {
+        writeLine(io.stdout, "DRY-RUN: wrote nothing (no cursor, marker, report, or upload).");
+    }
+    writeLine(io.stdout, `Verify after upload: ${options.dashboardUrl}/my-work`);
+}
+export function writeHumanBackfillResult(result, io) {
+    if (result.status === "complete") {
+        writeLine(io.stdout, `PASS: ${result.counts.backfilled} backfilled, ${result.counts.skipped} skipped (table). Verify: ${result.verify_url}`);
+        writeReasonTable(io, result.counts.reasons);
+        return;
+    }
+    if (result.status === "partial") {
+        const at = result.blocked_at;
+        if (at) {
+            writeLine(io.stderr, `BLOCKED: ${at.what} at batch ${at.batch_index}/${at.batch_total}, ${at.done}/${at.total} done`);
+        }
+        writeLine(io.stderr, `Failure: ${result.failure_reason ?? "partial_backfill"}`);
+        writeLine(io.stderr, `Retry: ${result.retry_command}`);
+        writeLine(io.stderr, `Stopped: ${result.counts.remaining} remaining — rerun cockpit backfill to continue`);
+        writeReasonTable(io, result.counts.reasons);
+        return;
+    }
+    const at = result.blocked_at;
+    if (at) {
+        writeLine(io.stderr, `BLOCKED: ${at.what} at batch ${at.batch_index}/${at.batch_total}, ${at.done}/${at.total} done`);
+    }
+    else {
+        writeLine(io.stderr, "BLOCKED: backfill could not run.");
+    }
+    writeLine(io.stderr, `Failure: ${result.failure_reason ?? "no_sessions"}`);
+    writeLine(io.stderr, `Retry: ${result.retry_command}`);
+}
+function writeReasonTable(io, reasons) {
+    if (reasons.length === 0)
+        return;
+    writeLine(io.stdout, "Skip/reason table:");
+    for (const reason of reasons) {
+        writeLine(io.stdout, `- ${reason.reason}: ${reason.count} (${reason.classification}; ${reason.note})`);
+    }
+}
+export async function confirmAllBackfill(io) {
+    const answer = await readLine(io, "Proceed with --all backfill upload? [y/N] ");
+    return answer.trim().toLowerCase() === "y" || answer.trim().toLowerCase() === "yes";
+}
+async function readLine(io, prompt) {
+    io.stdout.write(prompt);
+    io.stdin.setEncoding("utf8");
+    return new Promise((resolve) => {
+        const onData = (chunk) => {
+            io.stdin.removeListener("data", onData);
+            io.stdin.pause();
+            resolve(chunk);
+        };
+        io.stdin.resume();
+        io.stdin.on("data", onData);
+    });
+}
+export function isInteractiveStdin(io) {
+    return Boolean(io.stdin.isTTY);
+}
+export function writeLine(stream, text) {
+    stream.write(`${text}\n`);
+}

package/dist/commands/backfill-plan.js

@@ -0,0 +1,71 @@
+/**
+ * PLAN: given the scan, should this run actually upload anything right now?
+ * `--all` without `--yes` needs interactive confirmation; `--dry-run` reports
+ * what would happen and stops. Both terminal answers are results in their own
+ * right, so the operator still gets the full reason table either way.
+ */
+import { blockingScanIssues } from "./backfill-issues.js";
+import { confirmAllBackfill, writeDryRunSummary } from "./backfill-output.js";
+import { backfillResultBaseArgs, baseBackfillResult, } from "./backfill-result.js";
+/**
+ * PLAN: given the scan, should this run actually upload anything right now?
+ * `--all` without `--yes` needs interactive confirmation; `--dry-run` reports
+ * what would happen and stops there. Either returns a terminal result;
+ * anything else proceeds to UPLOAD.
+ */
+export async function planBackfillRun(command, io, ctx) {
+    const { dashboardUrl, scan, reasonCounts } = ctx;
+    if (command.all && !command.yes) {
+        if (!command.json) {
+            writeDryRunSummary(io, {
+                candidates: scan.candidates,
+                reasonCounts,
+                dashboardUrl,
+                dryRunOnly: false,
+            });
+        }
+        const confirmed = await confirmAllBackfill(io);
+        if (!confirmed) {
+            return { kind: "result", result: confirmationDeclinedResult(command, ctx) };
+        }
+    }
+    if (command.dryRun) {
+        if (!command.json) {
+            writeDryRunSummary(io, {
+                candidates: scan.candidates,
+                reasonCounts,
+                dashboardUrl,
+                dryRunOnly: true,
+            });
+        }
+        return { kind: "result", result: dryRunResult(command, ctx) };
+    }
+    return { kind: "proceed" };
+}
+/** The operator saw the `--all` review and said no. Nothing was written; everything remains. */
+function confirmationDeclinedResult(command, ctx) {
+    const base = baseBackfillResult(command, backfillResultBaseArgs(ctx));
+    return {
+        ...base,
+        status: "blocked",
+        counts: { ...base.counts, remaining: ctx.scan.candidates.length },
+        failure_reason: "confirmation_declined",
+    };
+}
+/**
+ * What a `--dry-run` would have done. It completes unless the scan itself hit
+ * something blocking, because a dry run that reports "complete" over a scan
+ * that could not see all the history would be the confident wrong answer.
+ */
+function dryRunResult(command, ctx) {
+    const blockingIssues = blockingScanIssues(ctx.scan.issues);
+    return {
+        ...baseBackfillResult(command, backfillResultBaseArgs(ctx)),
+        status: blockingIssues.length > 0 ? "partial" : "complete",
+        dry_run: true,
+        retry_command: ctx.retryCommand,
+        ...(blockingIssues.length > 0
+            ? { failure_reason: blockingIssues[0]?.reason }
+            : {}),
+    };
+}

package/dist/commands/backfill-reasons.js

@@ -0,0 +1,107 @@
+/**
+ * The skip-reason table printed under every backfill run: one row per reason,
+ * each saying whether a rerun can fix it and what would have to change.
+ * A reason with no verdict here is still reported — it falls back to the
+ * retryable default rather than going silent.
+ */
+import { increment } from "./backfill-issues.js";
+export function reasonCountsFor(candidates, guardCounts, scanIssues) {
+    const counts = new Map();
+    for (const candidate of candidates)
+        increment(counts, candidate.reason);
+    for (const [reason, count] of guardCounts) {
+        counts.set(reason, Math.max(counts.get(reason) ?? 0, count));
+    }
+    for (const issue of scanIssues) {
+        counts.set(issue.reason, Math.max(counts.get(issue.reason) ?? 0, issue.count));
+    }
+    for (const required of ["file_too_large", "repo_not_on_disk"]) {
+        counts.set(required, counts.get(required) ?? 0);
+    }
+    return [...counts.entries()]
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .map(([reason, count]) => ({
+        reason,
+        count,
+        ...reasonClassification(reason),
+    }));
+}
+/**
+ * The skip-reason table printed under every backfill run. These strings are
+ * what a person reads when coverage is short, so each note says what would
+ * have to change, not merely that something went wrong. Reason labels here
+ * must match the ones the scan and the guards emit verbatim.
+ */
+const REASON_VERDICTS = new Map([
+    [
+        "secret_like_content_guard",
+        {
+            classification: "retryable",
+            note: "historical guard result; collector now masks and retries",
+        },
+    ],
+    [
+        "secret_redaction_failed",
+        {
+            classification: "retryable",
+            note: "historical guard result; collector now masks and retries",
+        },
+    ],
+    [
+        "file_too_large",
+        {
+            classification: "retryable",
+            note: "until the evidence file cap is raised",
+        },
+    ],
+    [
+        "repo_not_on_disk",
+        { classification: "retryable", note: "repo must exist on disk" },
+    ],
+    [
+        "cwd_not_a_repo",
+        {
+            classification: "permanent",
+            note: "cwd exists but is not a repo or folder workspace",
+        },
+    ],
+    [
+        "multiple_transcript_origins",
+        {
+            classification: "permanent",
+            note: "multiple transcript origins; attribution is ambiguous",
+        },
+    ],
+    [
+        "single_repo_folder_fallback",
+        {
+            classification: "permanent",
+            note: "uploadable single-repo folder workspace fallback",
+        },
+    ],
+    [
+        "multi_repo_folder_workspace",
+        {
+            classification: "permanent",
+            note: "uploadable multi-repo folder workspace fallback",
+        },
+    ],
+]);
+const DEFERRED_REASON_VERDICT = {
+    classification: "retryable",
+    note: "rerun cockpit backfill to continue",
+};
+const UNKNOWN_REASON_VERDICT = {
+    classification: "retryable",
+    note: "rerun after fixing source or collector state",
+};
+function reasonClassification(reason) {
+    const known = REASON_VERDICTS.get(reason);
+    if (known)
+        return known;
+    // A budget deferral is the one family, rather than one label: whichever
+    // budget ran out, the remedy is the same rerun.
+    if (reason.startsWith("deferred_"))
+        return DEFERRED_REASON_VERDICT;
+    return UNKNOWN_REASON_VERDICT;
+}

package/dist/commands/backfill-report.js

@@ -0,0 +1,298 @@
+import { BACKFILL_COMPLETION_RECHECK_MS, BACKFILL_COVERAGE_VERSION, recordBackfillScanCoverage, writeBackfillCompletionMarker, writeBackfillCursor, } from "../cursors/backfill-cursor.js";
+import { describeError } from "../health-detail.js";
+import { readLocalWorkContextForRepo, startLocalWorkContext, } from "../local-state.js";
+import { postCodexSessionReport, } from "../upload.js";
+import { candidateCursorKey } from "./backfill-candidates.js";
+import { advanceBackfillCursorThroughResolvedPrefix, recordBackfillDurableSessionPointers, } from "./backfill-checkpoint.js";
+import { blockingScanIssues } from "./backfill-issues.js";
+import { backfillResultBaseArgs, baseBackfillResult, emptyReport, } from "./backfill-result.js";
+import { buildBackfillSessionReport } from "./backfill-session-report.js";
+/**
+ * REPORT: post the session report, record durable pointers, advance the
+ * cursor through the resolved contiguous prefix, decide completion, write the
+ * all-history completion marker when this run actually finished it, and
+ * assemble the final result. `remaining`/`completionBlocked` deliberately
+ * treat an oversized skip differently (BLI-2727): it still counts toward
+ * `remaining` so the JSON output never goes silent about it, but it never
+ * blocks completion on its own — every completion-gating computation below
+ * excludes it explicitly via `oversizedCandidateKeys`/`blockingScanIssues`.
+ */
+export async function reportBackfillOutcome(command, io, ctx, upload) {
+    let { blockedAt, failureReason } = upload;
+    const posted = await postBackfillSessionReport(command, io, ctx, upload);
+    if (!posted.acknowledged && !blockedAt) {
+        blockedAt = sessionReportStopPoint(upload);
+        failureReason ??= sessionReportFailureReason(posted);
+    }
+    if (posted.acknowledged) {
+        await checkpointResolvedBackfillProgress(ctx, upload);
+    }
+    const completion = summarizeBackfillCompletion({
+        ctx,
+        upload,
+        posted,
+        stoppedEarly: Boolean(blockedAt),
+        failureReason,
+    });
+    if (completion.status === "complete" && command.all) {
+        await writeAllHistoryCompletionMarker(ctx);
+    }
+    return assembleBackfillResult({
+        command,
+        ctx,
+        upload,
+        posted,
+        completion,
+        blockedAt,
+    });
+}
+/**
+ * Tells the server what this run observed about every session, uploaded or
+ * not. Its acknowledgement — not raw-evidence durability alone — is what makes
+ * a historical cursor position irreversible, so the caller gates the whole
+ * checkpoint on the `acknowledged` flag returned here.
+ */
+async function postBackfillSessionReport(command, io, ctx, upload) {
+    const sessions = buildBackfillSessionReport({
+        candidates: ctx.scan.candidates,
+        syncResults: upload.syncResults,
+        now: ctx.now,
+    });
+    if (sessions.length === 0) {
+        return {
+            sessions,
+            report: emptyReport("no_sessions_observed"),
+            acknowledged: true,
+        };
+    }
+    const reportContext = await ensureBackfillReportContext({
+        homeDir: command.homeDir,
+        paths: ctx.paths,
+        collectionRoots: ctx.collectionRoots,
+        worktrees: ctx.worktrees,
+        candidates: ctx.scan.candidates,
+    });
+    const report = await postCodexSessionReport({
+        homeDir: command.homeDir,
+        repoRoot: reportContext.repoRoot,
+        dashboardUrl: ctx.dashboardUrl,
+        sessions,
+        fetch: io.fetch,
+        now: ctx.now,
+    });
+    return {
+        sessions,
+        report,
+        acknowledged: report.posted && report.recorded_count >= sessions.length,
+    };
+}
+/** Where a run that uploaded cleanly but could not file its report stopped. */
+function sessionReportStopPoint(upload) {
+    return {
+        what: "session report failed",
+        batch_index: upload.completedBatches,
+        batch_total: upload.batches.length,
+        done: upload.done,
+        total: upload.uploadable.length,
+    };
+}
+/** A partial acknowledgement is its own reason; otherwise the post's own. */
+function sessionReportFailureReason(posted) {
+    return posted.report.posted &&
+        posted.report.recorded_count < posted.sessions.length
+        ? "session_report_ack_incomplete"
+        : posted.report.reason;
+}
+/**
+ * Makes this run's progress durable once the server has acknowledged it:
+ * session pointers first, then the backfill cursor through the contiguous
+ * resolved prefix. Raw-evidence durability is necessary but not sufficient —
+ * both writes wait on the report acknowledgement, because a cursor advanced
+ * past a session the server never recorded can never be walked back.
+ */
+async function checkpointResolvedBackfillProgress(ctx, upload) {
+    const { paths, scan, cursor, now } = ctx;
+    if (upload.durableCandidateKeys.size > 0) {
+        await recordBackfillDurableSessionPointers({
+            paths,
+            candidates: scan.candidates,
+            syncResults: upload.syncResults,
+            now,
+        });
+    }
+    const cursorAdvanced = advanceBackfillCursorThroughResolvedPrefix({
+        cursor,
+        candidates: scan.candidates,
+        durableCandidateKeys: upload.durableCandidateKeys,
+        retryableCandidateKeys: scan.retryable_candidate_keys,
+        discoveryComplete: !scan.issues.some((issue) => issue.scope === "global"),
+        now,
+    });
+    if (cursorAdvanced)
+        await writeBackfillCursor(paths, cursor);
+}
+/**
+ * Answers "is this backfill finished, and if not, why not" as pure arithmetic
+ * over what the scan found and what the upload resolved — no side effects, so
+ * the one place BLI-2727's oversized-skip rule is applied is readable in full.
+ * `remaining` is a reporting total that still counts oversized skips so the
+ * JSON never goes silent about them; every `*Blocking` count excludes them,
+ * because a file too large for the current cap is a permanent labeled fact
+ * that no rerun can resolve and completion must not wait on forever.
+ */
+function summarizeBackfillCompletion(options) {
+    const { scan, oversizedCandidateKeys } = options.ctx;
+    const { upload, posted } = options;
+    const unresolvedUploadableKeys = new Set(upload.uploadable
+        .filter((candidate) => !upload.durableCandidateKeys.has(candidateCursorKey(candidate)))
+        .map(candidateCursorKey));
+    const unresolvedUploadableBlocking = [...unresolvedUploadableKeys].filter((key) => !oversizedCandidateKeys.has(key)).length;
+    const unresolvedRetryableBlocking = [...scan.retryable_candidate_keys].filter((key) => !oversizedCandidateKeys.has(key)).length;
+    const remaining = countRemainingHistory({
+        scan,
+        unresolvedUploadableKeys,
+        posted,
+    });
+    const failed = Math.max(upload.failed, unresolvedUploadableBlocking);
+    const blockingIssues = blockingScanIssues(scan.issues);
+    const completionBlocked = blockingIssues.length > 0 ||
+        unresolvedUploadableBlocking > 0 ||
+        unresolvedRetryableBlocking > 0 ||
+        upload.deferred > 0 ||
+        failed > 0 ||
+        !posted.acknowledged;
+    let failureReason = options.failureReason;
+    if (!failureReason && completionBlocked) {
+        const retryableCandidateReason = scan.candidates.find((candidate) => scan.retryable_candidate_keys.has(candidateCursorKey(candidate)) &&
+            !oversizedCandidateKeys.has(candidateCursorKey(candidate)))?.reason;
+        failureReason =
+            blockingIssues[0]?.reason ??
+                (unresolvedUploadableBlocking > 0
+                    ? "durable_session_pointer_missing"
+                    : (retryableCandidateReason ?? "backfill_incomplete"));
+    }
+    return {
+        remaining,
+        failed,
+        status: options.stoppedEarly || completionBlocked ? "partial" : "complete",
+        failureReason,
+    };
+}
+/**
+ * How much history a rerun still has to do. A reporting total, not a
+ * completion gate: it counts every unresolved candidate — oversized skips
+ * included, so `--json` never goes silent about them (BLI-2727) — plus what
+ * selection dropped, what discovery never saw, and the report itself when the
+ * server has not acknowledged it.
+ */
+function countRemainingHistory(options) {
+    const unresolvedKnownKeys = new Set([
+        ...options.unresolvedUploadableKeys,
+        ...options.scan.retryable_candidate_keys,
+    ]);
+    const unseenGlobalFailures = options.scan.issues
+        .filter((issue) => issue.scope === "global")
+        .reduce((total, issue) => total + issue.count, 0);
+    const reportRetryable = options.posted.acknowledged
+        ? 0
+        : // The cursor is intentionally all-or-nothing for the report. Even
+            // chunks already accepted by the server are retried idempotently when
+            // another required chunk lacks an acknowledgement.
+            Math.max(1, options.posted.sessions.length);
+    return (unresolvedKnownKeys.size +
+        options.scan.omitted_candidate_count +
+        unseenGlobalFailures +
+        reportRetryable);
+}
+/**
+ * Proof, for doctor and status, that archived history is covered. Only an
+ * `--all` run earns it: a bounded `--since-days` run may complete its
+ * requested window, but that is not proof of an all-history backfill.
+ */
+async function writeAllHistoryCompletionMarker(ctx) {
+    const { paths, cursor, sources, scopedCursor, scan, oversizedCandidateKeys, now } = ctx;
+    recordBackfillScanCoverage(cursor, sources, now, now);
+    await writeBackfillCursor(paths, cursor);
+    const oversizedCandidates = scan.candidates.filter((candidate) => oversizedCandidateKeys.has(candidateCursorKey(candidate)));
+    await writeBackfillCompletionMarker(paths, {
+        schema_version: "cockpit-backfill-complete.v2",
+        coverage_version: BACKFILL_COVERAGE_VERSION,
+        collection_scope_id: scopedCursor.collection_scope_id,
+        sources,
+        completed_at: now.toISOString(),
+        revalidate_after: new Date(now.getTime() + BACKFILL_COMPLETION_RECHECK_MS).toISOString(),
+        cursor,
+        ...(oversizedCandidates.length > 0
+            ? {
+                oversized_skips: {
+                    reason: "file_too_large",
+                    count: oversizedCandidates.length,
+                    byte_sizes: oversizedCandidates.map((candidate) => candidate.byte_size),
+                },
+            }
+            : {}),
+    });
+}
+/** The `--json` payload: every stage's numbers merged onto the scan's baseline. */
+function assembleBackfillResult(options) {
+    const { command, ctx, upload, posted, completion } = options;
+    const base = baseBackfillResult(command, backfillResultBaseArgs(ctx));
+    return {
+        ...base,
+        status: completion.status,
+        retry_command: ctx.retryCommand,
+        counts: {
+            ...base.counts,
+            backfilled: upload.backfilledSessions,
+            failed: completion.failed,
+            deferred: upload.deferred,
+            remaining: completion.remaining,
+        },
+        batches: {
+            total: upload.batches.length,
+            completed: upload.completedBatches,
+            failed: upload.failedBatches,
+        },
+        report: posted.report,
+        server_acknowledged: {
+            codex_session_report_recorded_count: posted.report.recorded_count,
+            raw_evidence_uploaded_object_count: upload.uploadedObjects,
+            raw_evidence_uploaded_chunk_count: upload.uploadedChunks,
+        },
+        blocked_at: options.blockedAt,
+        failure_reason: completion.failureReason,
+    };
+}
+async function ensureBackfillReportContext(options) {
+    const candidateWorktree = options.candidates.find((candidate) => candidate.worktree)?.worktree;
+    const representative = candidateWorktree ?? options.worktrees[0] ?? null;
+    // A deleted repo or an approved folder workspace can legitimately produce
+    // report-only rows with no current git worktree. Use the exact approved root
+    // as a synthetic local context instead of falling through to process.cwd().
+    const repoRoot = representative?.repo_root ??
+        options.collectionRoots[0] ??
+        process.cwd();
+    try {
+        await readLocalWorkContextForRepo(options.paths, repoRoot);
+    }
+    catch {
+        // Reading it can legitimately fail — there is no context yet, which is
+        // precisely why the next line creates one. That read is a probe and stays
+        // silent; the CREATE is the branch that has to speak (BLI-3238).
+        await startLocalWorkContext({
+            homeDir: options.homeDir,
+            repoRoot,
+            branch: representative?.branch,
+        }).catch((error) => {
+            // Context creation is best-effort here: postCodexSessionReport converts
+            // a remaining local-context failure into a retryable report reason. But
+            // that reason is `collector_not_ready`, which points the operator at
+            // setup rather than at whatever actually failed here.
+            console.error("[cockpit-backfill] could not start a local work context for the batch", JSON.stringify({
+                reason: "work_context_start_failed",
+                ...describeError(error),
+            }));
+        });
+    }
+    return { repoRoot };
+}