@bli-cockpit/cli 0.2.25 → 0.2.26

package/dist/adapters/raw-evidence.js

@@ -15,8 +15,9 @@ const GIT_DIFF_TIMEOUT_MS = 3_000;
 export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
 export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
 // Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
-// transcript can approach the 256 MiB wire cap, so 2 GiB leaves room for
-// several files without starving the sync; overflow still defers and converges.
+// transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
+// so 2 GiB leaves room for several files without starving the sync; overflow
+// still defers and converges.
 export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
 export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
 const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;

package/dist/commands/backfill.js

@@ -129,6 +129,23 @@ export async function runBackfill(command, io) {
         });
     }
     const reasonCounts = reasonCountsFor(scan.candidates, guards.counts, scan.issues);
+    // BLI-2727: a deterministic, labeled oversized skip must never poison
+    // completion — it is a permanent, non-retryable fact about the file, not an
+    // in-flight problem a rerun can fix. `scan.issues`/`retryable_candidate_keys`
+    // still carry it (so it's never silently dropped from reporting); every
+    // completion-gating computation below excludes it explicitly instead.
+    const oversizedCandidateKeys = oversizedBackfillCandidateKeys(scan.candidates);
+    // Two scan issues describe the exact same oversized-main candidates:
+    // `backfillScanIssues` pushes the Claude-specific `claude_main_file_too_large`
+    // (from `claude?.main_file_oversized`) and `countReadOnlyGuards` separately
+    // pushes the source-agnostic `file_too_large` (same predicate as
+    // `oversizedBackfillCandidateKeys`, so this list can never drift from it).
+    // Both must be excluded from completion-gating together.
+    const OVERSIZED_SCAN_ISSUE_REASONS = new Set([
+        "file_too_large",
+        "claude_main_file_too_large",
+    ]);
+    const blockingScanIssues = (issues) => issues.filter((issue) => !OVERSIZED_SCAN_ISSUE_REASONS.has(issue.reason));
     if (command.all && !command.yes) {
         if (!command.json) {
             writeDryRunSummary(io, {
@@ -186,11 +203,11 @@ export async function runBackfill(command, io) {
                 scan,
                 reasonCounts,
             }),
-            status: scan.issues.length > 0 ? "partial" : "complete",
+            status: blockingScanIssues(scan.issues).length > 0 ? "partial" : "complete",
             dry_run: true,
             retry_command: retryCommand,
-            ...(scan.issues.length > 0
-                ? { failure_reason: scan.issues[0]?.reason }
+            ...(blockingScanIssues(scan.issues).length > 0
+                ? { failure_reason: blockingScanIssues(scan.issues)[0]?.reason }
                 : {}),
         };
     }
@@ -276,7 +293,7 @@ export async function runBackfill(command, io) {
         let backfilledSessions = 0;
         const durableCandidateKeys = new Set();
         let blockedAt;
-        let failureReason = scan.issues[0]?.reason;
+        let failureReason = blockingScanIssues(scan.issues)[0]?.reason;
         for (const [index, batch] of batches.entries()) {
             await lock.handle.heartbeat();
             const sync = await syncBackfillBatch({
@@ -300,7 +317,12 @@ export async function runBackfill(command, io) {
             for (const key of durableInBatch)
                 durableCandidateKeys.add(key);
             backfilledSessions = durableCandidateKeys.size;
-            const missingDurableMain = batch.candidates.length - durableInBatch.size;
+            // A candidate whose main is durably oversized-skipped (BLI-2727) is
+            // accounted for, not missing — it will never earn a durable pointer
+            // under the current cap, and treating it as "still missing" would fail
+            // every batch it happens to share with genuinely uploaded siblings.
+            const missingDurableMain = batch.candidates.filter((candidate) => !durableInBatch.has(candidateCursorKey(candidate)) &&
+                !oversizedCandidateKeys.has(candidateCursorKey(candidate))).length;
             const batchFailed = sync.status !== "uploaded" ||
                 countSessionUploadFailures(sync) > 0 ||
                 missingDurableMain > 0;
@@ -414,8 +436,11 @@ export async function runBackfill(command, io) {
         const unresolvedUploadableKeys = new Set(uploadable
             .filter((candidate) => !durableCandidateKeys.has(candidateCursorKey(candidate)))
             .map(candidateCursorKey));
-        const unresolvedUploadable = unresolvedUploadableKeys.size;
-        const unresolvedRetryable = scan.retryable_candidate_keys.size;
+        // "Blocking" views exclude a deterministic oversized skip (BLI-2727): it is
+        // still unresolved (it never gets a durable pointer), but it is a permanent,
+        // labeled fact rather than something completion should wait on forever.
+        const unresolvedUploadableBlocking = [...unresolvedUploadableKeys].filter((key) => !oversizedCandidateKeys.has(key)).length;
+        const unresolvedRetryableBlocking = [...scan.retryable_candidate_keys].filter((key) => !oversizedCandidateKeys.has(key)).length;
         const unresolvedKnownKeys = new Set([
             ...unresolvedUploadableKeys,
             ...scan.retryable_candidate_keys,
@@ -429,22 +454,27 @@ export async function runBackfill(command, io) {
             // chunks already accepted by the server are retried idempotently when
             // another required chunk lacks an acknowledgement.
             : Math.max(1, sessions.length);
+        // `remaining` is a reporting total, not a completion gate: it still counts
+        // every unresolved candidate, including oversized skips, so the JSON output
+        // never goes silent about them (BLI-2727).
         const remaining = unresolvedKnownKeys.size +
             scan.omitted_candidate_count +
             unseenGlobalFailures +
             reportRetryable;
-        failed = Math.max(failed, unresolvedUploadable);
-        const completionBlocked = scan.issues.length > 0 ||
-            unresolvedUploadable > 0 ||
-            unresolvedRetryable > 0 ||
+        failed = Math.max(failed, unresolvedUploadableBlocking);
+        const blockingIssues = blockingScanIssues(scan.issues);
+        const completionBlocked = blockingIssues.length > 0 ||
+            unresolvedUploadableBlocking > 0 ||
+            unresolvedRetryableBlocking > 0 ||
             deferred > 0 ||
             failed > 0 ||
             !reportAcknowledged;
         if (!failureReason && completionBlocked) {
-            const retryableCandidateReason = scan.candidates.find((candidate) => scan.retryable_candidate_keys.has(candidateCursorKey(candidate)))?.reason;
+            const retryableCandidateReason = scan.candidates.find((candidate) => scan.retryable_candidate_keys.has(candidateCursorKey(candidate)) &&
+                !oversizedCandidateKeys.has(candidateCursorKey(candidate)))?.reason;
             failureReason =
-                scan.issues[0]?.reason ??
-                    (unresolvedUploadable > 0
+                blockingIssues[0]?.reason ??
+                    (unresolvedUploadableBlocking > 0
                         ? "durable_session_pointer_missing"
                         : retryableCandidateReason ?? "backfill_incomplete");
         }
@@ -455,6 +485,7 @@ export async function runBackfill(command, io) {
         if (status === "complete" && command.all) {
             recordBackfillScanCoverage(cursor, sources, now, now);
             await writeBackfillCursor(paths, cursor);
+            const oversizedCandidates = scan.candidates.filter((candidate) => oversizedCandidateKeys.has(candidateCursorKey(candidate)));
             await writeBackfillCompletionMarker(paths, {
                 schema_version: "cockpit-backfill-complete.v2",
                 coverage_version: BACKFILL_COVERAGE_VERSION,
@@ -463,6 +494,15 @@ export async function runBackfill(command, io) {
                 completed_at: now.toISOString(),
                 revalidate_after: new Date(now.getTime() + BACKFILL_COMPLETION_RECHECK_MS).toISOString(),
                 cursor,
+                ...(oversizedCandidates.length > 0
+                    ? {
+                        oversized_skips: {
+                            reason: "file_too_large",
+                            count: oversizedCandidates.length,
+                            byte_sizes: oversizedCandidates.map((candidate) => candidate.byte_size),
+                        },
+                    }
+                    : {}),
             });
         }
         return {
@@ -889,6 +929,27 @@ function retryableCandidateKeys(candidates) {
             sidecar.skipped_reason === "file_too_large")))
         .map(candidateCursorKey));
 }
+/**
+ * A main session file whose only story is "too large to upload under the
+ * current cap" (BLI-2727). This mirrors exactly the two branches in
+ * `countReadOnlyGuards` that emit the `file_too_large` reason, so a candidate
+ * is in this set if and only if it contributed to that scan issue's count —
+ * one predicate, no drift between "why the issue fired" and "which candidate
+ * caused it". Deterministic and non-retryable: rerunning backfill cannot
+ * resolve it (only a larger cap or a smaller file can), so unlike a transient
+ * read failure it must never poison completion or a batch's success.
+ */
+function oversizedBackfillCandidateKeys(candidates) {
+    const keys = new Set();
+    for (const candidate of candidates) {
+        if ((candidate.source === "claude_code" &&
+            candidate.claude?.main_file_oversized) ||
+            candidate.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
+            keys.add(candidateCursorKey(candidate));
+        }
+    }
+    return keys;
+}
 async function countUnreadableClaudeSidecarDirs(candidates) {
     let unreadable = 0;
     for (const candidate of candidates) {

package/dist/commands/doctor.js

@@ -243,13 +243,36 @@ async function fixAutostartState(context) {
     }
     return ok("autostart-alive", "installed", "autostart installed and loaded");
 }
+/**
+ * Pure so it can be unit-tested without touching the real machine's home
+ * directory (`getCollectorRuntimePaths()` defaults to `os.homedir()` and
+ * doctor never threads `--home` through the backfill steps). Returns `null`
+ * when the marker does not cover the roots/sources — the caller falls
+ * through to the lock/never-run diagnosis in that case.
+ *
+ * BLI-2727: a marker whose only outstanding entries are deterministic
+ * oversized-file skips is still a completed backfill — it reads green with a
+ * named note, never a red `needs_fix`/`fail`, so an unliftable file cap never
+ * reads as "backfill never completed" on repeat doctor runs.
+ */
+export function backfillCompletionStepState(marker, roots) {
+    if (!backfillCompletionCovers(marker, roots, ["codex", "claude_code"])) {
+        return null;
+    }
+    const oversized = marker?.oversized_skips;
+    if (oversized && oversized.count > 0) {
+        return ok("backfill-complete", "complete_with_oversized_skips", `backfill completion covers the current saved roots and both session sources ` +
+            `(complete_with_oversized_skips · ${oversized.count} file${oversized.count === 1 ? "" : "s"} over the upload cap)`);
+    }
+    return ok("backfill-complete", "complete", "backfill completion covers the current saved roots and both session sources");
+}
 async function checkBackfillState(context) {
     const paths = getCollectorRuntimePaths();
     const roots = await doctorRoots(context);
     const marker = await readBackfillCompletionMarker(paths);
-    if (backfillCompletionCovers(marker, roots, ["codex", "claude_code"])) {
-        return ok("backfill-complete", "complete", "backfill completion covers the current saved roots and both session sources");
-    }
+    const covered = backfillCompletionStepState(marker, roots);
+    if (covered)
+        return covered;
     const lock = await inspectBackfillLock(paths);
     if (lock.held) {
         return needsFix("backfill-complete", "backfill_already_running", `backfill completion is not yet proven; another run holds the lock since ${lock.held_since ?? "unknown"}`);
@@ -268,6 +291,13 @@ async function fixBackfillState(context) {
     }, capture.io);
     const output = capture.stdout() + "\n" + capture.stderr();
     if (code === 0) {
+        // Re-read the marker this run just wrote instead of hand-rolling a second
+        // message: `checkBackfillState`'s pure core already knows how to say
+        // "complete" vs "complete_with_oversized_skips" (BLI-2727), and this way
+        // the two can never say something different for the same marker.
+        const recheck = await checkBackfillState(context);
+        if (recheck.status === "ok")
+            return recheck;
         return ok("backfill-complete", "completed", "ran `cockpit backfill --all --yes`");
     }
     const reason = jsonField(output, "failure_reason");
@@ -308,6 +338,66 @@ async function checkSyncState(context) {
     // and only turns green from those command receipts.
     return needsFix("sync-fresh", "per_root_verification_required", `fresh upload proof is required for ${roots.length} saved root${roots.length === 1 ? "" : "s"}`);
 }
+/**
+ * `cockpit sync --json` prints exactly one JSON document to stdout (stderr is
+ * for human text; see AGENTS.md logging conventions), so this is a real parse
+ * rather than the doctor module's usual regex field-scrape — which cannot
+ * disambiguate same-named fields nested under `codex_sessions.codex` vs
+ * `codex_sessions.claude` (BLI-2728).
+ */
+function parseDoctorSyncJson(stdout) {
+    try {
+        const parsed = JSON.parse(stdout.trim());
+        return parsed && typeof parsed === "object" ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * BLI-2728: a tick that only deferred objects past the per-tick raw-evidence
+ * object budget (`RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET`, adapters/raw-evidence.ts)
+ * is a backlog that is draining, not a failure — `attributedSyncRunStatus`
+ * marks the run not-fully-`ok` (so `cockpit sync` exits non-zero and doctor's
+ * exec sees `code !== 0`) purely because objects remain queued, with the
+ * per-repo upload itself still having succeeded. A genuine failure (auth,
+ * server rejection, network, a real upload_failed outcome, an unposted
+ * session report) must still read red — this only fires when NOTHING else in
+ * the tick's own summary looks wrong. Pure so it is unit-testable without a
+ * live exec/fs harness; the remaining-object count is read straight from the
+ * tick's own summary, never recomputed.
+ */
+export function syncBacklogDrainingVerdict(parsed) {
+    if (!parsed)
+        return null;
+    const deferredObjects = positiveNumberOrZero(parsed.raw_evidence_deferred_object_budget);
+    if (deferredObjects <= 0)
+        return null;
+    const deferredBytes = positiveNumberOrZero(parsed.raw_evidence_deferred_byte_budget);
+    const failedCount = positiveNumberOrZero(parsed.raw_evidence_failed_count);
+    const retryReasons = Array.isArray(parsed.raw_evidence_retry_reasons)
+        ? parsed.raw_evidence_retry_reasons.length
+        : 0;
+    const sessions = asRecord(parsed.codex_sessions);
+    const reportPosted = sessions?.["report_posted"];
+    const codexReadFailures = positiveNumberOrZero(asRecord(sessions?.["codex"])?.["read_failures"]);
+    const claudeSessions = asRecord(sessions?.["claude"]);
+    const claudeReadFailures = positiveNumberOrZero(claudeSessions?.["read_failures"]);
+    const claudeSidecarsFailed = positiveNumberOrZero(claudeSessions?.["sidecars_failed"]);
+    const onlyDeferredObjectBudget = deferredBytes === 0 &&
+        failedCount === 0 &&
+        retryReasons === 0 &&
+        reportPosted === true &&
+        codexReadFailures === 0 &&
+        claudeReadFailures === 0 &&
+        claudeSidecarsFailed === 0;
+    return onlyDeferredObjectBudget ? { remainingObjects: deferredObjects } : null;
+}
+function positiveNumberOrZero(value) {
+    return typeof value === "number" && Number.isFinite(value) && value > 0
+        ? value
+        : 0;
+}
 async function fixSyncState(context) {
     const exec = context.io.exec;
     if (!exec)
@@ -326,9 +416,16 @@ async function fixSyncState(context) {
             args.push("--dashboard-url", context.command.dashboardUrl);
         }
         const result = await exec("cockpit", args);
+        const parsed = parseDoctorSyncJson(result.stdout);
         const output = `${result.stdout}\n${result.stderr}`;
         const status = jsonField(output, "status");
         if (result.code !== 0) {
+            const draining = syncBacklogDrainingVerdict(parsed);
+            if (draining) {
+                return needsFix("sync-fresh", "backlog_draining", `${repoRoot}: raw-evidence backlog is still draining (${draining.remainingObjects} ` +
+                    `object${draining.remainingObjects === 1 ? "" : "s"} deferred this tick); ` +
+                    "rerun `cockpit sync` to continue");
+            }
             return fail("sync-fresh", status ?? "sync_failed", `sync failed for ${repoRoot}`);
         }
         if (status !== "uploaded") {

package/dist/commands/public-root.js

@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
   }
 
   if (command === "--version" || command === "-V" || command === "version") {
-    writeLine(io?.stdout ?? process.stdout, "0.2.25");
+    writeLine(io?.stdout ?? process.stdout, "0.2.26");
     return 0;
   }
 

package/dist/cursors/backfill-cursor.js

@@ -239,8 +239,26 @@ function parseBackfillCompletionMarker(value) {
         completed_at: completedAt,
         revalidate_after: revalidateAfter,
         cursor: parseBackfillCursor(record["cursor"]),
+        ...(parseOversizedSkips(record["oversized_skips"])
+            ? { oversized_skips: parseOversizedSkips(record["oversized_skips"]) }
+            : {}),
     };
 }
+function parseOversizedSkips(value) {
+    if (!value || typeof value !== "object")
+        return null;
+    const record = value;
+    if (record["reason"] !== "file_too_large")
+        return null;
+    const count = optionalNumber(record["count"]);
+    if (count === null || count <= 0)
+        return null;
+    const rawByteSizes = record["byte_sizes"];
+    const byteSizes = Array.isArray(rawByteSizes)
+        ? rawByteSizes.filter((entry) => typeof entry === "number" && Number.isFinite(entry) && entry >= 0)
+        : [];
+    return { reason: "file_too_large", count, byte_sizes: byteSizes };
+}
 function normalizeScopeRoot(value) {
     const windowsStyle = path.win32.isAbsolute(value) && !path.posix.isAbsolute(value);
     if (windowsStyle)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.25",
+  "version": "0.2.26",
   "private": false,
   "type": "module",
   "bin": {
@@ -26,6 +26,6 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/telemetry-core": "0.1.19"
+    "@bli-cockpit/telemetry-core": "0.1.20"
   }
 }