@bli-cockpit/cli 0.2.4 → 0.2.7

package/dist/commands/backfill.js

@@ -1,4 +1,5 @@
 import { containsSecretLikeContent, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
+import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
@@ -6,38 +7,41 @@ import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.j
 import { defaultCodexSessionDirs, scanAndAttributeCodexSessions } from "../adapters/codex-attribution.js";
 import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET } from "../adapters/raw-evidence.js";
 import { acquireBackfillLock } from "../backfill-lock.js";
-import { emptyBackfillCursorState, readBackfillCursor, recordBackfillCursorObservations, writeBackfillCompletionMarker, writeBackfillCursor } from "../cursors/backfill-cursor.js";
-import { getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, startLocalWorkContext } from "../local-state.js";
-import { discoverGitWorktrees } from "../repo-identity.js";
+import { BACKFILL_COMPLETION_RECHECK_MS, BACKFILL_COVERAGE_VERSION, emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCursor, recordBackfillCursorObservations, recordBackfillScanCoverage, writeBackfillCompletionMarker, writeBackfillCursor, } from "../cursors/backfill-cursor.js";
+import { getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, startLocalWorkContext } from "../local-state.js";
+import { collectionRootPathAliases, discoverGitWorktreesInRootsWithStatus, } from "../repo-identity.js";
+import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
 import { normalizeCollectionRoots } from "../root-normalization.js";
+import { acquireSyncLock } from "../sync-lock.js";
 import { LocalUploadBlockedError, postCodexSessionReport, syncLocalAmbientEnvelope } from "../upload.js";
 const BACKFILL_UPLOAD_BATCH_SESSIONS = 25;
 const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
 const ALL_BACKFILL_SINCE_MINUTES = 20 * 365 * 24 * 60;
 const DEFAULT_DISCOVERY_MAX_DEPTH = 3;
 const DEFAULT_DISCOVERY_MAX_REPOS = 50;
-const RETRY_COMMAND = "cockpit backfill";
 export async function runBackfillCommand(command, io) {
     if (!command.all && command.sinceDays === undefined) {
         const message = bareBackfillMessage();
+        const retryCommand = backfillRetryCommand(command);
         writeLine(io.stderr, message);
         if (command.json) {
             writeLine(io.stdout, JSON.stringify({
                 status: "blocked",
                 reason: "missing_window",
-                retry_command: RETRY_COMMAND,
+                retry_command: retryCommand,
             }, null, 2));
         }
         return 1;
     }
     if (command.all && !command.yes && !isInteractiveStdin(io)) {
         const message = "--all requires TTY confirmation; pass --yes for agent runs.";
+        const retryCommand = backfillRetryCommand(command);
         writeLine(io.stderr, message);
         if (command.json) {
             writeLine(io.stdout, JSON.stringify({
                 status: "blocked",
                 reason: "all_requires_confirmation",
-                retry_command: RETRY_COMMAND,
+                retry_command: retryCommand,
             }, null, 2));
         }
         return 1;
@@ -67,8 +71,10 @@ export async function runBackfill(command, io) {
     const pairedAt = parseRequiredDate(sessionFile.paired_at, "paired_at");
     const dashboardUrl = normalizeDashboardUrl(sessionFile.dashboard_url ?? config.dashboard_url);
     const roots = backfillCollectionRoots(command, config.default_repo_paths);
-    const worktrees = await discoverBackfillWorktrees(roots);
-    const collectionRoots = normalizeCollectionRoots(roots);
+    const collectionRoots = normalizeCollectionRoots(await collectionRootPathAliases(roots));
+    const worktreeDiscovery = await discoverBackfillWorktrees(collectionRoots, command);
+    const retryCommand = backfillDiscoveryRetryCommand(command, worktreeDiscovery);
+    const worktrees = worktreeDiscovery.worktrees;
     const sources = selectedSources(command.source);
     const window = backfillWindow(command, now, pairedAt);
     await validateBackfillReachability({
@@ -76,9 +82,11 @@ export async function runBackfill(command, io) {
         dashboardUrl,
         dryRun: command.dryRun,
     });
-    const cursor = command.dryRun
+    const storedCursor = command.dryRun
         ? emptyBackfillCursorState()
         : await readBackfillCursor(paths);
+    const scopedCursor = prepareBackfillCursorForScope(storedCursor, collectionRoots, sources);
+    const cursor = scopedCursor.cursor;
     const scan = await scanBackfillSessions({
         command,
         homeDir: command.homeDir ?? os.homedir(),
@@ -89,8 +97,42 @@ export async function runBackfill(command, io) {
         cursor,
         now,
     });
-    const guardCounts = await countReadOnlyGuards(scan.candidates);
-    const reasonCounts = reasonCountsFor(scan.candidates, guardCounts);
+    for (const reason of worktreeDiscovery.incomplete_reasons) {
+        scan.issues.push({
+            reason: `repo_discovery_${reason}`,
+            count: 1,
+            scope: "global",
+        });
+    }
+    scan.issues.sort((a, b) => scanIssuePriority(a.scope) - scanIssuePriority(b.scope) ||
+        a.reason.localeCompare(b.reason));
+    const guards = await countReadOnlyGuards(scan.candidates);
+    for (const candidate of scan.candidates) {
+        const permanentReason = guards.permanent_skip_reasons.get(candidateCursorKey(candidate));
+        if (!permanentReason)
+            continue;
+        candidate.state = "skipped";
+        candidate.reason = permanentReason;
+    }
+    scan.retryable_candidate_keys = new Set([
+        ...scan.retryable_candidate_keys,
+        ...guards.retryable_candidate_keys,
+    ]);
+    if (guards.counts.get("file_read_failed")) {
+        scan.issues.push({
+            reason: "candidate_file_read_failed",
+            count: guards.counts.get("file_read_failed") ?? 0,
+            scope: "candidate",
+        });
+    }
+    if (guards.counts.get("file_too_large")) {
+        scan.issues.push({
+            reason: "file_too_large",
+            count: guards.counts.get("file_too_large") ?? 0,
+            scope: "candidate",
+        });
+    }
+    const reasonCounts = reasonCountsFor(scan.candidates, guards.counts, scan.issues);
     if (command.all && !command.yes) {
         if (!command.json) {
             writeDryRunSummary(io, {
@@ -148,8 +190,12 @@ export async function runBackfill(command, io) {
                 scan,
                 reasonCounts,
             }),
-            status: scan.candidates.length === 0 ? "blocked" : "complete",
+            status: scan.issues.length > 0 ? "partial" : "complete",
             dry_run: true,
+            retry_command: retryCommand,
+            ...(scan.issues.length > 0
+                ? { failure_reason: scan.issues[0]?.reason }
+                : {}),
         };
     }
     const lock = await acquireBackfillLock(paths, now);
@@ -165,6 +211,7 @@ export async function runBackfill(command, io) {
                 reasonCounts,
             }),
             status: "blocked",
+            retry_command: retryCommand,
             failure_reason: "backfill_already_running",
             blocked_at: {
                 what: "backfill lock held",
@@ -175,7 +222,46 @@ export async function runBackfill(command, io) {
             },
         };
     }
+    let collectionLock;
     try {
+        collectionLock = await acquireSyncLock(paths);
+    }
+    catch (error) {
+        await lock.handle.release();
+        throw error;
+    }
+    if (!collectionLock.acquired) {
+        await lock.handle.release();
+        return {
+            ...baseBackfillResult(command, {
+                now,
+                dashboardUrl,
+                sources,
+                window,
+                cursor,
+                scan,
+                reasonCounts,
+            }),
+            status: "blocked",
+            retry_command: retryCommand,
+            failure_reason: "sync_already_running",
+            blocked_at: {
+                what: "sync collection lock held",
+                batch_index: 0,
+                batch_total: 0,
+                done: 0,
+                total: scan.candidates.length,
+            },
+        };
+    }
+    try {
+        // Backfill and scheduled/manual sync share upload spool and raw-evidence
+        // cursors. Holding the same collection lock makes their read-modify-write
+        // updates serial, while the dedicated backfill lock still prevents two
+        // historical scans from running together.
+        if (scopedCursor.reset) {
+            await writeBackfillCursor(paths, cursor);
+        }
         const uploadable = uploadableCandidates(scan.candidates);
         const batches = buildBackfillBatches(uploadable);
         const rawEvidenceBudget = {
@@ -192,8 +278,9 @@ export async function runBackfill(command, io) {
         let uploadedObjects = 0;
         let uploadedChunks = 0;
         let backfilledSessions = 0;
+        const durableCandidateKeys = new Set();
         let blockedAt;
-        let failureReason;
+        let failureReason = scan.issues[0]?.reason;
         for (const [index, batch] of batches.entries()) {
             await lock.handle.heartbeat();
             const sync = await syncBackfillBatch({
@@ -209,12 +296,18 @@ export async function runBackfill(command, io) {
             done += batch.candidates.length;
             uploadedObjects += sync.raw_evidence_uploaded_object_count;
             uploadedChunks += sync.raw_evidence_uploaded_chunk_count;
-            failed += sync.raw_evidence_failed_count;
+            failed += countSessionUploadFailures(sync);
             deferred +=
                 sync.raw_evidence_deferred_byte_budget +
                     sync.raw_evidence_deferred_object_budget;
-            backfilledSessions += countBackfilledSessions(sync);
-            const batchFailed = sync.status !== "uploaded" || sync.raw_evidence_failed_count > 0;
+            const durableInBatch = durableBackfillCandidateKeys(batch, sync);
+            for (const key of durableInBatch)
+                durableCandidateKeys.add(key);
+            backfilledSessions = durableCandidateKeys.size;
+            const missingDurableMain = batch.candidates.length - durableInBatch.size;
+            const batchFailed = sync.status !== "uploaded" ||
+                countSessionUploadFailures(sync) > 0 ||
+                missingDurableMain > 0;
             const batchDeferred = sync.raw_evidence_deferred_byte_budget +
                 sync.raw_evidence_deferred_object_budget >
                 0;
@@ -227,16 +320,10 @@ export async function runBackfill(command, io) {
             }
             if (!batchFailed && !batchDeferred) {
                 completedBatches += 1;
-                recordBackfillCursorObservations(cursor, batch.candidates.map((candidate) => ({
-                    source: candidate.source,
-                    state: candidate.state,
-                    reason: candidate.reason,
-                    session_file_mtime_ms: candidate.session_file_mtime_ms,
-                    session_file_mtime: candidate.session_file_mtime,
-                })), now);
-                await writeBackfillCursor(paths, cursor);
             }
-            writeLine(io.stdout, `Uploaded ${done}/${uploadable.length} (batch ${index + 1}/${batches.length})`);
+            if (!command.json) {
+                writeLine(io.stdout, `Uploaded ${done}/${uploadable.length} (batch ${index + 1}/${batches.length})`);
+            }
             await yieldToEventLoop();
             if (batchDeferred) {
                 blockedAt = {
@@ -258,7 +345,11 @@ export async function runBackfill(command, io) {
                     total: uploadable.length,
                 };
                 failureReason =
-                    sync.status === "spooled" ? sync.failure_reason : "upload_failed";
+                    sync.status === "spooled"
+                        ? sync.failure_reason
+                        : missingDurableMain > 0
+                            ? "durable_session_pointer_missing"
+                            : "upload_failed";
                 break;
             }
         }
@@ -267,17 +358,28 @@ export async function runBackfill(command, io) {
             syncResults,
             now,
         });
+        const reportContext = sessions.length
+            ? await ensureBackfillReportContext({
+                homeDir: command.homeDir,
+                paths,
+                collectionRoots,
+                worktrees,
+                candidates: scan.candidates,
+            })
+            : null;
         const report = sessions.length
             ? await postCodexSessionReport({
                 homeDir: command.homeDir,
-                repoRoot: worktrees[0]?.repo_root,
+                repoRoot: reportContext?.repoRoot,
                 dashboardUrl,
                 sessions,
                 fetch: io.fetch,
                 now,
             })
             : emptyReport("no_sessions_observed");
-        if (sessions.length > 0 && !report.posted && !blockedAt) {
+        const reportAcknowledged = sessions.length === 0 ||
+            (report.posted && report.recorded_count >= sessions.length);
+        if (!reportAcknowledged && !blockedAt) {
             blockedAt = {
                 what: "session report failed",
                 batch_index: completedBatches,
@@ -285,20 +387,77 @@ export async function runBackfill(command, io) {
                 done,
                 total: uploadable.length,
             };
-            failureReason = report.reason;
+            failureReason ??=
+                report.posted && report.recorded_count < sessions.length
+                    ? "session_report_ack_incomplete"
+                    : report.reason;
         }
-        const remaining = Math.max(0, uploadable.length - done);
-        const status = blockedAt
-            ? remaining > 0 || deferred > 0
-                ? "partial"
-                : "blocked"
-            : scan.candidates.length === 0
-                ? "blocked"
-                : "complete";
-        if (status === "complete" && deferred === 0 && failed === 0) {
+        // Raw evidence durability is necessary but not sufficient: the server must
+        // also acknowledge the session attribution rows before their historical
+        // cursor positions become irreversible.
+        if (reportAcknowledged) {
+            const cursorAdvanced = advanceBackfillCursorThroughResolvedPrefix({
+                cursor,
+                candidates: scan.candidates,
+                durableCandidateKeys,
+                retryableCandidateKeys: scan.retryable_candidate_keys,
+                discoveryComplete: !scan.issues.some((issue) => issue.scope === "global"),
+                now,
+            });
+            if (cursorAdvanced)
+                await writeBackfillCursor(paths, cursor);
+        }
+        const unresolvedUploadableKeys = new Set(uploadable
+            .filter((candidate) => !durableCandidateKeys.has(candidateCursorKey(candidate)))
+            .map(candidateCursorKey));
+        const unresolvedUploadable = unresolvedUploadableKeys.size;
+        const unresolvedRetryable = scan.retryable_candidate_keys.size;
+        const unresolvedKnownKeys = new Set([
+            ...unresolvedUploadableKeys,
+            ...scan.retryable_candidate_keys,
+        ]);
+        const unseenGlobalFailures = scan.issues
+            .filter((issue) => issue.scope === "global")
+            .reduce((total, issue) => total + issue.count, 0);
+        const reportRetryable = reportAcknowledged
+            ? 0
+            // The cursor is intentionally all-or-nothing for the report. Even
+            // chunks already accepted by the server are retried idempotently when
+            // another required chunk lacks an acknowledgement.
+            : Math.max(1, sessions.length);
+        const remaining = unresolvedKnownKeys.size +
+            scan.omitted_candidate_count +
+            unseenGlobalFailures +
+            reportRetryable;
+        failed = Math.max(failed, unresolvedUploadable);
+        const completionBlocked = scan.issues.length > 0 ||
+            unresolvedUploadable > 0 ||
+            unresolvedRetryable > 0 ||
+            deferred > 0 ||
+            failed > 0 ||
+            !reportAcknowledged;
+        if (!failureReason && completionBlocked) {
+            const retryableCandidateReason = scan.candidates.find((candidate) => scan.retryable_candidate_keys.has(candidateCursorKey(candidate)))?.reason;
+            failureReason =
+                scan.issues[0]?.reason ??
+                    (unresolvedUploadable > 0
+                        ? "durable_session_pointer_missing"
+                        : retryableCandidateReason ?? "backfill_incomplete");
+        }
+        const status = blockedAt || completionBlocked ? "partial" : "complete";
+        // The marker is consumed by doctor/status as proof that archived history
+        // is covered. A bounded --since-days run may complete its requested
+        // window, but it is not proof of an all-history backfill.
+        if (status === "complete" && command.all) {
+            recordBackfillScanCoverage(cursor, sources, now, now);
+            await writeBackfillCursor(paths, cursor);
             await writeBackfillCompletionMarker(paths, {
-                schema_version: "cockpit-backfill-complete.v1",
+                schema_version: "cockpit-backfill-complete.v2",
+                coverage_version: BACKFILL_COVERAGE_VERSION,
+                collection_scope_id: scopedCursor.collection_scope_id,
+                sources,
                 completed_at: now.toISOString(),
+                revalidate_after: new Date(now.getTime() + BACKFILL_COMPLETION_RECHECK_MS).toISOString(),
                 cursor,
             });
         }
@@ -313,6 +472,7 @@ export async function runBackfill(command, io) {
                 reasonCounts,
             }),
             status,
+            retry_command: retryCommand,
             counts: {
                 ...baseBackfillResult(command, {
                     now,
@@ -344,6 +504,7 @@ export async function runBackfill(command, io) {
         };
     }
     finally {
+        await collectionLock.handle.release();
         await lock.handle.release();
     }
 }
@@ -354,6 +515,58 @@ function bareBackfillMessage() {
         "Use `cockpit backfill --all` only after reviewing a dry-run; on headless agent runs add `--yes`.",
     ].join("\n");
 }
+/**
+ * Produces a copyable retry that preserves the requested history window and
+ * every discovery/selection override. Values that need quoting use syntax
+ * accepted by the supported native shells: POSIX shells on macOS and
+ * PowerShell on Windows.
+ */
+export function backfillRetryCommand(command) {
+    const parts = ["cockpit", "backfill"];
+    if (command.all) {
+        parts.push("--all", "--yes");
+    }
+    else if (command.sinceDays !== undefined) {
+        parts.push("--since-days", String(command.sinceDays));
+    }
+    if (command.source)
+        parts.push("--source", command.source);
+    if (command.maxFiles !== undefined) {
+        parts.push("--max-files", String(command.maxFiles));
+    }
+    if (command.maxDepth !== undefined) {
+        parts.push("--max-depth", String(command.maxDepth));
+    }
+    if (command.maxRepos !== undefined) {
+        parts.push("--max-repos", String(command.maxRepos));
+    }
+    if (command.repoRoot) {
+        parts.push("--workspace", quoteCliArgument(command.repoRoot));
+    }
+    if (command.dryRun)
+        parts.push("--dry-run");
+    return parts.join(" ");
+}
+function backfillDiscoveryRetryCommand(command, discovery) {
+    const retry = { ...command };
+    if (discovery.incomplete_reasons.includes("max_depth_reached")) {
+        retry.maxDepth =
+            (command.maxDepth ?? DEFAULT_DISCOVERY_MAX_DEPTH) + 1;
+    }
+    if (discovery.incomplete_reasons.includes("max_worktrees_reached")) {
+        retry.maxRepos =
+            (command.maxRepos ?? DEFAULT_DISCOVERY_MAX_REPOS) * 2;
+    }
+    return backfillRetryCommand(retry);
+}
+function quoteCliArgument(value) {
+    if (/^[a-z0-9_./:\\-]+$/iu.test(value))
+        return value;
+    const escaped = process.platform === "win32"
+        ? value.replaceAll("'", "''")
+        : value.replaceAll("'", `'\"'\"'`);
+    return `'${escaped}'`;
+}
 function backfillCollectionRoots(command, savedRoots) {
     if (command.repoRoot)
         return [path.resolve(command.repoRoot)];
@@ -363,27 +576,11 @@ function backfillCollectionRoots(command, savedRoots) {
     }
     return roots;
 }
-async function discoverBackfillWorktrees(roots) {
-    const discovered = [];
-    for (const root of roots) {
-        discovered.push(...(await discoverGitWorktrees(root, {
-            maxDepth: DEFAULT_DISCOVERY_MAX_DEPTH,
-            maxWorktrees: DEFAULT_DISCOVERY_MAX_REPOS,
-        })));
-    }
-    const seen = new Set();
-    const deduped = [];
-    for (const worktree of discovered) {
-        const key = worktree.worktree_fingerprint || path.resolve(worktree.repo_root);
-        if (seen.has(key))
-            continue;
-        seen.add(key);
-        deduped.push(worktree);
-    }
-    if (deduped.length === 0) {
-        throw new Error("No git repos found in the backfill collection roots.");
-    }
-    return deduped;
+async function discoverBackfillWorktrees(roots, command) {
+    return discoverGitWorktreesInRootsWithStatus(roots, {
+        maxDepth: command.maxDepth ?? DEFAULT_DISCOVERY_MAX_DEPTH,
+        maxWorktrees: command.maxRepos ?? DEFAULT_DISCOVERY_MAX_REPOS,
+    });
 }
 function selectedSources(source) {
     if (source === "codex")
@@ -426,9 +623,11 @@ async function validateBackfillReachability(options) {
 }
 async function scanBackfillSessions(options) {
     const scanLimit = options.command.maxFiles ?? 10_000;
-    const codexAttribution = options.sources.includes("codex")
+    const codexSessionDirs = defaultCodexSessionDirs(options.homeDir);
+    const claudeProjectsDir = path.join(options.homeDir, ".claude", "projects");
+    let codexAttribution = options.sources.includes("codex")
         ? await scanAndAttributeCodexSessions({
-            sessionsDirs: defaultCodexSessionDirs(options.homeDir),
+            sessionsDirs: codexSessionDirs,
             worktrees: options.worktrees,
             now: options.now,
             sinceMinutes: options.window.since_minutes,
@@ -436,9 +635,22 @@ async function scanBackfillSessions(options) {
             collectionRoots: options.collectionRoots,
         })
         : null;
-    const claudeAttribution = options.sources.includes("claude_code")
+    if (codexAttribution?.session_limit_applied) {
+        // The adapter limit is a memory guard, not permission to silently leave
+        // history undiscovered. Re-scan the exact discovered population so a
+        // user-facing --max-files cap can resume from a truthful ordering.
+        codexAttribution = await scanAndAttributeCodexSessions({
+            sessionsDirs: codexSessionDirs,
+            worktrees: options.worktrees,
+            now: options.now,
+            sinceMinutes: options.window.since_minutes,
+            limit: Math.max(scanLimit + 1, codexAttribution.discovered_file_count),
+            collectionRoots: options.collectionRoots,
+        });
+    }
+    let claudeAttribution = options.sources.includes("claude_code")
         ? await scanAndAttributeClaudeSessions({
-            projectsDir: path.join(options.homeDir, ".claude", "projects"),
+            projectsDir: claudeProjectsDir,
             worktrees: options.worktrees,
             now: options.now,
             sinceMinutes: options.window.since_minutes,
@@ -446,14 +658,40 @@ async function scanBackfillSessions(options) {
             collectionRoots: options.collectionRoots,
         })
         : null;
-    const candidates = [
+    if (claudeAttribution?.session_limit_applied) {
+        claudeAttribution = await scanAndAttributeClaudeSessions({
+            projectsDir: claudeProjectsDir,
+            worktrees: options.worktrees,
+            now: options.now,
+            sinceMinutes: options.window.since_minutes,
+            limit: Math.max(scanLimit + 1, claudeAttribution.discovered_session_count),
+            collectionRoots: options.collectionRoots,
+        });
+    }
+    const allCandidates = [
         ...(codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
         ...(claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
     ]
         .filter((candidate) => isAfterCursor(candidate, options.cursor))
-        .sort((a, b) => b.session_file_mtime_ms - a.session_file_mtime_ms)
-        .slice(0, options.command.maxFiles ?? Number.MAX_SAFE_INTEGER);
-    return { candidates, codexAttribution, claudeAttribution };
+        .sort(compareBackfillCandidates);
+    const candidates = allCandidates.slice(0, options.command.maxFiles ?? Number.MAX_SAFE_INTEGER);
+    const omittedCandidateCount = allCandidates.length - candidates.length;
+    const issues = await backfillScanIssues({
+        codexAttribution,
+        claudeAttribution,
+        codexSessionDirs,
+        claudeProjectsDir,
+        candidates,
+        omittedCandidateCount,
+    });
+    return {
+        candidates,
+        codexAttribution,
+        claudeAttribution,
+        issues,
+        retryable_candidate_keys: retryableCandidateKeys(candidates),
+        omitted_candidate_count: omittedCandidateCount,
+    };
 }
 function normalizeCodexCandidate(result) {
     return {
@@ -495,43 +733,191 @@ function normalizeClaudeCandidate(result) {
     };
 }
 function isAfterCursor(candidate, cursor) {
-    const oldest = cursor.sources[candidate.source]?.oldest_mtime_ms_processed;
-    return oldest === null || oldest === undefined || candidate.session_file_mtime_ms < oldest;
+    const source = cursor.sources[candidate.source];
+    const oldest = source?.oldest_mtime_ms_processed;
+    if (oldest === null || oldest === undefined)
+        return true;
+    if (candidate.session_file_mtime_ms < oldest)
+        return true;
+    const key = candidateCursorKey(candidate);
+    if (candidate.session_file_mtime_ms === oldest) {
+        return !source.processed_keys_at_oldest_mtime.includes(key);
+    }
+    const newest = source.newest_mtime_ms_covered;
+    // A cursor written before upper-edge coverage existed gets one safe
+    // migration pass across the previously processed interval. Once that pass is
+    // acknowledged, subsequent --all runs only inspect genuinely newer files.
+    if (newest === null)
+        return true;
+    if (candidate.session_file_mtime_ms > newest)
+        return true;
+    if (candidate.session_file_mtime_ms < newest)
+        return false;
+    // A legacy cursor has no boundary identities. Rechecking the equal-time
+    // boundary is safe because durable evidence is content-addressed.
+    return !source.processed_keys_at_newest_mtime.includes(key);
+}
+function compareBackfillCandidates(a, b) {
+    return (b.session_file_mtime_ms - a.session_file_mtime_ms ||
+        candidateCursorKey(a).localeCompare(candidateCursorKey(b)));
+}
+function candidateCursorKey(candidate) {
+    return crypto
+        .createHash("sha256")
+        .update(JSON.stringify([
+        candidate.source,
+        candidate.session_id,
+        normalizedCursorPath(candidate.file_path),
+    ]))
+        .digest("hex");
+}
+function normalizedCursorPath(filePath) {
+    const windowsStyle = path.win32.isAbsolute(filePath) && !path.posix.isAbsolute(filePath);
+    if (windowsStyle)
+        return path.win32.normalize(filePath).toLowerCase();
+    const normalized = path.resolve(filePath);
+    return process.platform === "win32" ? normalized.toLowerCase() : normalized;
+}
+async function backfillScanIssues(options) {
+    const issues = [];
+    const add = (reason, count, scope) => {
+        if (count > 0)
+            issues.push({ reason, count, scope });
+    };
+    if (options.codexAttribution) {
+        add("codex_session_limit_applied", options.codexAttribution.session_limit_applied
+            ? Math.max(1, options.codexAttribution.discovered_file_count -
+                options.codexAttribution.scanned_file_count)
+            : 0, "global");
+        const missingTopLevelDirs = (await Promise.all(options.codexSessionDirs.map(isMissingPath))).filter(Boolean).length;
+        add("codex_directory_read_failed", Math.max(0, options.codexAttribution.directory_read_failed_count -
+            missingTopLevelDirs), "global");
+        add("codex_session_stat_failed", options.codexAttribution.stat_failed_count, "global");
+    }
+    if (options.claudeAttribution) {
+        add("claude_session_limit_applied", options.claudeAttribution.session_limit_applied
+            ? Math.max(1, options.claudeAttribution.discovered_session_count -
+                options.claudeAttribution.scanned_session_count)
+            : 0, "global");
+        const projectsDirMissing = await isMissingPath(options.claudeProjectsDir);
+        add("claude_project_dir_read_failed", Math.max(0, options.claudeAttribution.project_dir_read_failed_count -
+            (projectsDirMissing ? 1 : 0)), "global");
+        add("claude_session_stat_failed", options.claudeAttribution.session_stat_failed_count, "global");
+        add("claude_sidecar_stat_failed", options.claudeAttribution.sidecar_stat_failed_count, "global");
+        add("claude_sidecar_dir_read_failed", await countUnreadableClaudeSidecarDirs(options.candidates), "global");
+    }
+    add("backfill_max_files_applied", options.omittedCandidateCount, "selection");
+    add("candidate_file_read_failed", options.candidates.filter((candidate) => candidate.reason === "file_read_failed").length, "candidate");
+    add("candidate_worktree_unavailable", options.candidates.filter((candidate) => isRawEvidenceUploadableAttributionState(candidate.state) &&
+        candidate.worktree === null).length, "candidate");
+    add("claude_main_file_too_large", options.candidates.filter((candidate) => candidate.source === "claude_code" &&
+        candidate.claude?.main_file_oversized).length, "candidate");
+    add("claude_sidecar_limit_applied", options.candidates.reduce((total, candidate) => total + (candidate.claude?.sidecars_capped ?? 0), 0), "candidate");
+    add("claude_sidecar_file_unreadable", options.candidates.reduce((total, candidate) => total +
+        (candidate.claude?.sidecar_files.filter((sidecar) => sidecar.skipped_reason === "file_read_failed" ||
+            sidecar.skipped_reason === "file_too_large").length ?? 0), 0), "candidate");
+    return issues.sort((a, b) => scanIssuePriority(a.scope) - scanIssuePriority(b.scope) ||
+        a.reason.localeCompare(b.reason));
+}
+function scanIssuePriority(scope) {
+    if (scope === "global")
+        return 0;
+    if (scope === "candidate")
+        return 1;
+    return 2;
+}
+function retryableCandidateKeys(candidates) {
+    return new Set(candidates
+        .filter((candidate) => candidate.reason === "file_read_failed" ||
+        candidate.reason === "repo_not_on_disk" ||
+        (isRawEvidenceUploadableAttributionState(candidate.state) &&
+            !candidate.worktree) ||
+        Boolean(candidate.claude?.main_file_oversized) ||
+        (candidate.claude?.sidecars_capped ?? 0) > 0 ||
+        Boolean(candidate.claude?.sidecar_files.some((sidecar) => sidecar.skipped_reason === "file_read_failed" ||
+            sidecar.skipped_reason === "file_too_large")))
+        .map(candidateCursorKey));
+}
+async function countUnreadableClaudeSidecarDirs(candidates) {
+    let unreadable = 0;
+    for (const candidate of candidates) {
+        if (candidate.source !== "claude_code")
+            continue;
+        const subagentsDir = path.join(path.dirname(candidate.file_path), path.basename(candidate.file_path).replace(/\.jsonl$/i, ""), "subagents");
+        try {
+            await fs.readdir(subagentsDir);
+        }
+        catch (error) {
+            if (!isMissingFsError(error))
+                unreadable += 1;
+        }
+    }
+    return unreadable;
+}
+async function isMissingPath(filePath) {
+    try {
+        await fs.stat(filePath);
+        return false;
+    }
+    catch (error) {
+        return isMissingFsError(error);
+    }
+}
+function isMissingFsError(error) {
+    if (!error || typeof error !== "object")
+        return false;
+    const code = error.code;
+    return code === "ENOENT" || code === "ENOTDIR";
 }
 async function countReadOnlyGuards(candidates) {
     const counts = new Map();
+    const retryableCandidateKeys = new Set();
+    const permanentSkipReasons = new Map();
     for (const candidate of candidates) {
-        if (candidate.reason === "repo_not_on_disk")
+        if (candidate.reason === "repo_not_on_disk") {
             increment(counts, "repo_not_on_disk");
+            retryableCandidateKeys.add(candidateCursorKey(candidate));
+        }
         if (candidate.source === "claude_code" && candidate.claude?.main_file_oversized) {
             increment(counts, "file_too_large");
+            retryableCandidateKeys.add(candidateCursorKey(candidate));
             continue;
         }
         if (candidate.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
             increment(counts, "file_too_large");
+            retryableCandidateKeys.add(candidateCursorKey(candidate));
             continue;
         }
-        if (!isUploadableState(candidate.state))
+        if (!isRawEvidenceUploadableAttributionState(candidate.state))
             continue;
         try {
             const raw = await fs.readFile(candidate.file_path, "utf8");
             if (containsSecretLikeContent(raw)) {
                 increment(counts, "secret_like_content_guard");
+                permanentSkipReasons.set(candidateCursorKey(candidate), "secret_like_content_guard");
             }
         }
         catch {
             increment(counts, "file_read_failed");
+            retryableCandidateKeys.add(candidateCursorKey(candidate));
         }
     }
-    return counts;
+    return {
+        counts,
+        retryable_candidate_keys: retryableCandidateKeys,
+        permanent_skip_reasons: permanentSkipReasons,
+    };
 }
-function reasonCountsFor(candidates, guardCounts) {
+function reasonCountsFor(candidates, guardCounts, scanIssues) {
     const counts = new Map();
     for (const candidate of candidates)
         increment(counts, candidate.reason);
     for (const [reason, count] of guardCounts) {
         counts.set(reason, Math.max(counts.get(reason) ?? 0, count));
     }
+    for (const issue of scanIssues) {
+        counts.set(issue.reason, Math.max(counts.get(issue.reason) ?? 0, issue.count));
+    }
     for (const required of [
         "secret_like_content_guard",
         "file_too_large",
@@ -602,10 +988,8 @@ function reasonClassification(reason) {
     };
 }
 function uploadableCandidates(candidates) {
-    return candidates.filter((candidate) => isUploadableState(candidate.state) && candidate.worktree);
-}
-function isUploadableState(state) {
-    return state === "attributed" || state === "attributed_fallback";
+    return candidates.filter((candidate) => isRawEvidenceUploadableAttributionState(candidate.state) &&
+        candidate.worktree);
 }
 function buildBackfillBatches(candidates) {
     const byWorktree = new Map();
@@ -629,6 +1013,29 @@ function buildBackfillBatches(candidates) {
     }
     return batches;
 }
+async function ensureBackfillReportContext(options) {
+    const candidateWorktree = options.candidates.find((candidate) => candidate.worktree)?.worktree;
+    const representative = candidateWorktree ?? options.worktrees[0] ?? null;
+    // A deleted repo or an approved folder workspace can legitimately produce
+    // report-only rows with no current git worktree. Use the exact approved root
+    // as a synthetic local context instead of falling through to process.cwd().
+    const repoRoot = representative?.repo_root ??
+        options.collectionRoots[0] ??
+        process.cwd();
+    try {
+        await readLocalWorkContextForRepo(options.paths, repoRoot);
+    }
+    catch {
+        // Context creation is best-effort here. postCodexSessionReport converts a
+        // remaining local-context failure into a retryable report reason.
+        await startLocalWorkContext({
+            homeDir: options.homeDir,
+            repoRoot,
+            branch: representative?.branch,
+        }).catch(() => undefined);
+    }
+    return { repoRoot };
+}
 async function syncBackfillBatch(options) {
     const syncOptions = {
         homeDir: options.command.homeDir,
@@ -737,7 +1144,7 @@ function buildBackfillSessionReport(options) {
                     raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
                     upload_state: upload.upload_state,
                 }
-                : isUploadableState(candidate.state)
+                : isRawEvidenceUploadableAttributionState(candidate.state)
                     ? { upload_state: "not_uploaded" }
                     : {}),
         };
@@ -759,17 +1166,105 @@ function rank(state) {
             return 0;
     }
 }
-function countBackfilledSessions(sync) {
-    const keys = new Set();
+function durableBackfillCandidateKeys(batch, sync) {
+    const durable = new Set();
+    if (sync.status !== "uploaded")
+        return durable;
+    if (sync.raw_evidence_deferred_byte_budget > 0 ||
+        sync.raw_evidence_deferred_object_budget > 0) {
+        // Deferred outcomes have no per-file identity. Advancing any candidate in
+        // this batch could therefore strand the deferred main transcript.
+        return durable;
+    }
+    const outcomesBySession = new Map();
     for (const outcome of sync.raw_evidence_outcomes) {
-        if ((outcome.kind === "codex_jsonl" || outcome.kind === "claude_jsonl") &&
-            outcome.codex_session_id &&
+        const source = backfillSourceForEvidenceKind(outcome.kind);
+        if (!source || !outcome.codex_session_id)
+            continue;
+        const key = `${source}:${outcome.codex_session_id}`;
+        const summary = outcomesBySession.get(key) ?? {
+            durableMainCount: 0,
+            durableSidecarCount: 0,
+            failed: false,
+        };
+        if (outcome.upload_state === "upload_failed") {
+            summary.failed = true;
+        }
+        else if (outcome.raw_evidence_pointer_id &&
             (outcome.upload_state === "uploaded" ||
                 outcome.upload_state === "reused_existing")) {
-            keys.add(`${outcome.kind}:${outcome.codex_session_id}`);
+            if (outcome.kind === "codex_jsonl" ||
+                outcome.kind === "claude_jsonl") {
+                summary.durableMainCount += 1;
+            }
+            else if (outcome.kind === "claude_jsonl_sidecar") {
+                summary.durableSidecarCount += 1;
+            }
+        }
+        outcomesBySession.set(key, summary);
+    }
+    for (const candidate of batch.candidates) {
+        const summary = outcomesBySession.get(`${candidate.source}:${candidate.session_id}`);
+        const expectedSidecars = candidate.source === "claude_code"
+            ? (candidate.claude?.sidecar_files.filter((sidecar) => !sidecar.skipped_reason).length ?? 0)
+            : 0;
+        if (summary &&
+            !summary.failed &&
+            summary.durableMainCount > 0 &&
+            summary.durableSidecarCount >= expectedSidecars) {
+            durable.add(candidateCursorKey(candidate));
+        }
+    }
+    return durable;
+}
+function countSessionUploadFailures(sync) {
+    return sync.raw_evidence_outcomes.filter((outcome) => Boolean(outcome.codex_session_id) &&
+        outcome.upload_state === "upload_failed" &&
+        Boolean(backfillSourceForEvidenceKind(outcome.kind))).length;
+}
+function backfillSourceForEvidenceKind(kind) {
+    if (kind === "codex_jsonl" || kind === "codex_image_attachment") {
+        return "codex";
+    }
+    if (kind === "claude_jsonl" ||
+        kind === "claude_jsonl_sidecar" ||
+        kind === "claude_image_attachment") {
+        return "claude_code";
+    }
+    return null;
+}
+function advanceBackfillCursorThroughResolvedPrefix(options) {
+    if (!options.discoveryComplete)
+        return false;
+    const observations = [];
+    for (const source of ["codex", "claude_code"]) {
+        const remaining = options.candidates
+            .filter((candidate) => candidate.source === source &&
+            isAfterCursor(candidate, options.cursor))
+            .sort(compareBackfillCandidates);
+        for (const candidate of remaining) {
+            const key = candidateCursorKey(candidate);
+            if (options.retryableCandidateKeys.has(key))
+                break;
+            const resolved = isRawEvidenceUploadableAttributionState(candidate.state)
+                ? options.durableCandidateKeys.has(key)
+                : true;
+            if (!resolved)
+                break;
+            observations.push({
+                source: candidate.source,
+                cursor_key: key,
+                state: candidate.state,
+                reason: candidate.reason,
+                session_file_mtime_ms: candidate.session_file_mtime_ms,
+                session_file_mtime: candidate.session_file_mtime,
+            });
         }
     }
-    return keys.size;
+    if (observations.length === 0)
+        return false;
+    recordBackfillCursorObservations(options.cursor, observations, options.now);
+    return true;
 }
 function worktreeInventoryForRepo(current, worktrees) {
     return worktrees
@@ -820,7 +1315,7 @@ function baseBackfillResult(command, options) {
             failed: 0,
         },
         resume_cursor: options.cursor,
-        retry_command: RETRY_COMMAND,
+        retry_command: backfillRetryCommand(command),
         report: emptyReport("not_posted"),
         server_acknowledged: {
             codex_session_report_recorded_count: 0,
@@ -860,7 +1355,7 @@ function blockedBackfillResult(command, options) {
         },
         batches: { total: 0, completed: 0, failed: 0 },
         resume_cursor: options.cursor,
-        retry_command: RETRY_COMMAND,
+        retry_command: backfillRetryCommand(command),
         report: emptyReport("not_posted"),
         server_acknowledged: {
             codex_session_report_recorded_count: 0,