@bli-cockpit/cli 0.2.5 → 0.2.7

package/dist/upload.js

@@ -5,7 +5,7 @@ import { runLocalSourceCollectors } from "./adapters/local-sources.js";
 import { defaultCodexSessionDirs, } from "./adapters/codex-attribution.js";
 import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
 import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
-import { recordUploadBlocked, recordUploadFailure, recordUploadSuccess, } from "./spool/local-spool.js";
+import { readLocalUploadSpoolState, recordPendingSessionReport, recordSessionReportFailure, recordSessionReportSuccess, recordUploadBlocked, recordUploadFailure, recordUploadSuccess, } from "./spool/local-spool.js";
 export class LocalUploadBlockedError extends Error {
     blocker;
     retry_hint;
@@ -20,7 +20,7 @@ export async function buildLocalAmbientEnvelope(options = {}) {
     const now = options.now ?? new Date();
     const paths = getCollectorRuntimePaths(options.homeDir);
     const config = await readLocalCollectorConfig(paths).catch(() => {
-        throw new LocalUploadBlockedError("not_installed", "Local collector config missing. Install/update the CLI, then run `cockpit onboard` before `cockpit sync`.", "npm install -g @bli-cockpit/cli@latest && cockpit onboard --email <email> --workspace \"$PWD\"");
+        throw new LocalUploadBlockedError("not_installed", "Local collector config missing. Install/update the CLI, then run `cockpit do-everything` before `cockpit sync`.", "npm install -g @bli-cockpit/cli@latest && cockpit do-everything");
     });
     const sessionFile = await readLocalCollectorSessionFile(paths).catch(() => {
         throw new LocalUploadBlockedError("unpaired", "No paired collector session found. Run `cockpit login` or `cockpit pair` before `cockpit sync`.", "cockpit login");
@@ -150,7 +150,6 @@ export async function syncLocalAmbientEnvelope(options = {}) {
     }
     let uploadOutcomes = [];
     let uploadedChunkCount = 0;
-    let ingestAccepted = false;
     try {
         if (built.raw_evidence_upload_files.length > 0 && provenance) {
             const upload = await uploadRawEvidenceFilesChunked({
@@ -177,7 +176,11 @@ export async function syncLocalAmbientEnvelope(options = {}) {
         if (!response.ok) {
             throw new Error(responseErrorMessage(responseBody, `Ambient ingest failed with HTTP ${response.status}`));
         }
-        ingestAccepted = true;
+        // A 2xx response may have persisted the envelope even when a proxy or
+        // incompatible dashboard mangles the receipt. Keep uploaded objects in
+        // that ambiguous case, but do not advance cursors or clear retry state
+        // until the first-party route proves the exact submitted row counts.
+        assertAmbientIngestReceipt(response, responseBody, envelope);
         for (const outcome of uploadOutcomes) {
             if (outcome.upload_state === "upload_failed")
                 continue;
@@ -209,7 +212,30 @@ export async function syncLocalAmbientEnvelope(options = {}) {
             reason: "agent_artifact_report_local_error",
             recorded_count: 0,
         }));
-        await recordUploadSuccess(paths, { attemptedAt });
+        const retryableEvidenceGap = hasRetryableEvidenceGap(built.raw_evidence_facts, uploadOutcomes);
+        const retrySources = retrySourcesForFailedSync(options, built.raw_evidence_facts);
+        await recordUploadSuccess(paths, {
+            attemptedAt,
+            workContextId: built.envelope.work_context.work_context_id,
+            clearPendingForContext: !retryableEvidenceGap,
+        });
+        if (retryableEvidenceGap) {
+            await recordUploadFailure(paths, {
+                last_attempt_at: attemptedAt,
+                dashboard_url: built.dashboard_url,
+                work_context_id: built.envelope.work_context.work_context_id,
+                ticket_id: built.ticket_id,
+                repo_label: built.repo_label,
+                branch: built.envelope.work_context.branch,
+                event_count: built.event_count,
+                source_scan_count: built.source_scan_count,
+                risk_flag_count: built.risk_flag_count,
+                raw_evidence_file_count: built.raw_evidence_upload_files.length,
+                retry_sources: retrySources,
+                failure_reason: retryableEvidenceGapReason(built.raw_evidence_facts, uploadOutcomes),
+                retry_command: "cockpit sync",
+            });
+        }
         return {
             status: "uploaded",
             dashboard_url: built.dashboard_url,
@@ -225,23 +251,11 @@ export async function syncLocalAmbientEnvelope(options = {}) {
     }
     catch (error) {
         const failureReason = error instanceof Error ? error.message : String(error);
-        // Once ingest accepted the envelope, the evidence refs are indexed and the
-        // objects are load-bearing — a later local write failure (cursor, spool)
-        // must not delete durable evidence.
-        const cleanupFailureReason = ingestAccepted
-            ? null
-            : await cleanupUploadedRawEvidenceObjects({
-                fetchImpl,
-                dashboardUrl: built.dashboard_url,
-                deviceToken: built.device_token,
-                envelope: built.envelope,
-                objectKeys: uploadOutcomes
-                    .filter((outcome) => outcome.upload_state === "uploaded")
-                    .map((outcome) => outcome.object_key),
-            });
-        const spooledFailureReason = cleanupFailureReason
-            ? `${failureReason}; raw evidence cleanup failed: ${cleanupFailureReason}`
-            : failureReason;
+        // A committed content-addressed object may be shared by concurrent syncs.
+        // Never delete it from this error path: an ingest retry can reuse it, while
+        // client-side cleanup cannot prove exclusive ownership without racing a
+        // second sync that is about to index the same object.
+        const spooledFailureReason = failureReason;
         const entry = await recordUploadFailure(paths, {
             last_attempt_at: attemptedAt,
             dashboard_url: built.dashboard_url,
@@ -253,6 +267,7 @@ export async function syncLocalAmbientEnvelope(options = {}) {
             source_scan_count: built.source_scan_count,
             risk_flag_count: built.risk_flag_count,
             raw_evidence_file_count: built.raw_evidence_upload_files.length,
+            retry_sources: retrySourcesForFailedSync(options, built.raw_evidence_facts),
             failure_reason: spooledFailureReason,
             retry_command: "cockpit sync",
         });
@@ -272,6 +287,73 @@ export async function syncLocalAmbientEnvelope(options = {}) {
         };
     }
 }
+function retrySourcesForFailedSync(options, facts) {
+    const sources = new Set();
+    if ((options.codexSessionFiles?.length ?? 0) > 0 ||
+        (options.codexAttributionScan?.directory_read_failed_count ?? 0) > 0 ||
+        (options.codexAttributionScan?.stat_failed_count ?? 0) > 0 ||
+        facts?.evidence_completeness.source_counts.some((count) => count.source.startsWith("codex_") &&
+            count.scanned_count + count.included_count + count.reused_count > 0)) {
+        sources.add("codex");
+    }
+    if ((options.claudeSessionFiles?.length ?? 0) > 0 ||
+        (options.claudeAttributionScan?.project_dir_read_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.session_stat_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.sidecar_dir_read_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.sidecar_stat_failed_count ?? 0) > 0 ||
+        facts?.evidence_completeness.source_counts.some((count) => count.source.startsWith("claude_") &&
+            count.scanned_count + count.included_count + count.reused_count > 0)) {
+        sources.add("claude_code");
+    }
+    return [...sources];
+}
+function hasRetryableEvidenceGap(facts, outcomes) {
+    if (outcomes.some((outcome) => outcome.upload_state === "upload_failed")) {
+        return true;
+    }
+    if (!facts)
+        return false;
+    if (facts.deferred_byte_budget_count > 0 ||
+        facts.deferred_object_budget_count > 0) {
+        return true;
+    }
+    if (facts.evidence_completeness.status === "failed" ||
+        facts.evidence_completeness.totals.failed_count > 0 ||
+        facts.evidence_completeness.failure_reasons.length > 0) {
+        return true;
+    }
+    return facts.evidence_completeness.skip_reasons.some(({ reason }) => /(?:read|stat|directory)_failed|session_limit_overflow/iu.test(reason));
+}
+function retryableEvidenceGapReason(facts, outcomes) {
+    const reasons = new Set();
+    for (const outcome of outcomes) {
+        if (outcome.upload_state === "upload_failed") {
+            reasons.add(outcome.reason ?? "upload_failed");
+        }
+    }
+    if (facts) {
+        if (facts.deferred_byte_budget_count > 0)
+            reasons.add("deferred_byte_budget");
+        if (facts.deferred_object_budget_count > 0) {
+            reasons.add("deferred_object_budget");
+        }
+        for (const { reason } of facts.evidence_completeness.failure_reasons) {
+            reasons.add(reason);
+        }
+        if (facts.evidence_completeness.status === "failed" &&
+            facts.evidence_completeness.failure_reasons.length === 0) {
+            reasons.add("evidence_completeness_failed");
+        }
+        for (const { reason } of facts.evidence_completeness.skip_reasons) {
+            if (/(?:read|stat|directory)_failed|session_limit_overflow/iu.test(reason)) {
+                reasons.add(reason);
+            }
+        }
+    }
+    return `partial_raw_evidence_retry_required:${[
+        ...reasons,
+    ].sort().join(",") || "unknown"}`;
+}
 /**
  * Posts a Codex session attribution report using the paired collector
  * credentials and the work context of a representative repo. Failures come
@@ -319,6 +401,141 @@ export async function postCodexSessionReport(options) {
         return emptyCodexSessionReportResult("collector_not_ready");
     }
 }
+/**
+ * Queues the safe session-attribution rows before the network request. The
+ * queue is merged per work context, so an endpoint outage retains every
+ * observed session without growing one duplicate report per scheduler pass.
+ */
+export async function queueCodexSessionReport(options) {
+    if (options.sessions.length === 0)
+        return null;
+    const paths = getCollectorRuntimePaths(options.homeDir);
+    return await recordPendingSessionReport(paths, {
+        attempted_at: options.generatedAt,
+        dashboard_url: normalizeDashboardUrl(options.dashboardUrl),
+        generated_at: options.generatedAt,
+        work_context_id: options.workContextId,
+        repo_label: safeRepoLabel(options.repoLabel),
+        branch: options.branch,
+        repo_fingerprint: options.repoFingerprint,
+        repo_origin_url: options.repoOriginUrl,
+        worktree_label: options.worktreeLabel,
+        worktree_fingerprint: options.worktreeFingerprint,
+        worktree_is_primary: options.worktreeIsPrimary,
+        sessions: options.sessions,
+    });
+}
+/**
+ * Flushes every durable session-attribution report using current credentials.
+ * Report payloads do not depend on the source files still being inside the live
+ * scan window, so a transient endpoint failure cannot silently age them out.
+ */
+export async function flushPendingCodexSessionReports(options) {
+    const paths = getCollectorRuntimePaths(options.homeDir);
+    const state = await readLocalUploadSpoolState(paths);
+    if (state.pending_session_reports.length === 0) {
+        return emptyCodexSessionReportResult("no_pending_session_reports");
+    }
+    const attemptedAt = (options.now ?? new Date()).toISOString();
+    const fetchImpl = options.fetch ?? globalThis.fetch;
+    if (!fetchImpl) {
+        throw new Error("global fetch is unavailable; use Node.js 20 or newer.");
+    }
+    let sessionFile;
+    let session;
+    try {
+        [sessionFile, session] = await Promise.all([
+            readLocalCollectorSessionFile(paths),
+            readLocalSessionReference(paths),
+        ]);
+    }
+    catch {
+        return await failPendingSessionReports(paths, state.pending_session_reports, attemptedAt, "collector_not_ready");
+    }
+    if (session.session_state !== "valid") {
+        return await failPendingSessionReports(paths, state.pending_session_reports, attemptedAt, "collector_not_paired");
+    }
+    const results = [];
+    for (const pending of state.pending_session_reports) {
+        const provenance = {
+            capture_source: "collector_runtime",
+            capture_adapter_version: LOCAL_COLLECTOR_VERSION,
+            collector_version: LOCAL_COLLECTOR_VERSION,
+            repo: pending.repo_label,
+            branch: pending.branch,
+            repo_label: pending.repo_label,
+            repo_fingerprint: pending.repo_fingerprint,
+            ...(pending.repo_origin_url
+                ? { repo_origin_url: pending.repo_origin_url }
+                : {}),
+            worktree_label: pending.worktree_label,
+            worktree_fingerprint: pending.worktree_fingerprint,
+            worktree_is_primary: pending.worktree_is_primary,
+            operator_id: session.operator_id,
+            session_id: session.session_id,
+            work_context_id: pending.work_context_id,
+        };
+        const result = await reportCodexSessionAttributions({
+            fetchImpl,
+            dashboardUrl: normalizeDashboardUrl(pending.dashboard_url || sessionFile.dashboard_url),
+            deviceToken: sessionFile.device_token,
+            provenance,
+            generatedAt: pending.generated_at,
+            sessions: pending.sessions,
+            maxAttemptsPerRequest: options.maxAttemptsPerRequest,
+            sleep: options.sleep,
+        });
+        results.push(result);
+        if (result.posted) {
+            await recordSessionReportSuccess(paths, {
+                reportId: pending.report_id,
+                attemptedAt,
+            });
+        }
+        else {
+            await recordSessionReportFailure(paths, {
+                reportId: pending.report_id,
+                attemptedAt,
+                reason: result.reason,
+            });
+        }
+    }
+    return combineCodexSessionReportResults(results);
+}
+async function failPendingSessionReports(paths, pendingReports, attemptedAt, reason) {
+    for (const pending of pendingReports) {
+        await recordSessionReportFailure(paths, {
+            reportId: pending.report_id,
+            attemptedAt,
+            reason,
+        });
+    }
+    return {
+        posted: false,
+        reason,
+        chunk_count: 0,
+        recorded_count: 0,
+        failed_count: pendingReports.length,
+        chunks: [],
+    };
+}
+function combineCodexSessionReportResults(results) {
+    if (results.length === 0) {
+        return emptyCodexSessionReportResult("no_pending_session_reports");
+    }
+    const failed = results.filter((result) => !result.posted);
+    return {
+        posted: failed.length === 0,
+        reason: failed[0]?.reason ??
+            (results.every((result) => result.reason === "recorded")
+                ? "recorded"
+                : results[0]?.reason ?? "recorded"),
+        chunk_count: results.reduce((total, result) => total + result.chunk_count, 0),
+        recorded_count: results.reduce((total, result) => total + result.recorded_count, 0),
+        failed_count: results.reduce((total, result) => total + result.failed_count, 0),
+        chunks: results.flatMap((result) => result.chunks),
+    };
+}
 /**
  * Reports Codex session attribution outcomes after sync. Non-fatal by design:
  * older dashboards without the endpoint must not fail the harvest, so the
@@ -355,6 +572,7 @@ async function postCodexSessionAttributionChunk(options) {
     const maxAttempts = options.maxAttemptsPerRequest ?? 3;
     const sleep = options.sleep ?? defaultReportSleep;
     let lastStatus = null;
+    let lastFailureReason = null;
     for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
         try {
             const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/codex-sessions`, {
@@ -382,14 +600,35 @@ async function postCodexSessionAttributionChunk(options) {
             }
             if (response.ok) {
                 const parsed = CodexSessionAttributionReportResponseSchema.safeParse(body);
-                return codexSessionReportChunkResult(options, {
-                    posted: true,
-                    reason: "recorded",
-                    httpStatus: response.status,
-                    recordedCount: parsed.success ? parsed.data.recorded_count : 0,
-                });
+                if (!parsed.success) {
+                    lastFailureReason = "report_invalid_response";
+                }
+                else if (parsed.data.recorded_count <
+                    requiredCodexSessionAcknowledgementCount(options.sessions)) {
+                    lastFailureReason = "report_incomplete_acknowledgement";
+                }
+                else {
+                    return codexSessionReportChunkResult(options, {
+                        posted: true,
+                        reason: "recorded",
+                        httpStatus: response.status,
+                        recordedCount: parsed.data.recorded_count,
+                    });
+                }
+            }
+            else {
+                lastFailureReason = `report_failed_http_${response.status}`;
             }
             if (response.status < 500 && response.status !== 429) {
+                if (response.ok) {
+                    // A malformed or short 2xx response is retryable. The server may have
+                    // failed between persisting the rows and producing its acknowledgement,
+                    // so only a schema-valid, sufficiently large acknowledgement is safe
+                    // to use for deleting a durable pending report.
+                    if (attempt < maxAttempts)
+                        await sleep(250 * attempt);
+                    continue;
+                }
                 return codexSessionReportChunkResult(options, {
                     posted: false,
                     reason: `report_failed_http_${response.status}`,
@@ -400,15 +639,17 @@ async function postCodexSessionAttributionChunk(options) {
         }
         catch {
             lastStatus = null;
+            lastFailureReason = "report_network_error";
         }
         if (attempt < maxAttempts)
             await sleep(250 * attempt);
     }
     return codexSessionReportChunkResult(options, {
         posted: false,
-        reason: lastStatus === null
-            ? "report_network_error"
-            : `report_failed_http_${lastStatus}`,
+        reason: lastFailureReason ??
+            (lastStatus === null
+                ? "report_network_error"
+                : `report_failed_http_${lastStatus}`),
         httpStatus: lastStatus,
         recordedCount: 0,
     });
@@ -440,6 +681,9 @@ function chunkArray(items, size) {
     }
     return chunks;
 }
+function requiredCodexSessionAcknowledgementCount(sessions) {
+    return new Set(sessions.map((session) => `${session.source ?? "codex"}:${session.codex_session_id}`)).size;
+}
 function defaultReportSleep(milliseconds) {
     return new Promise((resolve) => setTimeout(resolve, milliseconds));
 }
@@ -573,6 +817,10 @@ function rawEvidenceSummary(built, outcomes, uploadedChunkCount, cursor) {
     const cursorReused = built.raw_evidence_facts?.reused ?? [];
     const serverReusedCount = outcomes.filter((outcome) => outcome.upload_state === "reused_existing").length;
     const failed = outcomes.filter((outcome) => outcome.upload_state === "upload_failed");
+    const retryRequired = hasRetryableEvidenceGap(built.raw_evidence_facts, outcomes);
+    const retryReason = retryRequired
+        ? retryableEvidenceGapReason(built.raw_evidence_facts, outcomes)
+        : null;
     const cursorReusedOutcomes = cursorReused.map((entry) => ({
         object_key: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
         raw_evidence_pointer_id: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
@@ -594,6 +842,13 @@ function rawEvidenceSummary(built, outcomes, uploadedChunkCount, cursor) {
         raw_evidence_failure_reasons: [
             ...new Set(failed.map((outcome) => outcome.reason ?? "unknown")),
         ],
+        raw_evidence_retry_required: retryRequired,
+        raw_evidence_retry_reasons: retryReason
+            ? retryReason
+                .replace(/^partial_raw_evidence_retry_required:/u, "")
+                .split(",")
+                .filter(Boolean)
+            : [],
         raw_evidence_outcomes: [
             ...outcomes.map((outcome) => ({
                 object_key: outcome.object_key,
@@ -864,41 +1119,6 @@ function makeSourceScanCompletedEvent(options) {
         raw_evidence_pointers: rawEvidencePointers,
     });
 }
-const CLEANUP_BATCH_SIZE = 25;
-async function cleanupUploadedRawEvidenceObjects(options) {
-    if (options.objectKeys.length === 0)
-        return null;
-    const provenance = options.envelope.work_context.provenance;
-    if (!provenance)
-        return "missing collector provenance";
-    const failures = [];
-    for (let offset = 0; offset < options.objectKeys.length; offset += CLEANUP_BATCH_SIZE) {
-        const batch = options.objectKeys.slice(offset, offset + CLEANUP_BATCH_SIZE);
-        try {
-            const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/evidence/upload`, {
-                method: "DELETE",
-                headers: {
-                    "Authorization": `Bearer ${options.deviceToken}`,
-                    "Content-Type": "application/json",
-                },
-                body: JSON.stringify({
-                    schema_version: "ambient-raw-evidence-cleanup.v1",
-                    generated_at: options.envelope.generated_at,
-                    provenance,
-                    object_keys: batch,
-                }),
-            });
-            const responseBody = await readResponseJson(response);
-            if (!response.ok) {
-                failures.push(responseErrorMessage(responseBody, `Raw evidence cleanup failed with HTTP ${response.status}`));
-            }
-        }
-        catch (error) {
-            failures.push(error instanceof Error ? error.message : String(error));
-        }
-    }
-    return failures.length > 0 ? failures.join("; ") : null;
-}
 function makeCollectorProvenance(options) {
     return {
         capture_source: "collector_runtime",
@@ -985,6 +1205,29 @@ function responseErrorMessage(value, fallback) {
     }
     return fallback;
 }
+function assertAmbientIngestReceipt(response, value, envelope) {
+    if (response.status !== 202 || !value || typeof value !== "object") {
+        throw new Error("Ambient ingest returned an invalid durable receipt.");
+    }
+    const record = value;
+    const ingest = record["ingest"] && typeof record["ingest"] === "object"
+        ? record["ingest"]
+        : null;
+    const expectedFactCount = envelope.events.length;
+    const expectedRiskFlagCount = envelope.events.reduce((total, event) => total + event.risk_flags.length, 0);
+    const expectedEvidenceRefCount = envelope.events.reduce((total, event) => total + event.raw_evidence_pointers.length, 0);
+    if (record["ok"] !== true ||
+        !ingest ||
+        !isNonEmptyString(ingest["work_session_id"]) ||
+        ingest["fact_count"] !== expectedFactCount ||
+        ingest["risk_flag_count"] !== expectedRiskFlagCount ||
+        ingest["evidence_ref_count"] !== expectedEvidenceRefCount) {
+        throw new Error("Ambient ingest returned an incomplete durable receipt.");
+    }
+}
+function isNonEmptyString(value) {
+    return typeof value === "string" && value.trim().length > 0;
+}
 function normalizeDashboardUrl(value) {
     const normalized = value.trim().replace(/\/+$/, "");
     if (!normalized)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.5",
+  "version": "0.2.7",
   "private": false,
   "type": "module",
   "bin": {
@@ -26,6 +26,6 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/telemetry-core": "0.1.11"
+    "@bli-cockpit/telemetry-core": "0.1.13"
   }
 }