@bli-cockpit/cli 0.2.26 → 0.2.28

package/dist/upload.js

@@ -5,6 +5,7 @@ import { runLocalSourceCollectors } from "./adapters/local-sources.js";
 import { defaultCodexSessionDirs, } from "./adapters/codex-attribution.js";
 import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
 import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
+import { clearDeliveryAttempt, DELIVERY_BACKOFF_HOLDING_REASON, deliveryHold, readRawEvidenceStagingState, recordDeliveryFailure, summarizeStuckEvidence, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
 import { readLocalUploadSpoolState, recordPendingSessionReport, recordSessionReportFailure, recordSessionReportSuccess, recordUploadBlocked, recordUploadFailure, recordUploadSuccess, } from "./spool/local-spool.js";
 export class LocalUploadBlockedError extends Error {
     blocker;
@@ -150,18 +151,37 @@ export async function syncLocalAmbientEnvelope(options = {}) {
     }
     let uploadOutcomes = [];
     let uploadedChunkCount = 0;
+    const attemptedDate = options.now ?? new Date(attemptedAt);
+    const staging = await readRawEvidenceStagingState(paths.state_dir);
     try {
         if (built.raw_evidence_upload_files.length > 0 && provenance) {
-            const upload = await uploadRawEvidenceFilesChunked({
-                fetchImpl,
-                dashboardUrl: built.dashboard_url,
-                deviceToken: built.device_token,
-                provenance,
-                generatedAt: built.envelope.generated_at,
-                files: built.raw_evidence_upload_files,
-            });
-            uploadOutcomes = upload.outcomes;
-            uploadedChunkCount = upload.uploaded_chunk_count;
+            // Objects still inside their backoff window are not offered at all. They
+            // become held outcomes below, which keeps them out of the envelope, keeps
+            // the sync in retry_pending, and names the reason — the 559th identical
+            // attempt is what BLI-3066 made impossible.
+            const { deliverable, held } = partitionHeldEvidenceFiles(built.raw_evidence_upload_files, staging, attemptedDate);
+            if (held.length > 0) {
+                console.error("[cockpit-sync] raw evidence held by delivery backoff", JSON.stringify({
+                    attempted_at: attemptedAt,
+                    reason: DELIVERY_BACKOFF_HOLDING_REASON,
+                    object_count: held.length,
+                    byte_size: held.reduce((sum, outcome) => sum + (outcome.pointer.byte_size ?? 0), 0),
+                }));
+            }
+            uploadOutcomes = held;
+            if (deliverable.length > 0) {
+                const upload = await uploadRawEvidenceFilesChunked({
+                    fetchImpl,
+                    dashboardUrl: built.dashboard_url,
+                    deviceToken: built.device_token,
+                    provenance,
+                    generatedAt: built.envelope.generated_at,
+                    files: deliverable,
+                });
+                uploadOutcomes = [...held, ...upload.outcomes];
+                uploadedChunkCount = upload.uploaded_chunk_count;
+            }
+            await persistDeliveryAttempts(paths.state_dir, staging, uploadOutcomes, attemptedDate);
         }
         const envelope = applyRawEvidenceUploadOutcomes(built.envelope, uploadOutcomes);
         const response = await fetchImpl(`${built.dashboard_url}/api/ambient/ingest`, {
@@ -267,7 +287,7 @@ export async function syncLocalAmbientEnvelope(options = {}) {
             source_scan_count: built.source_scan_count,
             risk_flag_count: built.risk_flag_count,
             http_status: response.status,
-            ...rawEvidenceSummary(built, uploadOutcomes, uploadedChunkCount, cursor),
+            ...rawEvidenceSummary(built, uploadOutcomes, uploadedChunkCount, cursor, staging, attemptedDate),
         };
     }
     catch (error) {
@@ -304,10 +324,98 @@ export async function syncLocalAmbientEnvelope(options = {}) {
             failure_reason: spooledFailureReason,
             spool_entry_id: entry.spool_id,
             retry_command: entry.retry_command,
-            ...rawEvidenceSummary(built, uploadOutcomes, uploadedChunkCount, cursor),
+            ...rawEvidenceSummary(built, uploadOutcomes, uploadedChunkCount, cursor, staging, attemptedDate),
         };
     }
 }
+/**
+ * Split the pack's files into "offer these now" and "still in backoff".
+ *
+ * A held file becomes an `upload_failed` outcome labelled
+ * `delivery_backoff_holding`. That is deliberate rather than a quiet omission:
+ * the pointer gets pruned from the envelope (the object is genuinely not
+ * durable), the sync stays in `retry_pending`, and the reason travels to the
+ * status output. A hold that read as success would be the green-status-hiding-
+ * missing-collection failure the fleet contract forbids.
+ */
+export function partitionHeldEvidenceFiles(files, staging, now) {
+    const deliverable = [];
+    const held = [];
+    for (const file of files) {
+        const hold = deliveryHold(staging, file.pointer.content_hash_sha256, now);
+        if (!hold) {
+            deliverable.push(file);
+            continue;
+        }
+        held.push({
+            pointer: file.pointer,
+            object_key: file.pointer.object_key ?? "",
+            codex_session_id: file.codex_session_id ?? null,
+            kind: file.kind ?? "raw_evidence",
+            ...(file.artifact_metadata
+                ? { artifact_metadata: file.artifact_metadata }
+                : {}),
+            upload_state: "upload_failed",
+            reason: DELIVERY_BACKOFF_HOLDING_REASON,
+            uploaded_chunk_count: 0,
+        });
+    }
+    return { deliverable, held };
+}
+/**
+ * Count what just happened to each object, and when it may be offered again.
+ *
+ * Written immediately after the upload pass and before ingest, so an ingest
+ * failure cannot lose the attempt counts — losing them resets every backoff to
+ * zero and the fleet is back to 15-minute retries forever. A held outcome is
+ * not itself an attempt: counting it would push its own next attempt further
+ * out on every sync and eventually never retry at all.
+ */
+async function persistDeliveryAttempts(stateDir, staging, outcomes, attemptedAt) {
+    let changed = false;
+    for (const outcome of outcomes) {
+        const contentHash = outcome.pointer.content_hash_sha256;
+        if (!contentHash)
+            continue;
+        if (outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON)
+            continue;
+        if (outcome.upload_state === "upload_failed") {
+            const entry = recordDeliveryFailure(staging, contentHash, {
+                reason: outcome.reason ?? "upload_failed",
+                attemptedAt,
+                byteSize: outcome.pointer.byte_size ?? 0,
+            });
+            changed = true;
+            console.error("[cockpit-sync] raw evidence delivery failed", JSON.stringify({
+                reason: entry.last_reason,
+                kind: outcome.kind,
+                attempts: entry.attempts,
+                first_failed_at: entry.first_failed_at,
+                next_attempt_at: entry.next_attempt_at,
+                byte_size: entry.byte_size,
+            }));
+            continue;
+        }
+        if (clearDeliveryAttempt(staging, contentHash)) {
+            changed = true;
+            console.error("[cockpit-sync] raw evidence delivery recovered", JSON.stringify({
+                reason: "delivery_recovered",
+                kind: outcome.kind,
+                upload_state: outcome.upload_state,
+            }));
+        }
+    }
+    if (!changed)
+        return;
+    staging.updated_at = attemptedAt.toISOString();
+    await writeRawEvidenceStagingState(stateDir, staging).catch((error) => {
+        console.error("[cockpit-sync] delivery attempt state write failed", JSON.stringify({
+            reason: "staging_state_write_failed",
+            detail: error instanceof Error ? error.name : typeof error,
+            tracked_count: Object.keys(staging.delivery_attempts).length,
+        }));
+    });
+}
 function retrySourcesForFailedSync(options, facts) {
     const sources = new Set();
     if ((options.codexSessionFiles?.length ?? 0) > 0 ||
@@ -372,7 +480,20 @@ function hasRetryableEvidenceGap(facts, outcomes) {
         facts.evidence_completeness.failure_reasons.length > 0) {
         return true;
     }
-    return facts.evidence_completeness.skip_reasons.some(({ reason }) => /(?:read|stat|directory)_failed|session_limit_overflow/iu.test(reason));
+    return facts.evidence_completeness.skip_reasons.some(({ reason }) => isRetryableEvidenceSkipReason(reason));
+}
+/**
+ * Skip reasons a later sync can still turn into evidence.
+ *
+ * `delivery_backoff_holding` is one of them, and it has to be: a source held by
+ * backoff is missing collection right now. If it did not land here the sync
+ * would read clean while a transcript sat undelivered for nine days, which is
+ * the exact shape of BLI-3066.
+ */
+function isRetryableEvidenceSkipReason(reason) {
+    if (reason === DELIVERY_BACKOFF_HOLDING_REASON)
+        return true;
+    return /(?:read|stat|directory)_failed|session_limit_overflow/iu.test(reason);
 }
 function retryableEvidenceGapReason(facts, outcomes) {
     const reasons = new Set();
@@ -393,7 +514,7 @@ function retryableEvidenceGapReason(facts, outcomes) {
             reasons.add("evidence_completeness_failed");
         }
         for (const { reason } of facts.evidence_completeness.skip_reasons) {
-            if (/(?:read|stat|directory)_failed|session_limit_overflow/iu.test(reason)) {
+            if (isRetryableEvidenceSkipReason(reason)) {
                 reasons.add(reason);
             }
         }
@@ -861,7 +982,8 @@ function agentArtifactFromMetadata(options) {
         },
     };
 }
-function rawEvidenceSummary(built, outcomes, uploadedChunkCount, cursor) {
+function rawEvidenceSummary(built, outcomes, uploadedChunkCount, cursor, staging, now) {
+    const stuck = summarizeStuckEvidence(staging, now);
     const cursorReused = built.raw_evidence_facts?.reused ?? [];
     const serverReusedCount = outcomes.filter((outcome) => outcome.upload_state === "reused_existing").length;
     const failed = outcomes.filter((outcome) => outcome.upload_state === "upload_failed");
@@ -914,6 +1036,10 @@ function rawEvidenceSummary(built, outcomes, uploadedChunkCount, cursor) {
         raw_evidence_deferred_byte_budget: built.raw_evidence_facts?.deferred_byte_budget_count ?? 0,
         raw_evidence_deferred_object_budget: built.raw_evidence_facts?.deferred_object_budget_count ?? 0,
         cursor_tracked_object_count: Object.keys(cursor.objects).length,
+        raw_evidence_delivery_held_count: outcomes.filter((outcome) => outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON).length + (built.raw_evidence_facts?.delivery_held_count ?? 0),
+        raw_evidence_stuck_object_count: stuck.stuck_object_count,
+        raw_evidence_max_delivery_attempts: stuck.max_attempts,
+        raw_evidence_oldest_delivery_failure_at: stuck.oldest_first_failed_at,
     };
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.26",
+  "version": "0.2.28",
   "private": false,
   "type": "module",
   "bin": {