@bli-cockpit/cli 0.2.56 → 0.2.58

package/dist/upload-sync.js

@@ -0,0 +1,291 @@
+/**
+ * `syncLocalAmbientEnvelope`: one ambient sync, end to end.
+ *
+ * 1. Assemble the envelope — or record why this machine cannot upload at all
+ *    and rethrow, so `cockpit status` names the blocker.
+ * 2. Deliver the raw evidence objects, skipping anything inside its delivery
+ *    backoff window (`upload-evidence-delivery.ts`).
+ * 3. Reconcile the envelope with what actually became durable, POST it to
+ *    ingest, and refuse to believe a receipt we cannot read
+ *    (`upload-ingest-receipt.ts`).
+ * 4. Advance the content cursor, report any agent images, and write the ledger
+ *    row — a retry when a later sync could still rescue the gap, a named
+ *    blocker when it never can, nothing extra when the sync was clean.
+ * 5. On any failure, spool a retry that carries its own reason.
+ *
+ * The whole file is built around one rule from the fleet contract: a sync that
+ * did not collect everything must never read green. Objects held by backoff are
+ * reported as named failures, permanent rejections are logged and blocked
+ * rather than silently retried forever, and the ledger row always says why.
+ */
+import { getCollectorRuntimePaths } from "./local-state.js";
+import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
+import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
+import { readRawEvidenceStagingState } from "./raw-evidence-staging.js";
+import { recordUploadBlocked, recordUploadFailure, recordUploadSuccess, } from "./spool/local-spool.js";
+import { buildLocalAmbientEnvelope, LocalUploadBlockedError, } from "./upload-envelope.js";
+import { applyRawEvidenceUploadOutcomes, hasRetryableEvidenceGap, logEvidenceBackoffBypassed, logEvidenceHeldByBackoff, logPermanentlyRejectedEvidence, partitionHeldEvidenceFiles, permanentEvidenceFailureReason, persistDeliveryAttempts, retrySourcesForFailedSync, retryableEvidenceGapReason, summarizeRawEvidenceDelivery, } from "./upload-evidence-delivery.js";
+import { reportAgentImageArtifacts } from "./upload-agent-artifacts.js";
+import { describeError } from "./health-detail.js";
+import { postEnvelopeToIngest } from "./upload-ingest-receipt.js";
+import { classifySyncFailure, } from "./upload-failure-reason.js";
+export async function syncLocalAmbientEnvelope(options = {}) {
+    const attemptedAt = (options.now ?? new Date()).toISOString();
+    const paths = getCollectorRuntimePaths(options.homeDir);
+    const cursor = await readRawEvidenceCursor(paths);
+    const built = await buildEnvelopeOrRecordBlocker(paths, options, cursor.objects, attemptedAt);
+    const fetchImpl = options.fetch ?? globalThis.fetch;
+    if (!fetchImpl) {
+        throw new Error("global fetch is unavailable; use Node.js 20 or newer.");
+    }
+    const provenance = built.envelope.work_context.provenance;
+    if (!provenance) {
+        throw new Error("Ambient upload requires collector provenance.");
+    }
+    // Assigned progressively so the spool path below can still report what did
+    // land if the upload pass throws part-way through.
+    let uploadOutcomes = [];
+    let uploadedChunkCount = 0;
+    const attemptedDate = options.now ?? new Date(attemptedAt);
+    const staging = await readRawEvidenceStagingState(paths.state_dir);
+    try {
+        if (built.raw_evidence_upload_files.length > 0) {
+            // Objects still inside their backoff window are not offered at all. They
+            // become held outcomes below, which keeps them out of the envelope, keeps
+            // the sync in retry_pending, and names the reason — the 559th identical
+            // attempt is what BLI-3066 made impossible. An operator-invoked retry
+            // (`cockpit backfill`, doctor) offers them anyway and says so: the window
+            // is the scheduler's cadence, not an answer to a person asking now
+            // (BLI-3118).
+            const { deliverable, held, bypassed } = partitionHeldEvidenceFiles(built.raw_evidence_upload_files, staging, attemptedDate, options.evidenceDeliveryMode);
+            logEvidenceHeldByBackoff(held, attemptedAt);
+            logEvidenceBackoffBypassed(bypassed, attemptedAt);
+            uploadOutcomes = held;
+            if (deliverable.length > 0) {
+                const upload = await uploadRawEvidenceFilesChunked({
+                    fetchImpl,
+                    dashboardUrl: built.dashboard_url,
+                    deviceToken: built.device_token,
+                    provenance,
+                    generatedAt: built.envelope.generated_at,
+                    files: deliverable,
+                });
+                uploadOutcomes = [...held, ...upload.outcomes];
+                uploadedChunkCount = upload.uploaded_chunk_count;
+            }
+            await persistDeliveryAttempts(paths.state_dir, staging, uploadOutcomes, attemptedDate);
+        }
+        const response = await postEnvelopeToIngest({
+            fetchImpl,
+            built,
+            envelope: applyRawEvidenceUploadOutcomes(built.envelope, uploadOutcomes),
+        });
+        advanceRawEvidenceCursor(cursor, uploadOutcomes, attemptedAt);
+        await writeRawEvidenceCursor(paths, cursor);
+        await reportAgentImageArtifacts({
+            fetchImpl,
+            dashboardUrl: built.dashboard_url,
+            deviceToken: built.device_token,
+            provenance,
+            generatedAt: built.envelope.generated_at,
+            ticketId: built.ticket_id,
+            repoFingerprint: built.envelope.work_context.repo_fingerprint,
+            worktreeFingerprint: built.envelope.work_context.worktree_fingerprint,
+            outcomes: uploadOutcomes,
+            reused: built.raw_evidence_facts?.reused ?? [],
+            cursorObjects: cursor.objects,
+        }).catch((error) => {
+            // The whole point of `agent_artifact_report_local_error` is that it is
+            // NOT the network error the reporter already labels itself — it is a
+            // throw from our own code on the way in. That distinction is only
+            // useful if the throw says what it was (BLI-3238).
+            console.error("[cockpit-sync] agent image artifact report threw locally", JSON.stringify({
+                reason: "agent_artifact_report_local_error",
+                ...describeError(error),
+            }));
+            return {
+                posted: false,
+                reason: "agent_artifact_report_local_error",
+                recorded_count: 0,
+            };
+        });
+        await recordIngestedSyncOutcome({
+            paths,
+            options,
+            built,
+            uploadOutcomes,
+            attemptedAt,
+        });
+        // The success branch logs too. A line that only fires on failure cannot
+        // answer "did anything land at all today?", which is the question that
+        // would have caught BLI-2528 in a week instead of 57 days.
+        console.error("[cockpit-sync] ingest accepted the envelope", JSON.stringify({
+            reason: "ingest_accepted",
+            http_status: response.status,
+            event_count: built.event_count,
+            raw_evidence_file_count: built.raw_evidence_upload_files.length,
+            uploaded_chunk_count: uploadedChunkCount,
+            failed_object_count: uploadOutcomes.filter((outcome) => outcome.upload_state === "upload_failed").length,
+        }));
+        return {
+            status: "uploaded",
+            dashboard_url: built.dashboard_url,
+            ticket_id: built.ticket_id,
+            work_context_id: built.envelope.work_context.work_context_id,
+            head_sha: built.head_sha,
+            event_count: built.event_count,
+            source_scan_count: built.source_scan_count,
+            risk_flag_count: built.risk_flag_count,
+            http_status: response.status,
+            ...summarizeRawEvidenceDelivery(built, uploadOutcomes, uploadedChunkCount, cursor, staging, attemptedDate),
+        };
+    }
+    catch (error) {
+        // The spool row used to store `error.message` verbatim, so every distinct
+        // server sentence became its own "reason" and nothing could count how many
+        // machines were failing the same way (BLI-3483). It now stores a label from
+        // a closed set plus a bounded, redacted cause.
+        const classified = classifySyncFailure(error);
+        // A committed content-addressed object may be shared by concurrent syncs.
+        // Never delete it from this error path: an ingest retry can reuse it, while
+        // client-side cleanup cannot prove exclusive ownership without racing a
+        // second sync that is about to index the same object.
+        const spooledFailureReason = classified.failure_reason;
+        console.error("[cockpit-sync] sync spooled for retry", JSON.stringify({
+            reason: classified.reason,
+            ...(classified.http_status === null
+                ? {}
+                : { http_status: classified.http_status }),
+            ...(classified.detail ? { detail: classified.detail } : {}),
+            event_count: built.event_count,
+            raw_evidence_file_count: built.raw_evidence_upload_files.length,
+            uploaded_chunk_count: uploadedChunkCount,
+        }));
+        const entry = await recordUploadFailure(paths, {
+            last_attempt_at: attemptedAt,
+            dashboard_url: built.dashboard_url,
+            work_context_id: built.envelope.work_context.work_context_id,
+            ticket_id: built.ticket_id,
+            repo_label: built.repo_label,
+            branch: built.envelope.work_context.branch,
+            event_count: built.event_count,
+            source_scan_count: built.source_scan_count,
+            risk_flag_count: built.risk_flag_count,
+            raw_evidence_file_count: built.raw_evidence_upload_files.length,
+            retry_sources: retrySourcesForFailedSync(options, built.raw_evidence_facts),
+            failure_reason: spooledFailureReason,
+            retry_command: "cockpit sync",
+        });
+        return {
+            status: "spooled",
+            dashboard_url: built.dashboard_url,
+            ticket_id: built.ticket_id,
+            work_context_id: built.envelope.work_context.work_context_id,
+            head_sha: built.head_sha,
+            event_count: built.event_count,
+            source_scan_count: built.source_scan_count,
+            risk_flag_count: built.risk_flag_count,
+            failure_reason: spooledFailureReason,
+            failure_class: classified.reason,
+            failure_http_status: classified.http_status,
+            spool_entry_id: entry.spool_id,
+            retry_command: entry.retry_command,
+            ...summarizeRawEvidenceDelivery(built, uploadOutcomes, uploadedChunkCount, cursor, staging, attemptedDate),
+        };
+    }
+}
+/**
+ * Assemble the envelope, and if this machine simply cannot upload, say so in
+ * the spool before the error leaves. A blocker that only ever surfaced as a
+ * thrown exception would be invisible to the next `cockpit status`.
+ */
+async function buildEnvelopeOrRecordBlocker(paths, options, cursorObjects, attemptedAt) {
+    try {
+        return await buildLocalAmbientEnvelope({
+            ...options,
+            cursorObjects: options.cursorObjects ?? cursorObjects,
+        });
+    }
+    catch (error) {
+        if (error instanceof LocalUploadBlockedError) {
+            await recordUploadBlocked(paths, {
+                attemptedAt,
+                reason: error.message,
+            });
+        }
+        throw error;
+    }
+}
+/**
+ * Remember every object that is now durable, so the next sync can reuse the
+ * bytes instead of re-uploading them. A failed upload is deliberately absent:
+ * recording it would make the next sync skip a file that never landed.
+ */
+function advanceRawEvidenceCursor(cursor, outcomes, attemptedAt) {
+    for (const outcome of outcomes) {
+        if (outcome.upload_state === "upload_failed")
+            continue;
+        const contentHash = outcome.pointer.content_hash_sha256;
+        if (!contentHash)
+            continue;
+        markObjectCommitted(cursor, contentHash, {
+            object_key: outcome.object_key,
+            byte_size: outcome.pointer.byte_size ?? 0,
+            committed_at: attemptedAt,
+        });
+    }
+    cursor.updated_at = attemptedAt;
+}
+/**
+ * What the operator is owed after ingest accepted the envelope, as a decision
+ * table on the evidence this sync left behind:
+ *
+ * | evidence state                        | ledger                              |
+ * |---------------------------------------|-------------------------------------|
+ * | a gap a later sync could rescue        | success, then a queued retry naming why |
+ * | a gap no retry can rescue             | success, then a blocker naming why, plus a stderr record |
+ * | nothing missing                       | success, pending state cleared      |
+ *
+ * The middle row is the one that matters. A permanently rejected object gets no
+ * retry — nothing would change — but it must still be named, or `cockpit status`
+ * reads clean on a machine with missing collection (BLI-2528).
+ */
+async function recordIngestedSyncOutcome(options) {
+    const { paths, built, uploadOutcomes, attemptedAt } = options;
+    const retryableEvidenceGap = hasRetryableEvidenceGap(built.raw_evidence_facts, uploadOutcomes);
+    await recordUploadSuccess(paths, {
+        attemptedAt,
+        workContextId: built.envelope.work_context.work_context_id,
+        clearPendingForContext: !retryableEvidenceGap,
+    });
+    if (retryableEvidenceGap) {
+        await recordUploadFailure(paths, {
+            last_attempt_at: attemptedAt,
+            dashboard_url: built.dashboard_url,
+            work_context_id: built.envelope.work_context.work_context_id,
+            ticket_id: built.ticket_id,
+            repo_label: built.repo_label,
+            branch: built.envelope.work_context.branch,
+            event_count: built.event_count,
+            source_scan_count: built.source_scan_count,
+            risk_flag_count: built.risk_flag_count,
+            raw_evidence_file_count: built.raw_evidence_upload_files.length,
+            retry_sources: retrySourcesForFailedSync(options.options, built.raw_evidence_facts),
+            failure_reason: retryableEvidenceGapReason(built.raw_evidence_facts, uploadOutcomes),
+            retry_command: "cockpit sync",
+        });
+        return;
+    }
+    const permanentReason = permanentEvidenceFailureReason(uploadOutcomes);
+    if (!permanentReason)
+        return;
+    await recordUploadBlocked(paths, {
+        attemptedAt,
+        reason: permanentReason,
+    });
+    logPermanentlyRejectedEvidence({
+        reason: permanentReason,
+        outcomes: uploadOutcomes,
+        attemptedAt,
+    });
+}