@bli-cockpit/cli 0.2.28 → 0.2.29

package/dist/upload-evidence-delivery.js

@@ -0,0 +1,505 @@
+/**
+ * Did the bytes land, and what does the operator still owe?
+ *
+ * Everything in this file exists to keep one promise from the fleet contract:
+ * a green status must never hide missing collection. Three questions, in order:
+ *
+ * 1. **What may we even offer this sync?** `partitionHeldEvidenceFiles` keeps
+ *    objects inside their delivery-backoff window off the wire, and turns each
+ *    one into a named failure rather than a silent omission.
+ * 2. **What just happened to each object?** `persistDeliveryAttempts` writes the
+ *    attempt counts that drive the backoff, before ingest, so an ingest failure
+ *    cannot reset every window to zero.
+ * 3. **What do we tell the operator?** `hasRetryableEvidenceGap` decides whether
+ *    a retry is queued, `retryableEvidenceGapReason` and
+ *    `permanentEvidenceFailureReason` name why, and
+ *    `summarizeRawEvidenceDelivery` builds the counts `cockpit status` reads.
+ *
+ * The one distinction that runs through all of it: a failure a later sync could
+ * still rescue is not the same thing as one storage has already refused on its
+ * own terms. Confusing them is what kept a Mac in `retry_pending` through 13
+ * identical syncs (BLI-2528), and what let a transcript sit undelivered for nine
+ * days while the sync read clean (BLI-3066).
+ */
+import { EvidenceCompletenessPayloadSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
+import { clearDeliveryAttempt, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, deliveryHold, recordDeliveryFailure, summarizeStuckEvidence, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
+// --------------------------------------------------------------------------
+// 1. What may we offer this sync?
+// --------------------------------------------------------------------------
+/**
+ * Split the pack's files into "offer these now" and "still in backoff".
+ *
+ * A held file becomes an `upload_failed` outcome labelled
+ * `delivery_backoff_holding`. That is deliberate rather than a quiet omission:
+ * the pointer gets pruned from the envelope (the object is genuinely not
+ * durable), the sync stays in `retry_pending`, and the reason travels to the
+ * status output. A hold that read as success would be the green-status-hiding-
+ * missing-collection failure the fleet contract forbids.
+ */
+export function partitionHeldEvidenceFiles(files, staging, now, mode) {
+    const deliverable = [];
+    const held = [];
+    const bypassed = [];
+    const backoffApplies = deliveryBackoffApplies(mode);
+    for (const file of files) {
+        const hold = deliveryHold(staging, file.pointer.content_hash_sha256, now);
+        if (hold && !backoffApplies) {
+            // BLI-3118: a person asked for this one now. Offering it is the whole
+            // point of the retry command Cockpit printed, and the bypass is logged
+            // rather than assumed.
+            bypassed.push(hold);
+            deliverable.push(file);
+            continue;
+        }
+        if (!hold) {
+            deliverable.push(file);
+            continue;
+        }
+        held.push({
+            pointer: file.pointer,
+            object_key: file.pointer.object_key ?? "",
+            codex_session_id: file.codex_session_id ?? null,
+            kind: file.kind ?? "raw_evidence",
+            ...(file.artifact_metadata
+                ? { artifact_metadata: file.artifact_metadata }
+                : {}),
+            upload_state: "upload_failed",
+            reason: DELIVERY_BACKOFF_HOLDING_REASON,
+            uploaded_chunk_count: 0,
+        });
+    }
+    return { deliverable, held, bypassed };
+}
+/**
+ * Say out loud that bytes were withheld on purpose.
+ *
+ * Counts and sizes only, never a path. Without this line an unattended machine
+ * withholds evidence for hours and leaves no trace of having done so.
+ */
+export function logEvidenceHeldByBackoff(held, attemptedAt) {
+    if (held.length === 0)
+        return;
+    console.error("[cockpit-sync] raw evidence held by delivery backoff", JSON.stringify({
+        attempted_at: attemptedAt,
+        reason: DELIVERY_BACKOFF_HOLDING_REASON,
+        object_count: held.length,
+        byte_size: held.reduce((sum, outcome) => sum + (outcome.pointer.byte_size ?? 0), 0),
+    }));
+}
+/**
+ * Say out loud that an operator's retry ignored a live backoff window.
+ *
+ * The success branch of BLI-3118: without this line the only trace of the
+ * decision is an object that was held on one run and offered on the next, and
+ * nothing on the machine says which rule made the difference.
+ */
+export function logEvidenceBackoffBypassed(bypassed, attemptedAt) {
+    if (bypassed.length === 0)
+        return;
+    console.error("[cockpit-sync] raw evidence delivery backoff bypassed", JSON.stringify({
+        attempted_at: attemptedAt,
+        reason: DELIVERY_BACKOFF_BYPASS_REASON,
+        object_count: bypassed.length,
+        byte_size: bypassed.reduce((sum, entry) => sum + entry.byte_size, 0),
+        max_attempts: bypassed.reduce((max, entry) => Math.max(max, entry.attempts), 0),
+        last_reasons: [...new Set(bypassed.map((entry) => entry.last_reason))]
+            .sort(),
+    }));
+}
+// --------------------------------------------------------------------------
+// 2. What just happened to each object?
+// --------------------------------------------------------------------------
+/**
+ * Count what just happened to each object, and when it may be offered again.
+ *
+ * Written immediately after the upload pass and before ingest, so an ingest
+ * failure cannot lose the attempt counts — losing them resets every backoff to
+ * zero and the fleet is back to 15-minute retries forever. A held outcome is
+ * not itself an attempt: counting it would push its own next attempt further
+ * out on every sync and eventually never retry at all.
+ */
+export async function persistDeliveryAttempts(stateDir, staging, outcomes, attemptedAt) {
+    let changed = false;
+    for (const outcome of outcomes) {
+        const contentHash = outcome.pointer.content_hash_sha256;
+        if (!contentHash)
+            continue;
+        if (outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON)
+            continue;
+        if (outcome.upload_state === "upload_failed") {
+            const entry = recordDeliveryFailure(staging, contentHash, {
+                reason: outcome.reason ?? "upload_failed",
+                attemptedAt,
+                byteSize: outcome.pointer.byte_size ?? 0,
+            });
+            changed = true;
+            console.error("[cockpit-sync] raw evidence delivery failed", JSON.stringify({
+                reason: entry.last_reason,
+                kind: outcome.kind,
+                attempts: entry.attempts,
+                first_failed_at: entry.first_failed_at,
+                next_attempt_at: entry.next_attempt_at,
+                byte_size: entry.byte_size,
+            }));
+            continue;
+        }
+        if (clearDeliveryAttempt(staging, contentHash)) {
+            changed = true;
+            console.error("[cockpit-sync] raw evidence delivery recovered", JSON.stringify({
+                reason: "delivery_recovered",
+                kind: outcome.kind,
+                upload_state: outcome.upload_state,
+            }));
+        }
+    }
+    if (!changed)
+        return;
+    staging.updated_at = attemptedAt.toISOString();
+    await writeRawEvidenceStagingState(stateDir, staging).catch((error) => {
+        console.error("[cockpit-sync] delivery attempt state write failed", JSON.stringify({
+            reason: "staging_state_write_failed",
+            detail: error instanceof Error ? error.name : typeof error,
+            tracked_count: Object.keys(staging.delivery_attempts).length,
+        }));
+    });
+}
+// --------------------------------------------------------------------------
+// 3. What do we still owe the operator?
+// --------------------------------------------------------------------------
+/**
+ * Which adapters a queued retry has to re-run.
+ *
+ * Anything this sync touched or tried to touch counts, including a directory it
+ * could not read — a source that failed to scan is exactly the one a retry must
+ * come back to.
+ */
+export function retrySourcesForFailedSync(options, facts) {
+    const sources = new Set();
+    if ((options.codexSessionFiles?.length ?? 0) > 0 ||
+        (options.codexAttributionScan?.directory_read_failed_count ?? 0) > 0 ||
+        (options.codexAttributionScan?.stat_failed_count ?? 0) > 0 ||
+        facts?.evidence_completeness.source_counts.some((count) => count.source.startsWith("codex_") &&
+            count.scanned_count + count.included_count + count.reused_count > 0)) {
+        sources.add("codex");
+    }
+    if ((options.claudeSessionFiles?.length ?? 0) > 0 ||
+        (options.claudeAttributionScan?.project_dir_read_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.session_stat_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.sidecar_dir_read_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.sidecar_stat_failed_count ?? 0) > 0 ||
+        facts?.evidence_completeness.source_counts.some((count) => count.source.startsWith("claude_") &&
+            count.scanned_count + count.included_count + count.reused_count > 0)) {
+        sources.add("claude_code");
+    }
+    return [...sources];
+}
+/**
+ * The failed uploads a later attempt could still rescue.
+ *
+ * An object storage has already refused on its own terms is not one of them,
+ * and counting it as one is what kept Edward's Mac in `retry_pending` through
+ * 13 consecutive syncs that were never going to end differently (BLI-2528).
+ */
+function retryableFailedOutcomes(outcomes) {
+    return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
+        !isPermanentUploadFailure(outcome.reason));
+}
+/** Failed uploads that no retry can rescue, kept so they can still be named. */
+function permanentFailedOutcomes(outcomes) {
+    return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
+        isPermanentUploadFailure(outcome.reason));
+}
+/**
+ * The reason to show for objects that failed for good.
+ *
+ * Returns null when there are none. These never queue a retry, but they must
+ * never disappear either: a machine with a permanently rejected object has
+ * missing collection, and a status that reads clean would hide it.
+ */
+export function permanentEvidenceFailureReason(outcomes) {
+    const reasons = new Set(permanentFailedOutcomes(outcomes).map((outcome) => outcome.reason ?? "unknown"));
+    if (reasons.size === 0)
+        return null;
+    return `raw_evidence_permanently_rejected:${[...reasons].sort().join(",")}`;
+}
+/**
+ * Leave a dated record of the objects this machine gave up on.
+ *
+ * stderr, which launchd captures to `sync.err.log`, so an unattended machine
+ * still says what it lost. Reason labels and counts only — never a path or a
+ * byte of content.
+ */
+export function logPermanentlyRejectedEvidence(options) {
+    console.error("[cockpit-sync] raw evidence permanently rejected", JSON.stringify({
+        attempted_at: options.attemptedAt,
+        reason: options.reason,
+        object_count: permanentFailedOutcomes(options.outcomes).length,
+    }));
+}
+/**
+ * Is there anything a later sync could still turn into evidence?
+ *
+ * Any one of these is enough: an upload that failed for a rescuable reason,
+ * evidence deferred by this sync's byte or object budget, a completeness
+ * payload that reports failure, or a skip a retry could undo.
+ */
+export function hasRetryableEvidenceGap(facts, outcomes) {
+    if (retryableFailedOutcomes(outcomes).length > 0) {
+        return true;
+    }
+    if (!facts)
+        return false;
+    if (facts.deferred_byte_budget_count > 0 ||
+        facts.deferred_object_budget_count > 0) {
+        return true;
+    }
+    if (facts.evidence_completeness.status === "failed" ||
+        facts.evidence_completeness.totals.failed_count > 0 ||
+        facts.evidence_completeness.failure_reasons.length > 0) {
+        return true;
+    }
+    return facts.evidence_completeness.skip_reasons.some(({ reason }) => isRetryableEvidenceSkipReason(reason));
+}
+/**
+ * Skip reasons a later sync can still turn into evidence.
+ *
+ * `delivery_backoff_holding` is one of them, and it has to be: a source held by
+ * backoff is missing collection right now. If it did not land here the sync
+ * would read clean while a transcript sat undelivered for nine days, which is
+ * the exact shape of BLI-3066.
+ */
+function isRetryableEvidenceSkipReason(reason) {
+    if (reason === DELIVERY_BACKOFF_HOLDING_REASON)
+        return true;
+    return /(?:read|stat|directory)_failed|session_limit_overflow/iu.test(reason);
+}
+/** Every distinct reason behind the gap, sorted, as one spool-ready label. */
+export function retryableEvidenceGapReason(facts, outcomes) {
+    const reasons = new Set();
+    for (const outcome of retryableFailedOutcomes(outcomes)) {
+        reasons.add(outcome.reason ?? "upload_failed");
+    }
+    if (facts) {
+        if (facts.deferred_byte_budget_count > 0)
+            reasons.add("deferred_byte_budget");
+        if (facts.deferred_object_budget_count > 0) {
+            reasons.add("deferred_object_budget");
+        }
+        for (const { reason } of facts.evidence_completeness.failure_reasons) {
+            reasons.add(reason);
+        }
+        if (facts.evidence_completeness.status === "failed" &&
+            facts.evidence_completeness.failure_reasons.length === 0) {
+            reasons.add("evidence_completeness_failed");
+        }
+        for (const { reason } of facts.evidence_completeness.skip_reasons) {
+            if (isRetryableEvidenceSkipReason(reason)) {
+                reasons.add(reason);
+            }
+        }
+    }
+    return `partial_raw_evidence_retry_required:${[
+        ...reasons,
+    ].sort().join(",") || "unknown"}`;
+}
+/**
+ * The counts `cockpit sync --json` and `cockpit status` read.
+ *
+ * Two kinds of reuse are added together on purpose: files this machine skipped
+ * because its own cursor already had the bytes, and files the server recognised
+ * from their content hash.
+ */
+export function summarizeRawEvidenceDelivery(built, outcomes, uploadedChunkCount, cursor, staging, now) {
+    const stuck = summarizeStuckEvidence(staging, now);
+    const cursorReused = built.raw_evidence_facts?.reused ?? [];
+    const serverReusedCount = outcomes.filter((outcome) => outcome.upload_state === "reused_existing").length;
+    const failed = outcomes.filter((outcome) => outcome.upload_state === "upload_failed");
+    const retryRequired = hasRetryableEvidenceGap(built.raw_evidence_facts, outcomes);
+    const retryReason = retryRequired
+        ? retryableEvidenceGapReason(built.raw_evidence_facts, outcomes)
+        : null;
+    const cursorReusedOutcomes = cursorReused.map((entry) => ({
+        object_key: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
+        raw_evidence_pointer_id: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
+        kind: entry.kind,
+        codex_session_id: entry.codex_session_id,
+        ...(entry.artifact_metadata
+            ? { artifact_metadata: entry.artifact_metadata }
+            : {}),
+        upload_state: "reused_existing",
+        reason: "cursor_content_match",
+    }));
+    return {
+        raw_evidence_file_count: built.raw_evidence_upload_files.length,
+        raw_evidence_uploaded_object_count: outcomes.filter((outcome) => outcome.upload_state === "uploaded").length,
+        raw_evidence_uploaded_chunk_count: uploadedChunkCount,
+        raw_evidence_reused_count: cursorReused.length + serverReusedCount,
+        raw_evidence_failed_count: failed.length,
+        raw_evidence_sanitized_count: built.raw_evidence_facts?.sanitized_count ?? 0,
+        raw_evidence_failure_reasons: [
+            ...new Set(failed.map((outcome) => outcome.reason ?? "unknown")),
+        ],
+        raw_evidence_retry_required: retryRequired,
+        raw_evidence_retry_reasons: retryReason
+            ? retryReason
+                .replace(/^partial_raw_evidence_retry_required:/u, "")
+                .split(",")
+                .filter(Boolean)
+            : [],
+        raw_evidence_outcomes: [
+            ...outcomes.map((outcome) => ({
+                object_key: outcome.object_key,
+                raw_evidence_pointer_id: outcome.pointer.raw_evidence_pointer_id,
+                kind: outcome.kind,
+                codex_session_id: outcome.codex_session_id,
+                ...(outcome.artifact_metadata
+                    ? { artifact_metadata: outcome.artifact_metadata }
+                    : {}),
+                upload_state: outcome.upload_state,
+                reason: outcome.reason,
+            })),
+            ...cursorReusedOutcomes,
+        ],
+        raw_evidence_deferred_byte_budget: built.raw_evidence_facts?.deferred_byte_budget_count ?? 0,
+        raw_evidence_deferred_object_budget: built.raw_evidence_facts?.deferred_object_budget_count ?? 0,
+        cursor_tracked_object_count: Object.keys(cursor.objects).length,
+        raw_evidence_delivery_held_count: outcomes.filter((outcome) => outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON).length + (built.raw_evidence_facts?.delivery_held_count ?? 0),
+        raw_evidence_stuck_object_count: stuck.stuck_object_count,
+        raw_evidence_max_delivery_attempts: stuck.max_attempts,
+        raw_evidence_oldest_delivery_failure_at: stuck.oldest_first_failed_at,
+    };
+}
+// --------------------------------------------------------------------------
+// 4. Reconciling the envelope with what actually became durable
+// --------------------------------------------------------------------------
+/**
+ * Ingest refuses pointers whose objects never became durable, so failed
+ * uploads are pruned from the envelope instead of failing the whole sync.
+ * Successful upload responses can also carry server-side sanitized hash and
+ * redaction metadata; apply those before ingest so refs describe the bytes
+ * actually stored in the durable bucket.
+ */
+export function applyRawEvidenceUploadOutcomes(envelope, outcomes) {
+    // Content-addressed keys mean one pointer id can carry several outcomes
+    // (byte-identical files); the pointer is durable if ANY outcome succeeded.
+    const durablePointers = new Map();
+    for (const outcome of outcomes) {
+        if (outcome.upload_state === "upload_failed")
+            continue;
+        durablePointers.set(outcome.pointer.raw_evidence_pointer_id, outcome.pointer);
+    }
+    const durablePointerIds = new Set([...durablePointers.keys()]);
+    const failedPointerIds = new Set(outcomes
+        .filter((outcome) => outcome.upload_state === "upload_failed" &&
+        !durablePointerIds.has(outcome.pointer.raw_evidence_pointer_id))
+        .map((outcome) => outcome.pointer.raw_evidence_pointer_id));
+    const failedOutcomes = outcomes.filter((outcome) => failedPointerIds.has(outcome.pointer.raw_evidence_pointer_id));
+    if (failedPointerIds.size === 0 && durablePointers.size === 0)
+        return envelope;
+    return {
+        ...envelope,
+        events: envelope.events.map((event) => ({
+            ...event,
+            metrics: failedPointerIds.size > 0
+                ? {
+                    ...event.metrics,
+                    evidence_failed_count: (event.metrics["evidence_failed_count"] ?? 0) +
+                        failedOutcomes.length,
+                }
+                : event.metrics,
+            attributes: failedPointerIds.size > 0 && event.evidence_completeness
+                ? {
+                    ...event.attributes,
+                    evidence_completeness_schema_version: event.evidence_completeness.schema_version,
+                    evidence_completeness_status: "partial",
+                    evidence_incomplete: true,
+                }
+                : event.attributes,
+            evidence_completeness: failedPointerIds.size > 0 && event.evidence_completeness
+                ? markCompletenessUploadFailures(event.evidence_completeness, failedOutcomes)
+                : event.evidence_completeness,
+            raw_evidence_pointers: event.raw_evidence_pointers.flatMap((pointer) => {
+                const pointerId = pointer.raw_evidence_pointer_id;
+                if (failedPointerIds.has(pointerId))
+                    return [];
+                return [durablePointers.get(pointerId) ?? pointer];
+            }),
+            redaction: failedPointerIds.size > 0
+                ? {
+                    ...event.redaction,
+                    raw_evidence_pointer_ids: event.redaction.raw_evidence_pointer_ids.filter((pointerId) => !failedPointerIds.has(pointerId)),
+                }
+                : event.redaction,
+        })),
+    };
+}
+/**
+ * Fold upload failures back into the completeness payload the event carries, so
+ * a downstream reader sees `partial` and the per-source failure counts rather
+ * than a complete-looking payload with fewer pointers than it claims.
+ */
+function markCompletenessUploadFailures(completeness, failedOutcomes) {
+    const failureCounts = new Map();
+    for (const outcome of failedOutcomes) {
+        const source = outcome.kind ?? "raw_evidence";
+        failureCounts.set(source, (failureCounts.get(source) ?? 0) + 1);
+    }
+    const totalFailures = [...failureCounts.values()].reduce((sum, count) => sum + count, 0);
+    const sourceCounts = [...completeness.source_counts];
+    for (const [source, count] of failureCounts) {
+        const existingIndex = sourceCounts.findIndex((entry) => entry.source === source);
+        if (existingIndex === -1) {
+            sourceCounts.push({
+                source,
+                scanned_count: 0,
+                included_count: 0,
+                skipped_count: 0,
+                truncated_count: 0,
+                deferred_count: 0,
+                reused_count: 0,
+                failed_count: count,
+            });
+            continue;
+        }
+        const existing = sourceCounts[existingIndex];
+        if (!existing)
+            continue;
+        sourceCounts[existingIndex] = {
+            ...existing,
+            failed_count: existing.failed_count + count,
+        };
+    }
+    const failureReasons = [...completeness.failure_reasons];
+    for (const [source, count] of failureCounts) {
+        const reason = "upload_failed";
+        const existingIndex = failureReasons.findIndex((entry) => entry.source === source && entry.reason === reason);
+        if (existingIndex === -1) {
+            failureReasons.push({ source, reason, count });
+        }
+        else {
+            const existing = failureReasons[existingIndex];
+            if (existing) {
+                failureReasons[existingIndex] = {
+                    ...existing,
+                    count: existing.count + count,
+                };
+            }
+        }
+    }
+    return EvidenceCompletenessPayloadSchema.parse({
+        ...completeness,
+        status: "partial",
+        source_counts: sourceCounts,
+        totals: {
+            ...completeness.totals,
+            failed_count: completeness.totals.failed_count + totalFailures,
+        },
+        failure_reasons: failureReasons.sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
+        notes: [
+            ...new Set([
+                ...completeness.notes,
+                "Some collected evidence did not become durable; downstream analysis should lower confidence.",
+            ]),
+        ],
+    });
+}

package/dist/upload-http.js

@@ -0,0 +1,46 @@
+/**
+ * The small shared vocabulary for talking to the dashboard.
+ *
+ * A sync touches three separate ambient endpoints — ingest, codex-sessions and
+ * agent-artifacts — and all three need the same two things: read whatever the
+ * server sent back without letting a non-JSON body throw, and turn a normal
+ * dashboard URL into one that can be concatenated with a path.
+ */
+/**
+ * Reads a reply body as JSON, and never throws doing it.
+ *
+ * An empty body reads as `{}`; a body that is not JSON at all (a proxy's HTML
+ * error page, say) reads as `{ message: <text> }` so the caller can still put a
+ * reason in front of a human.
+ */
+export async function readResponseJson(response) {
+    const text = await response.text();
+    if (!text)
+        return {};
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return { message: text };
+    }
+}
+/** The server's own words when it supplied any, otherwise our fallback. */
+export function responseErrorMessage(value, fallback) {
+    if (value && typeof value === "object") {
+        const record = value;
+        const message = record["message"] ?? record["error"];
+        if (typeof message === "string" && message.trim())
+            return message;
+    }
+    return fallback;
+}
+/** Trims a dashboard URL to a bare origin so `${url}/api/...` is well formed. */
+export function normalizeDashboardUrl(value) {
+    const normalized = value.trim().replace(/\/+$/, "");
+    if (!normalized)
+        throw new Error("Dashboard URL cannot be empty.");
+    return normalized;
+}
+export function isNonEmptyString(value) {
+    return typeof value === "string" && value.trim().length > 0;
+}