@bli-cockpit/cli 0.2.54 → 0.2.56

package/dist/upload-evidence-delivery-reconcile.js

@@ -0,0 +1,134 @@
+import { EvidenceCompletenessPayloadSchema, } from "@bli-cockpit/telemetry-core";
+/**
+ * 4. Reconciling the envelope with what actually became durable.
+ *
+ * Ingest refuses pointers whose objects never became durable, so failed
+ * uploads are pruned from the envelope instead of failing the whole sync.
+ * Successful upload responses can also carry server-side sanitized hash and
+ * redaction metadata; apply those before ingest so refs describe the bytes
+ * actually stored in the durable bucket.
+ */
+export function applyRawEvidenceUploadOutcomes(envelope, outcomes) {
+    // Content-addressed keys mean one pointer id can carry several outcomes
+    // (byte-identical files); the pointer is durable if ANY outcome succeeded.
+    const durablePointers = new Map();
+    for (const outcome of outcomes) {
+        if (outcome.upload_state === "upload_failed")
+            continue;
+        durablePointers.set(outcome.pointer.raw_evidence_pointer_id, outcome.pointer);
+    }
+    const durablePointerIds = new Set([...durablePointers.keys()]);
+    const failedPointerIds = new Set(outcomes
+        .filter((outcome) => outcome.upload_state === "upload_failed" &&
+        !durablePointerIds.has(outcome.pointer.raw_evidence_pointer_id))
+        .map((outcome) => outcome.pointer.raw_evidence_pointer_id));
+    const failedOutcomes = outcomes.filter((outcome) => failedPointerIds.has(outcome.pointer.raw_evidence_pointer_id));
+    if (failedPointerIds.size === 0 && durablePointers.size === 0)
+        return envelope;
+    return {
+        ...envelope,
+        events: envelope.events.map((event) => ({
+            ...event,
+            metrics: failedPointerIds.size > 0
+                ? {
+                    ...event.metrics,
+                    evidence_failed_count: (event.metrics["evidence_failed_count"] ?? 0) +
+                        failedOutcomes.length,
+                }
+                : event.metrics,
+            attributes: failedPointerIds.size > 0 && event.evidence_completeness
+                ? {
+                    ...event.attributes,
+                    evidence_completeness_schema_version: event.evidence_completeness.schema_version,
+                    evidence_completeness_status: "partial",
+                    evidence_incomplete: true,
+                }
+                : event.attributes,
+            evidence_completeness: failedPointerIds.size > 0 && event.evidence_completeness
+                ? markCompletenessUploadFailures(event.evidence_completeness, failedOutcomes)
+                : event.evidence_completeness,
+            raw_evidence_pointers: event.raw_evidence_pointers.flatMap((pointer) => {
+                const pointerId = pointer.raw_evidence_pointer_id;
+                if (failedPointerIds.has(pointerId))
+                    return [];
+                return [durablePointers.get(pointerId) ?? pointer];
+            }),
+            redaction: failedPointerIds.size > 0
+                ? {
+                    ...event.redaction,
+                    raw_evidence_pointer_ids: event.redaction.raw_evidence_pointer_ids.filter((pointerId) => !failedPointerIds.has(pointerId)),
+                }
+                : event.redaction,
+        })),
+    };
+}
+/**
+ * Fold upload failures back into the completeness payload the event carries, so
+ * a downstream reader sees `partial` and the per-source failure counts rather
+ * than a complete-looking payload with fewer pointers than it claims.
+ */
+function markCompletenessUploadFailures(completeness, failedOutcomes) {
+    const failureCounts = new Map();
+    for (const outcome of failedOutcomes) {
+        const source = outcome.kind ?? "raw_evidence";
+        failureCounts.set(source, (failureCounts.get(source) ?? 0) + 1);
+    }
+    const totalFailures = [...failureCounts.values()].reduce((sum, count) => sum + count, 0);
+    const sourceCounts = [...completeness.source_counts];
+    for (const [source, count] of failureCounts) {
+        const existingIndex = sourceCounts.findIndex((entry) => entry.source === source);
+        if (existingIndex === -1) {
+            sourceCounts.push({
+                source,
+                scanned_count: 0,
+                included_count: 0,
+                skipped_count: 0,
+                truncated_count: 0,
+                deferred_count: 0,
+                reused_count: 0,
+                failed_count: count,
+            });
+            continue;
+        }
+        const existing = sourceCounts[existingIndex];
+        if (!existing)
+            continue;
+        sourceCounts[existingIndex] = {
+            ...existing,
+            failed_count: existing.failed_count + count,
+        };
+    }
+    const failureReasons = [...completeness.failure_reasons];
+    for (const [source, count] of failureCounts) {
+        const reason = "upload_failed";
+        const existingIndex = failureReasons.findIndex((entry) => entry.source === source && entry.reason === reason);
+        if (existingIndex === -1) {
+            failureReasons.push({ source, reason, count });
+        }
+        else {
+            const existing = failureReasons[existingIndex];
+            if (existing) {
+                failureReasons[existingIndex] = {
+                    ...existing,
+                    count: existing.count + count,
+                };
+            }
+        }
+    }
+    return EvidenceCompletenessPayloadSchema.parse({
+        ...completeness,
+        status: "partial",
+        source_counts: sourceCounts,
+        totals: {
+            ...completeness.totals,
+            failed_count: completeness.totals.failed_count + totalFailures,
+        },
+        failure_reasons: failureReasons.sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
+        notes: [
+            ...new Set([
+                ...completeness.notes,
+                "Some collected evidence did not become durable; downstream analysis should lower confidence.",
+            ]),
+        ],
+    });
+}

package/dist/upload-evidence-delivery-summary.js

@@ -0,0 +1,205 @@
+import { isPermanentUploadFailure } from "@bli-cockpit/telemetry-core";
+import { DELIVERY_BACKOFF_HOLDING_REASON, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
+/**
+ * Which adapters a queued retry has to re-run.
+ *
+ * Anything this sync touched or tried to touch counts, including a directory it
+ * could not read — a source that failed to scan is exactly the one a retry must
+ * come back to.
+ */
+export function retrySourcesForFailedSync(options, facts) {
+    const sources = new Set();
+    if ((options.codexSessionFiles?.length ?? 0) > 0 ||
+        (options.codexAttributionScan?.directory_read_failed_count ?? 0) > 0 ||
+        (options.codexAttributionScan?.stat_failed_count ?? 0) > 0 ||
+        facts?.evidence_completeness.source_counts.some((count) => count.source.startsWith("codex_") &&
+            count.scanned_count + count.included_count + count.reused_count > 0)) {
+        sources.add("codex");
+    }
+    if ((options.claudeSessionFiles?.length ?? 0) > 0 ||
+        (options.claudeAttributionScan?.project_dir_read_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.session_stat_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.sidecar_dir_read_failed_count ?? 0) > 0 ||
+        (options.claudeAttributionScan?.sidecar_stat_failed_count ?? 0) > 0 ||
+        facts?.evidence_completeness.source_counts.some((count) => count.source.startsWith("claude_") &&
+            count.scanned_count + count.included_count + count.reused_count > 0)) {
+        sources.add("claude_code");
+    }
+    return [...sources];
+}
+/**
+ * The failed uploads a later attempt could still rescue.
+ *
+ * An object storage has already refused on its own terms is not one of them,
+ * and counting it as one is what kept Edward's Mac in `retry_pending` through
+ * 13 consecutive syncs that were never going to end differently (BLI-2528).
+ */
+function retryableFailedOutcomes(outcomes) {
+    return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
+        !isPermanentUploadFailure(outcome.reason));
+}
+/** Failed uploads that no retry can rescue, kept so they can still be named. */
+function permanentFailedOutcomes(outcomes) {
+    return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
+        isPermanentUploadFailure(outcome.reason));
+}
+/**
+ * The reason to show for objects that failed for good.
+ *
+ * Returns null when there are none. These never queue a retry, but they must
+ * never disappear either: a machine with a permanently rejected object has
+ * missing collection, and a status that reads clean would hide it.
+ */
+export function permanentEvidenceFailureReason(outcomes) {
+    const reasons = new Set(permanentFailedOutcomes(outcomes).map((outcome) => outcome.reason ?? "unknown"));
+    if (reasons.size === 0)
+        return null;
+    return `raw_evidence_permanently_rejected:${[...reasons].sort().join(",")}`;
+}
+/**
+ * Leave a dated record of the objects this machine gave up on.
+ *
+ * stderr, which launchd captures to `sync.err.log`, so an unattended machine
+ * still says what it lost. Reason labels and counts only — never a path or a
+ * byte of content.
+ */
+export function logPermanentlyRejectedEvidence(options) {
+    console.error("[cockpit-sync] raw evidence permanently rejected", JSON.stringify({
+        attempted_at: options.attemptedAt,
+        reason: options.reason,
+        object_count: permanentFailedOutcomes(options.outcomes).length,
+    }));
+}
+/**
+ * Is there anything a later sync could still turn into evidence?
+ *
+ * Any one of these is enough: an upload that failed for a rescuable reason,
+ * evidence deferred by this sync's byte or object budget, a completeness
+ * payload that reports failure, or a skip a retry could undo.
+ */
+export function hasRetryableEvidenceGap(facts, outcomes) {
+    if (retryableFailedOutcomes(outcomes).length > 0) {
+        return true;
+    }
+    if (!facts)
+        return false;
+    if (facts.deferred_byte_budget_count > 0 ||
+        facts.deferred_object_budget_count > 0) {
+        return true;
+    }
+    if (facts.evidence_completeness.status === "failed" ||
+        facts.evidence_completeness.totals.failed_count > 0 ||
+        facts.evidence_completeness.failure_reasons.length > 0) {
+        return true;
+    }
+    return facts.evidence_completeness.skip_reasons.some(({ reason }) => isRetryableEvidenceSkipReason(reason));
+}
+/**
+ * Skip reasons a later sync can still turn into evidence.
+ *
+ * `delivery_backoff_holding` is one of them, and it has to be: a source held by
+ * backoff is missing collection right now. If it did not land here the sync
+ * would read clean while a transcript sat undelivered for nine days, which is
+ * the exact shape of BLI-3066.
+ */
+function isRetryableEvidenceSkipReason(reason) {
+    if (reason === DELIVERY_BACKOFF_HOLDING_REASON)
+        return true;
+    return /(?:read|stat|directory)_failed|session_limit_overflow/iu.test(reason);
+}
+/** Every distinct reason behind the gap, sorted, as one spool-ready label. */
+export function retryableEvidenceGapReason(facts, outcomes) {
+    const reasons = new Set();
+    for (const outcome of retryableFailedOutcomes(outcomes)) {
+        reasons.add(outcome.reason ?? "upload_failed");
+    }
+    if (facts) {
+        if (facts.deferred_byte_budget_count > 0)
+            reasons.add("deferred_byte_budget");
+        if (facts.deferred_object_budget_count > 0) {
+            reasons.add("deferred_object_budget");
+        }
+        for (const { reason } of facts.evidence_completeness.failure_reasons) {
+            reasons.add(reason);
+        }
+        if (facts.evidence_completeness.status === "failed" &&
+            facts.evidence_completeness.failure_reasons.length === 0) {
+            reasons.add("evidence_completeness_failed");
+        }
+        for (const { reason } of facts.evidence_completeness.skip_reasons) {
+            if (isRetryableEvidenceSkipReason(reason)) {
+                reasons.add(reason);
+            }
+        }
+    }
+    return `partial_raw_evidence_retry_required:${[
+        ...reasons,
+    ].sort().join(",") || "unknown"}`;
+}
+/**
+ * The counts `cockpit sync --json` and `cockpit status` read.
+ *
+ * Two kinds of reuse are added together on purpose: files this machine skipped
+ * because its own cursor already had the bytes, and files the server recognised
+ * from their content hash.
+ */
+export function summarizeRawEvidenceDelivery(built, outcomes, uploadedChunkCount, cursor, staging, now) {
+    const stuck = summarizeStuckEvidence(staging, now);
+    const cursorReused = built.raw_evidence_facts?.reused ?? [];
+    const serverReusedCount = outcomes.filter((outcome) => outcome.upload_state === "reused_existing").length;
+    const failed = outcomes.filter((outcome) => outcome.upload_state === "upload_failed");
+    const retryRequired = hasRetryableEvidenceGap(built.raw_evidence_facts, outcomes);
+    const retryReason = retryRequired
+        ? retryableEvidenceGapReason(built.raw_evidence_facts, outcomes)
+        : null;
+    const cursorReusedOutcomes = cursorReused.map((entry) => ({
+        object_key: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
+        raw_evidence_pointer_id: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
+        kind: entry.kind,
+        codex_session_id: entry.codex_session_id,
+        ...(entry.artifact_metadata
+            ? { artifact_metadata: entry.artifact_metadata }
+            : {}),
+        upload_state: "reused_existing",
+        reason: "cursor_content_match",
+    }));
+    return {
+        raw_evidence_file_count: built.raw_evidence_upload_files.length,
+        raw_evidence_uploaded_object_count: outcomes.filter((outcome) => outcome.upload_state === "uploaded").length,
+        raw_evidence_uploaded_chunk_count: uploadedChunkCount,
+        raw_evidence_reused_count: cursorReused.length + serverReusedCount,
+        raw_evidence_failed_count: failed.length,
+        raw_evidence_sanitized_count: built.raw_evidence_facts?.sanitized_count ?? 0,
+        raw_evidence_failure_reasons: [
+            ...new Set(failed.map((outcome) => outcome.reason ?? "unknown")),
+        ],
+        raw_evidence_retry_required: retryRequired,
+        raw_evidence_retry_reasons: retryReason
+            ? retryReason
+                .replace(/^partial_raw_evidence_retry_required:/u, "")
+                .split(",")
+                .filter(Boolean)
+            : [],
+        raw_evidence_outcomes: [
+            ...outcomes.map((outcome) => ({
+                object_key: outcome.object_key,
+                raw_evidence_pointer_id: outcome.pointer.raw_evidence_pointer_id,
+                kind: outcome.kind,
+                codex_session_id: outcome.codex_session_id,
+                ...(outcome.artifact_metadata
+                    ? { artifact_metadata: outcome.artifact_metadata }
+                    : {}),
+                upload_state: outcome.upload_state,
+                reason: outcome.reason,
+            })),
+            ...cursorReusedOutcomes,
+        ],
+        raw_evidence_deferred_byte_budget: built.raw_evidence_facts?.deferred_byte_budget_count ?? 0,
+        raw_evidence_deferred_object_budget: built.raw_evidence_facts?.deferred_object_budget_count ?? 0,
+        cursor_tracked_object_count: Object.keys(cursor.objects).length,
+        raw_evidence_delivery_held_count: outcomes.filter((outcome) => outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON).length + (built.raw_evidence_facts?.delivery_held_count ?? 0),
+        raw_evidence_stuck_object_count: stuck.stuck_object_count,
+        raw_evidence_max_delivery_attempts: stuck.max_attempts,
+        raw_evidence_oldest_delivery_failure_at: stuck.oldest_first_failed_at,
+    };
+}