@bli-cockpit/cli 0.2.54 → 0.2.56

package/dist/upload-evidence-delivery.js

@@ -20,486 +20,16 @@
  * own terms. Confusing them is what kept a Mac in `retry_pending` through 13
  * identical syncs (BLI-2528), and what let a transcript sit undelivered for nine
  * days while the sync read clean (BLI-3066).
- */
-import { EvidenceCompletenessPayloadSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
-import { clearDeliveryAttempt, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, deliveryHold, recordDeliveryFailure, summarizeStuckEvidence, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
-// --------------------------------------------------------------------------
-// 1. What may we offer this sync?
-// --------------------------------------------------------------------------
-/**
- * Split the pack's files into "offer these now" and "still in backoff".
- *
- * A held file becomes an `upload_failed` outcome labelled
- * `delivery_backoff_holding`. That is deliberate rather than a quiet omission:
- * the pointer gets pruned from the envelope (the object is genuinely not
- * durable), the sync stays in `retry_pending`, and the reason travels to the
- * status output. A hold that read as success would be the green-status-hiding-
- * missing-collection failure the fleet contract forbids.
- */
-export function partitionHeldEvidenceFiles(files, staging, now, mode) {
-    const deliverable = [];
-    const held = [];
-    const bypassed = [];
-    const backoffApplies = deliveryBackoffApplies(mode);
-    for (const file of files) {
-        const hold = deliveryHold(staging, file.pointer.content_hash_sha256, now);
-        if (hold && !backoffApplies) {
-            // BLI-3118: a person asked for this one now. Offering it is the whole
-            // point of the retry command Cockpit printed, and the bypass is logged
-            // rather than assumed.
-            bypassed.push(hold);
-            deliverable.push(file);
-            continue;
-        }
-        if (!hold) {
-            deliverable.push(file);
-            continue;
-        }
-        held.push({
-            pointer: file.pointer,
-            object_key: file.pointer.object_key ?? "",
-            codex_session_id: file.codex_session_id ?? null,
-            kind: file.kind ?? "raw_evidence",
-            ...(file.artifact_metadata
-                ? { artifact_metadata: file.artifact_metadata }
-                : {}),
-            upload_state: "upload_failed",
-            reason: DELIVERY_BACKOFF_HOLDING_REASON,
-            uploaded_chunk_count: 0,
-        });
-    }
-    return { deliverable, held, bypassed };
-}
-/**
- * Say out loud that bytes were withheld on purpose.
  *
- * Counts and sizes only, never a path. Without this line an unattended machine
- * withholds evidence for hours and leaves no trace of having done so.
- */
-export function logEvidenceHeldByBackoff(held, attemptedAt) {
-    if (held.length === 0)
-        return;
-    console.error("[cockpit-sync] raw evidence held by delivery backoff", JSON.stringify({
-        attempted_at: attemptedAt,
-        reason: DELIVERY_BACKOFF_HOLDING_REASON,
-        object_count: held.length,
-        byte_size: held.reduce((sum, outcome) => sum + (outcome.pointer.byte_size ?? 0), 0),
-    }));
-}
-/**
- * Say out loud that an operator's retry ignored a live backoff window.
- *
- * The success branch of BLI-3118: without this line the only trace of the
- * decision is an object that was held on one run and offered on the next, and
- * nothing on the machine says which rule made the difference.
- */
-export function logEvidenceBackoffBypassed(bypassed, attemptedAt) {
-    if (bypassed.length === 0)
-        return;
-    console.error("[cockpit-sync] raw evidence delivery backoff bypassed", JSON.stringify({
-        attempted_at: attemptedAt,
-        reason: DELIVERY_BACKOFF_BYPASS_REASON,
-        object_count: bypassed.length,
-        byte_size: bypassed.reduce((sum, entry) => sum + entry.byte_size, 0),
-        max_attempts: bypassed.reduce((max, entry) => Math.max(max, entry.attempts), 0),
-        last_reasons: [...new Set(bypassed.map((entry) => entry.last_reason))]
-            .sort(),
-    }));
-}
-// --------------------------------------------------------------------------
-// 2. What just happened to each object?
-// --------------------------------------------------------------------------
-/**
- * Count what just happened to each object, and when it may be offered again.
- *
- * Written immediately after the upload pass and before ingest, so an ingest
- * failure cannot lose the attempt counts — losing them resets every backoff to
- * zero and the fleet is back to 15-minute retries forever. A held outcome is
- * not itself an attempt: counting it would push its own next attempt further
- * out on every sync and eventually never retry at all.
- */
-export async function persistDeliveryAttempts(stateDir, staging, outcomes, attemptedAt) {
-    let changed = false;
-    for (const outcome of outcomes) {
-        const contentHash = outcome.pointer.content_hash_sha256;
-        if (!contentHash)
-            continue;
-        if (outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON)
-            continue;
-        if (outcome.upload_state === "upload_failed") {
-            const entry = recordDeliveryFailure(staging, contentHash, {
-                reason: outcome.reason ?? "upload_failed",
-                attemptedAt,
-                byteSize: outcome.pointer.byte_size ?? 0,
-            });
-            changed = true;
-            console.error("[cockpit-sync] raw evidence delivery failed", JSON.stringify({
-                reason: entry.last_reason,
-                kind: outcome.kind,
-                attempts: entry.attempts,
-                first_failed_at: entry.first_failed_at,
-                next_attempt_at: entry.next_attempt_at,
-                byte_size: entry.byte_size,
-            }));
-            continue;
-        }
-        if (clearDeliveryAttempt(staging, contentHash)) {
-            changed = true;
-            console.error("[cockpit-sync] raw evidence delivery recovered", JSON.stringify({
-                reason: "delivery_recovered",
-                kind: outcome.kind,
-                upload_state: outcome.upload_state,
-            }));
-        }
-    }
-    if (!changed)
-        return;
-    staging.updated_at = attemptedAt.toISOString();
-    await writeRawEvidenceStagingState(stateDir, staging).catch((error) => {
-        console.error("[cockpit-sync] delivery attempt state write failed", JSON.stringify({
-            reason: "staging_state_write_failed",
-            detail: error instanceof Error ? error.name : typeof error,
-            tracked_count: Object.keys(staging.delivery_attempts).length,
-        }));
-    });
-}
-// --------------------------------------------------------------------------
-// 3. What do we still owe the operator?
-// --------------------------------------------------------------------------
-/**
- * Which adapters a queued retry has to re-run.
- *
- * Anything this sync touched or tried to touch counts, including a directory it
- * could not read — a source that failed to scan is exactly the one a retry must
- * come back to.
- */
-export function retrySourcesForFailedSync(options, facts) {
-    const sources = new Set();
-    if ((options.codexSessionFiles?.length ?? 0) > 0 ||
-        (options.codexAttributionScan?.directory_read_failed_count ?? 0) > 0 ||
-        (options.codexAttributionScan?.stat_failed_count ?? 0) > 0 ||
-        facts?.evidence_completeness.source_counts.some((count) => count.source.startsWith("codex_") &&
-            count.scanned_count + count.included_count + count.reused_count > 0)) {
-        sources.add("codex");
-    }
-    if ((options.claudeSessionFiles?.length ?? 0) > 0 ||
-        (options.claudeAttributionScan?.project_dir_read_failed_count ?? 0) > 0 ||
-        (options.claudeAttributionScan?.session_stat_failed_count ?? 0) > 0 ||
-        (options.claudeAttributionScan?.sidecar_dir_read_failed_count ?? 0) > 0 ||
-        (options.claudeAttributionScan?.sidecar_stat_failed_count ?? 0) > 0 ||
-        facts?.evidence_completeness.source_counts.some((count) => count.source.startsWith("claude_") &&
-            count.scanned_count + count.included_count + count.reused_count > 0)) {
-        sources.add("claude_code");
-    }
-    return [...sources];
-}
-/**
- * The failed uploads a later attempt could still rescue.
- *
- * An object storage has already refused on its own terms is not one of them,
- * and counting it as one is what kept Edward's Mac in `retry_pending` through
- * 13 consecutive syncs that were never going to end differently (BLI-2528).
- */
-function retryableFailedOutcomes(outcomes) {
-    return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
-        !isPermanentUploadFailure(outcome.reason));
-}
-/** Failed uploads that no retry can rescue, kept so they can still be named. */
-function permanentFailedOutcomes(outcomes) {
-    return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
-        isPermanentUploadFailure(outcome.reason));
-}
-/**
- * The reason to show for objects that failed for good.
- *
- * Returns null when there are none. These never queue a retry, but they must
- * never disappear either: a machine with a permanently rejected object has
- * missing collection, and a status that reads clean would hide it.
- */
-export function permanentEvidenceFailureReason(outcomes) {
-    const reasons = new Set(permanentFailedOutcomes(outcomes).map((outcome) => outcome.reason ?? "unknown"));
-    if (reasons.size === 0)
-        return null;
-    return `raw_evidence_permanently_rejected:${[...reasons].sort().join(",")}`;
-}
-/**
- * Leave a dated record of the objects this machine gave up on.
- *
- * stderr, which launchd captures to `sync.err.log`, so an unattended machine
- * still says what it lost. Reason labels and counts only — never a path or a
- * byte of content.
- */
-export function logPermanentlyRejectedEvidence(options) {
-    console.error("[cockpit-sync] raw evidence permanently rejected", JSON.stringify({
-        attempted_at: options.attemptedAt,
-        reason: options.reason,
-        object_count: permanentFailedOutcomes(options.outcomes).length,
-    }));
-}
-/**
- * Is there anything a later sync could still turn into evidence?
- *
- * Any one of these is enough: an upload that failed for a rescuable reason,
- * evidence deferred by this sync's byte or object budget, a completeness
- * payload that reports failure, or a skip a retry could undo.
- */
-export function hasRetryableEvidenceGap(facts, outcomes) {
-    if (retryableFailedOutcomes(outcomes).length > 0) {
-        return true;
-    }
-    if (!facts)
-        return false;
-    if (facts.deferred_byte_budget_count > 0 ||
-        facts.deferred_object_budget_count > 0) {
-        return true;
-    }
-    if (facts.evidence_completeness.status === "failed" ||
-        facts.evidence_completeness.totals.failed_count > 0 ||
-        facts.evidence_completeness.failure_reasons.length > 0) {
-        return true;
-    }
-    return facts.evidence_completeness.skip_reasons.some(({ reason }) => isRetryableEvidenceSkipReason(reason));
-}
-/**
- * Skip reasons a later sync can still turn into evidence.
- *
- * `delivery_backoff_holding` is one of them, and it has to be: a source held by
- * backoff is missing collection right now. If it did not land here the sync
- * would read clean while a transcript sat undelivered for nine days, which is
- * the exact shape of BLI-3066.
- */
-function isRetryableEvidenceSkipReason(reason) {
-    if (reason === DELIVERY_BACKOFF_HOLDING_REASON)
-        return true;
-    return /(?:read|stat|directory)_failed|session_limit_overflow/iu.test(reason);
-}
-/** Every distinct reason behind the gap, sorted, as one spool-ready label. */
-export function retryableEvidenceGapReason(facts, outcomes) {
-    const reasons = new Set();
-    for (const outcome of retryableFailedOutcomes(outcomes)) {
-        reasons.add(outcome.reason ?? "upload_failed");
-    }
-    if (facts) {
-        if (facts.deferred_byte_budget_count > 0)
-            reasons.add("deferred_byte_budget");
-        if (facts.deferred_object_budget_count > 0) {
-            reasons.add("deferred_object_budget");
-        }
-        for (const { reason } of facts.evidence_completeness.failure_reasons) {
-            reasons.add(reason);
-        }
-        if (facts.evidence_completeness.status === "failed" &&
-            facts.evidence_completeness.failure_reasons.length === 0) {
-            reasons.add("evidence_completeness_failed");
-        }
-        for (const { reason } of facts.evidence_completeness.skip_reasons) {
-            if (isRetryableEvidenceSkipReason(reason)) {
-                reasons.add(reason);
-            }
-        }
-    }
-    return `partial_raw_evidence_retry_required:${[
-        ...reasons,
-    ].sort().join(",") || "unknown"}`;
-}
-/**
- * The counts `cockpit sync --json` and `cockpit status` read.
- *
- * Two kinds of reuse are added together on purpose: files this machine skipped
- * because its own cursor already had the bytes, and files the server recognised
- * from their content hash.
- */
-export function summarizeRawEvidenceDelivery(built, outcomes, uploadedChunkCount, cursor, staging, now) {
-    const stuck = summarizeStuckEvidence(staging, now);
-    const cursorReused = built.raw_evidence_facts?.reused ?? [];
-    const serverReusedCount = outcomes.filter((outcome) => outcome.upload_state === "reused_existing").length;
-    const failed = outcomes.filter((outcome) => outcome.upload_state === "upload_failed");
-    const retryRequired = hasRetryableEvidenceGap(built.raw_evidence_facts, outcomes);
-    const retryReason = retryRequired
-        ? retryableEvidenceGapReason(built.raw_evidence_facts, outcomes)
-        : null;
-    const cursorReusedOutcomes = cursorReused.map((entry) => ({
-        object_key: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
-        raw_evidence_pointer_id: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
-        kind: entry.kind,
-        codex_session_id: entry.codex_session_id,
-        ...(entry.artifact_metadata
-            ? { artifact_metadata: entry.artifact_metadata }
-            : {}),
-        upload_state: "reused_existing",
-        reason: "cursor_content_match",
-    }));
-    return {
-        raw_evidence_file_count: built.raw_evidence_upload_files.length,
-        raw_evidence_uploaded_object_count: outcomes.filter((outcome) => outcome.upload_state === "uploaded").length,
-        raw_evidence_uploaded_chunk_count: uploadedChunkCount,
-        raw_evidence_reused_count: cursorReused.length + serverReusedCount,
-        raw_evidence_failed_count: failed.length,
-        raw_evidence_sanitized_count: built.raw_evidence_facts?.sanitized_count ?? 0,
-        raw_evidence_failure_reasons: [
-            ...new Set(failed.map((outcome) => outcome.reason ?? "unknown")),
-        ],
-        raw_evidence_retry_required: retryRequired,
-        raw_evidence_retry_reasons: retryReason
-            ? retryReason
-                .replace(/^partial_raw_evidence_retry_required:/u, "")
-                .split(",")
-                .filter(Boolean)
-            : [],
-        raw_evidence_outcomes: [
-            ...outcomes.map((outcome) => ({
-                object_key: outcome.object_key,
-                raw_evidence_pointer_id: outcome.pointer.raw_evidence_pointer_id,
-                kind: outcome.kind,
-                codex_session_id: outcome.codex_session_id,
-                ...(outcome.artifact_metadata
-                    ? { artifact_metadata: outcome.artifact_metadata }
-                    : {}),
-                upload_state: outcome.upload_state,
-                reason: outcome.reason,
-            })),
-            ...cursorReusedOutcomes,
-        ],
-        raw_evidence_deferred_byte_budget: built.raw_evidence_facts?.deferred_byte_budget_count ?? 0,
-        raw_evidence_deferred_object_budget: built.raw_evidence_facts?.deferred_object_budget_count ?? 0,
-        cursor_tracked_object_count: Object.keys(cursor.objects).length,
-        raw_evidence_delivery_held_count: outcomes.filter((outcome) => outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON).length + (built.raw_evidence_facts?.delivery_held_count ?? 0),
-        raw_evidence_stuck_object_count: stuck.stuck_object_count,
-        raw_evidence_max_delivery_attempts: stuck.max_attempts,
-        raw_evidence_oldest_delivery_failure_at: stuck.oldest_first_failed_at,
-    };
-}
-// --------------------------------------------------------------------------
-// 4. Reconciling the envelope with what actually became durable
-// --------------------------------------------------------------------------
-/**
- * Ingest refuses pointers whose objects never became durable, so failed
- * uploads are pruned from the envelope instead of failing the whole sync.
- * Successful upload responses can also carry server-side sanitized hash and
- * redaction metadata; apply those before ingest so refs describe the bytes
- * actually stored in the durable bucket.
- */
-export function applyRawEvidenceUploadOutcomes(envelope, outcomes) {
-    // Content-addressed keys mean one pointer id can carry several outcomes
-    // (byte-identical files); the pointer is durable if ANY outcome succeeded.
-    const durablePointers = new Map();
-    for (const outcome of outcomes) {
-        if (outcome.upload_state === "upload_failed")
-            continue;
-        durablePointers.set(outcome.pointer.raw_evidence_pointer_id, outcome.pointer);
-    }
-    const durablePointerIds = new Set([...durablePointers.keys()]);
-    const failedPointerIds = new Set(outcomes
-        .filter((outcome) => outcome.upload_state === "upload_failed" &&
-        !durablePointerIds.has(outcome.pointer.raw_evidence_pointer_id))
-        .map((outcome) => outcome.pointer.raw_evidence_pointer_id));
-    const failedOutcomes = outcomes.filter((outcome) => failedPointerIds.has(outcome.pointer.raw_evidence_pointer_id));
-    if (failedPointerIds.size === 0 && durablePointers.size === 0)
-        return envelope;
-    return {
-        ...envelope,
-        events: envelope.events.map((event) => ({
-            ...event,
-            metrics: failedPointerIds.size > 0
-                ? {
-                    ...event.metrics,
-                    evidence_failed_count: (event.metrics["evidence_failed_count"] ?? 0) +
-                        failedOutcomes.length,
-                }
-                : event.metrics,
-            attributes: failedPointerIds.size > 0 && event.evidence_completeness
-                ? {
-                    ...event.attributes,
-                    evidence_completeness_schema_version: event.evidence_completeness.schema_version,
-                    evidence_completeness_status: "partial",
-                    evidence_incomplete: true,
-                }
-                : event.attributes,
-            evidence_completeness: failedPointerIds.size > 0 && event.evidence_completeness
-                ? markCompletenessUploadFailures(event.evidence_completeness, failedOutcomes)
-                : event.evidence_completeness,
-            raw_evidence_pointers: event.raw_evidence_pointers.flatMap((pointer) => {
-                const pointerId = pointer.raw_evidence_pointer_id;
-                if (failedPointerIds.has(pointerId))
-                    return [];
-                return [durablePointers.get(pointerId) ?? pointer];
-            }),
-            redaction: failedPointerIds.size > 0
-                ? {
-                    ...event.redaction,
-                    raw_evidence_pointer_ids: event.redaction.raw_evidence_pointer_ids.filter((pointerId) => !failedPointerIds.has(pointerId)),
-                }
-                : event.redaction,
-        })),
-    };
-}
-/**
- * Fold upload failures back into the completeness payload the event carries, so
- * a downstream reader sees `partial` and the per-source failure counts rather
- * than a complete-looking payload with fewer pointers than it claims.
- */
-function markCompletenessUploadFailures(completeness, failedOutcomes) {
-    const failureCounts = new Map();
-    for (const outcome of failedOutcomes) {
-        const source = outcome.kind ?? "raw_evidence";
-        failureCounts.set(source, (failureCounts.get(source) ?? 0) + 1);
-    }
-    const totalFailures = [...failureCounts.values()].reduce((sum, count) => sum + count, 0);
-    const sourceCounts = [...completeness.source_counts];
-    for (const [source, count] of failureCounts) {
-        const existingIndex = sourceCounts.findIndex((entry) => entry.source === source);
-        if (existingIndex === -1) {
-            sourceCounts.push({
-                source,
-                scanned_count: 0,
-                included_count: 0,
-                skipped_count: 0,
-                truncated_count: 0,
-                deferred_count: 0,
-                reused_count: 0,
-                failed_count: count,
-            });
-            continue;
-        }
-        const existing = sourceCounts[existingIndex];
-        if (!existing)
-            continue;
-        sourceCounts[existingIndex] = {
-            ...existing,
-            failed_count: existing.failed_count + count,
-        };
-    }
-    const failureReasons = [...completeness.failure_reasons];
-    for (const [source, count] of failureCounts) {
-        const reason = "upload_failed";
-        const existingIndex = failureReasons.findIndex((entry) => entry.source === source && entry.reason === reason);
-        if (existingIndex === -1) {
-            failureReasons.push({ source, reason, count });
-        }
-        else {
-            const existing = failureReasons[existingIndex];
-            if (existing) {
-                failureReasons[existingIndex] = {
-                    ...existing,
-                    count: existing.count + count,
-                };
-            }
-        }
-    }
-    return EvidenceCompletenessPayloadSchema.parse({
-        ...completeness,
-        status: "partial",
-        source_counts: sourceCounts,
-        totals: {
-            ...completeness.totals,
-            failed_count: completeness.totals.failed_count + totalFailures,
-        },
-        failure_reasons: failureReasons.sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
-        notes: [
-            ...new Set([
-                ...completeness.notes,
-                "Some collected evidence did not become durable; downstream analysis should lower confidence.",
-            ]),
-        ],
-    });
-}
+ * This file is the table of contents; every responsibility above is a
+ * `upload-evidence-delivery-*.ts` sibling and is re-exported here so the
+ * module's public surface never moves (BLI-3637):
+ *   -offer      questions 1 and 2 — what may we offer, what just happened
+ *   -summary    question 3 — retry sources, retryable/permanent reasons, and
+ *               the counts `cockpit status` reads
+ *   -reconcile  question 4 — folding upload outcomes back into the envelope
+ *               ingest actually sees
+ */
+export { logEvidenceBackoffBypassed, logEvidenceHeldByBackoff, partitionHeldEvidenceFiles, persistDeliveryAttempts, } from "./upload-evidence-delivery-offer.js";
+export { hasRetryableEvidenceGap, logPermanentlyRejectedEvidence, permanentEvidenceFailureReason, retrySourcesForFailedSync, retryableEvidenceGapReason, summarizeRawEvidenceDelivery, } from "./upload-evidence-delivery-summary.js";
+export { applyRawEvidenceUploadOutcomes } from "./upload-evidence-delivery-reconcile.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.54",
+  "version": "0.2.56",
   "private": false,
   "type": "module",
   "bin": {
@@ -27,7 +27,7 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.2",
-    "@bli-cockpit/telemetry-core": "0.1.27"
+    "@bli-cockpit/memory-mcp": "0.1.4",
+    "@bli-cockpit/telemetry-core": "0.1.28"
   }
 }