@bli-cockpit/cli 0.2.55 → 0.2.57

package/dist/upload.js

@@ -1,402 +1,30 @@
-import { getCollectorRuntimePaths } from "./local-state.js";
-import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
-import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
-import { readRawEvidenceStagingState } from "./raw-evidence-staging.js";
-import { recordUploadBlocked, recordUploadFailure, recordUploadSuccess, } from "./spool/local-spool.js";
-import { buildLocalAmbientEnvelope, LocalUploadBlockedError, } from "./upload-envelope.js";
-import { applyRawEvidenceUploadOutcomes, hasRetryableEvidenceGap, logEvidenceBackoffBypassed, logEvidenceHeldByBackoff, logPermanentlyRejectedEvidence, partitionHeldEvidenceFiles, permanentEvidenceFailureReason, persistDeliveryAttempts, retrySourcesForFailedSync, retryableEvidenceGapReason, summarizeRawEvidenceDelivery, } from "./upload-evidence-delivery.js";
-import { reportAgentImageArtifacts } from "./upload-agent-artifacts.js";
-import { describeError } from "./health-detail.js";
-import { isNonEmptyString, readResponseJson, serverFailureDetail, } from "./upload-http.js";
-import { SyncDeliveryError, classifySyncFailure, } from "./upload-failure-reason.js";
-export { LocalUploadBlockedError, buildLocalAmbientEnvelope, } from "./upload-envelope.js";
-export { partitionHeldEvidenceFiles } from "./upload-evidence-delivery.js";
-export { reportAgentImageArtifacts } from "./upload-agent-artifacts.js";
-export { flushPendingCodexSessionReports, postCodexSessionReport, queueCodexSessionReport, reportCodexSessionAttributions, } from "./upload-session-reports.js";
-export async function syncLocalAmbientEnvelope(options = {}) {
-    const attemptedAt = (options.now ?? new Date()).toISOString();
-    const paths = getCollectorRuntimePaths(options.homeDir);
-    const cursor = await readRawEvidenceCursor(paths);
-    const built = await buildEnvelopeOrRecordBlocker(paths, options, cursor.objects, attemptedAt);
-    const fetchImpl = options.fetch ?? globalThis.fetch;
-    if (!fetchImpl) {
-        throw new Error("global fetch is unavailable; use Node.js 20 or newer.");
-    }
-    const provenance = built.envelope.work_context.provenance;
-    if (!provenance) {
-        throw new Error("Ambient upload requires collector provenance.");
-    }
-    // Assigned progressively so the spool path below can still report what did
-    // land if the upload pass throws part-way through.
-    let uploadOutcomes = [];
-    let uploadedChunkCount = 0;
-    const attemptedDate = options.now ?? new Date(attemptedAt);
-    const staging = await readRawEvidenceStagingState(paths.state_dir);
-    try {
-        if (built.raw_evidence_upload_files.length > 0) {
-            // Objects still inside their backoff window are not offered at all. They
-            // become held outcomes below, which keeps them out of the envelope, keeps
-            // the sync in retry_pending, and names the reason — the 559th identical
-            // attempt is what BLI-3066 made impossible. An operator-invoked retry
-            // (`cockpit backfill`, doctor) offers them anyway and says so: the window
-            // is the scheduler's cadence, not an answer to a person asking now
-            // (BLI-3118).
-            const { deliverable, held, bypassed } = partitionHeldEvidenceFiles(built.raw_evidence_upload_files, staging, attemptedDate, options.evidenceDeliveryMode);
-            logEvidenceHeldByBackoff(held, attemptedAt);
-            logEvidenceBackoffBypassed(bypassed, attemptedAt);
-            uploadOutcomes = held;
-            if (deliverable.length > 0) {
-                const upload = await uploadRawEvidenceFilesChunked({
-                    fetchImpl,
-                    dashboardUrl: built.dashboard_url,
-                    deviceToken: built.device_token,
-                    provenance,
-                    generatedAt: built.envelope.generated_at,
-                    files: deliverable,
-                });
-                uploadOutcomes = [...held, ...upload.outcomes];
-                uploadedChunkCount = upload.uploaded_chunk_count;
-            }
-            await persistDeliveryAttempts(paths.state_dir, staging, uploadOutcomes, attemptedDate);
-        }
-        const response = await postEnvelopeToIngest({
-            fetchImpl,
-            built,
-            envelope: applyRawEvidenceUploadOutcomes(built.envelope, uploadOutcomes),
-        });
-        advanceRawEvidenceCursor(cursor, uploadOutcomes, attemptedAt);
-        await writeRawEvidenceCursor(paths, cursor);
-        await reportAgentImageArtifacts({
-            fetchImpl,
-            dashboardUrl: built.dashboard_url,
-            deviceToken: built.device_token,
-            provenance,
-            generatedAt: built.envelope.generated_at,
-            ticketId: built.ticket_id,
-            repoFingerprint: built.envelope.work_context.repo_fingerprint,
-            worktreeFingerprint: built.envelope.work_context.worktree_fingerprint,
-            outcomes: uploadOutcomes,
-            reused: built.raw_evidence_facts?.reused ?? [],
-            cursorObjects: cursor.objects,
-        }).catch((error) => {
-            // The whole point of `agent_artifact_report_local_error` is that it is
-            // NOT the network error the reporter already labels itself — it is a
-            // throw from our own code on the way in. That distinction is only
-            // useful if the throw says what it was (BLI-3238).
-            console.error("[cockpit-sync] agent image artifact report threw locally", JSON.stringify({
-                reason: "agent_artifact_report_local_error",
-                ...describeError(error),
-            }));
-            return {
-                posted: false,
-                reason: "agent_artifact_report_local_error",
-                recorded_count: 0,
-            };
-        });
-        await recordIngestedSyncOutcome({
-            paths,
-            options,
-            built,
-            uploadOutcomes,
-            attemptedAt,
-        });
-        // The success branch logs too. A line that only fires on failure cannot
-        // answer "did anything land at all today?", which is the question that
-        // would have caught BLI-2528 in a week instead of 57 days.
-        console.error("[cockpit-sync] ingest accepted the envelope", JSON.stringify({
-            reason: "ingest_accepted",
-            http_status: response.status,
-            event_count: built.event_count,
-            raw_evidence_file_count: built.raw_evidence_upload_files.length,
-            uploaded_chunk_count: uploadedChunkCount,
-            failed_object_count: uploadOutcomes.filter((outcome) => outcome.upload_state === "upload_failed").length,
-        }));
-        return {
-            status: "uploaded",
-            dashboard_url: built.dashboard_url,
-            ticket_id: built.ticket_id,
-            work_context_id: built.envelope.work_context.work_context_id,
-            head_sha: built.head_sha,
-            event_count: built.event_count,
-            source_scan_count: built.source_scan_count,
-            risk_flag_count: built.risk_flag_count,
-            http_status: response.status,
-            ...summarizeRawEvidenceDelivery(built, uploadOutcomes, uploadedChunkCount, cursor, staging, attemptedDate),
-        };
-    }
-    catch (error) {
-        // The spool row used to store `error.message` verbatim, so every distinct
-        // server sentence became its own "reason" and nothing could count how many
-        // machines were failing the same way (BLI-3483). It now stores a label from
-        // a closed set plus a bounded, redacted cause.
-        const classified = classifySyncFailure(error);
-        // A committed content-addressed object may be shared by concurrent syncs.
-        // Never delete it from this error path: an ingest retry can reuse it, while
-        // client-side cleanup cannot prove exclusive ownership without racing a
-        // second sync that is about to index the same object.
-        const spooledFailureReason = classified.failure_reason;
-        console.error("[cockpit-sync] sync spooled for retry", JSON.stringify({
-            reason: classified.reason,
-            ...(classified.http_status === null
-                ? {}
-                : { http_status: classified.http_status }),
-            ...(classified.detail ? { detail: classified.detail } : {}),
-            event_count: built.event_count,
-            raw_evidence_file_count: built.raw_evidence_upload_files.length,
-            uploaded_chunk_count: uploadedChunkCount,
-        }));
-        const entry = await recordUploadFailure(paths, {
-            last_attempt_at: attemptedAt,
-            dashboard_url: built.dashboard_url,
-            work_context_id: built.envelope.work_context.work_context_id,
-            ticket_id: built.ticket_id,
-            repo_label: built.repo_label,
-            branch: built.envelope.work_context.branch,
-            event_count: built.event_count,
-            source_scan_count: built.source_scan_count,
-            risk_flag_count: built.risk_flag_count,
-            raw_evidence_file_count: built.raw_evidence_upload_files.length,
-            retry_sources: retrySourcesForFailedSync(options, built.raw_evidence_facts),
-            failure_reason: spooledFailureReason,
-            retry_command: "cockpit sync",
-        });
-        return {
-            status: "spooled",
-            dashboard_url: built.dashboard_url,
-            ticket_id: built.ticket_id,
-            work_context_id: built.envelope.work_context.work_context_id,
-            head_sha: built.head_sha,
-            event_count: built.event_count,
-            source_scan_count: built.source_scan_count,
-            risk_flag_count: built.risk_flag_count,
-            failure_reason: spooledFailureReason,
-            failure_class: classified.reason,
-            failure_http_status: classified.http_status,
-            spool_entry_id: entry.spool_id,
-            retry_command: entry.retry_command,
-            ...summarizeRawEvidenceDelivery(built, uploadOutcomes, uploadedChunkCount, cursor, staging, attemptedDate),
-        };
-    }
-}
 /**
- * Assemble the envelope, and if this machine simply cannot upload, say so in
- * the spool before the error leaves. A blocker that only ever surfaced as a
- * thrown exception would be invisible to the next `cockpit status`.
- */
-async function buildEnvelopeOrRecordBlocker(paths, options, cursorObjects, attemptedAt) {
-    try {
-        return await buildLocalAmbientEnvelope({
-            ...options,
-            cursorObjects: options.cursorObjects ?? cursorObjects,
-        });
-    }
-    catch (error) {
-        if (error instanceof LocalUploadBlockedError) {
-            await recordUploadBlocked(paths, {
-                attemptedAt,
-                reason: error.message,
-            });
-        }
-        throw error;
-    }
-}
-/**
- * POST the envelope and refuse to believe an unreadable answer.
+ * One ambient sync, end to end.
  *
- * A 2xx may have persisted the envelope even when a proxy or an incompatible
- * dashboard mangles the receipt. Uploaded objects are kept in that ambiguous
- * case — they are durable either way — but the caller must not advance cursors
- * or clear retry state until the first-party route proves the exact submitted
- * row counts, which is what the receipt check demands.
- */
-async function postEnvelopeToIngest(options) {
-    let response;
-    try {
-        response = await options.fetchImpl(`${options.built.dashboard_url}/api/ambient/ingest`, {
-            method: "POST",
-            headers: {
-                "Authorization": `Bearer ${options.built.device_token}`,
-                "Content-Type": "application/json",
-            },
-            body: JSON.stringify(options.envelope),
-        });
-    }
-    catch (error) {
-        // No status came back at all, so no server-side reason exists to read. The
-        // error's own name and code (`TypeError` / `ENOTFOUND` / `ECONNREFUSED`)
-        // are the whole answer, and they are what separates "this machine is
-        // offline" from "the dashboard refused us" — two words that used to be the
-        // same spool row.
-        const described = describeError(error);
-        console.error("[cockpit-sync] ingest request failed before a status came back", JSON.stringify({ reason: "ingest_transport_error", ...described }));
-        throw new SyncDeliveryError("ingest_transport_error", {
-            detail: [
-                described.error_name,
-                described.error_code,
-                described.cause_code ?? described.cause_name,
-                described.error_detail,
-            ]
-                .filter(Boolean)
-                .join(" "),
-            cause: error,
-        });
-    }
-    const responseBody = await readResponseJson(response);
-    if (!response.ok) {
-        const serverReason = serverFailureDetail(responseBody);
-        console.error("[cockpit-sync] ingest refused the envelope", JSON.stringify({
-            reason: "ingest_rejected",
-            http_status: response.status,
-            server_reason: serverReason ?? "none",
-            event_count: options.envelope.events.length,
-        }));
-        throw new SyncDeliveryError("ingest_rejected", {
-            httpStatus: response.status,
-            detail: `http ${response.status}; server reason ${serverReason ?? "none"}`,
-        });
-    }
-    assertAmbientIngestReceipt(response, responseBody, options.envelope);
-    return response;
-}
-/**
- * Remember every object that is now durable, so the next sync can reuse the
- * bytes instead of re-uploading them. A failed upload is deliberately absent:
- * recording it would make the next sync skip a file that never landed.
- */
-function advanceRawEvidenceCursor(cursor, outcomes, attemptedAt) {
-    for (const outcome of outcomes) {
-        if (outcome.upload_state === "upload_failed")
-            continue;
-        const contentHash = outcome.pointer.content_hash_sha256;
-        if (!contentHash)
-            continue;
-        markObjectCommitted(cursor, contentHash, {
-            object_key: outcome.object_key,
-            byte_size: outcome.pointer.byte_size ?? 0,
-            committed_at: attemptedAt,
-        });
-    }
-    cursor.updated_at = attemptedAt;
-}
-/**
- * What the operator is owed after ingest accepted the envelope, as a decision
- * table on the evidence this sync left behind:
+ * This file is the delivery half of a sync. `upload-envelope.ts` assembles what
+ * gets sent; the siblings below are about getting it there and telling the
+ * truth about what happened:
  *
- * | evidence state                        | ledger                              |
- * |---------------------------------------|-------------------------------------|
- * | a gap a later sync could rescue        | success, then a queued retry naming why |
- * | a gap no retry can rescue             | success, then a blocker naming why, plus a stderr record |
- * | nothing missing                       | success, pending state cleared      |
+ * - `upload-sync.ts` — `syncLocalAmbientEnvelope`, the five-step sequence:
+ *   assemble the envelope (or record why this machine cannot upload at all),
+ *   deliver raw evidence objects, POST to ingest, advance the cursor and write
+ *   the ledger row, and on any failure spool a retry that carries its own
+ *   reason.
+ * - `upload-ingest-receipt.ts` — POST the envelope and refuse to believe a
+ *   receipt we cannot read: `postEnvelopeToIngest` and the row-count proof
+ *   behind it.
  *
- * The middle row is the one that matters. A permanently rejected object gets no
- * retry — nothing would change — but it must still be named, or `cockpit status`
- * reads clean on a machine with missing collection (BLI-2528).
- */
-async function recordIngestedSyncOutcome(options) {
-    const { paths, built, uploadOutcomes, attemptedAt } = options;
-    const retryableEvidenceGap = hasRetryableEvidenceGap(built.raw_evidence_facts, uploadOutcomes);
-    await recordUploadSuccess(paths, {
-        attemptedAt,
-        workContextId: built.envelope.work_context.work_context_id,
-        clearPendingForContext: !retryableEvidenceGap,
-    });
-    if (retryableEvidenceGap) {
-        await recordUploadFailure(paths, {
-            last_attempt_at: attemptedAt,
-            dashboard_url: built.dashboard_url,
-            work_context_id: built.envelope.work_context.work_context_id,
-            ticket_id: built.ticket_id,
-            repo_label: built.repo_label,
-            branch: built.envelope.work_context.branch,
-            event_count: built.event_count,
-            source_scan_count: built.source_scan_count,
-            risk_flag_count: built.risk_flag_count,
-            raw_evidence_file_count: built.raw_evidence_upload_files.length,
-            retry_sources: retrySourcesForFailedSync(options.options, built.raw_evidence_facts),
-            failure_reason: retryableEvidenceGapReason(built.raw_evidence_facts, uploadOutcomes),
-            retry_command: "cockpit sync",
-        });
-        return;
-    }
-    const permanentReason = permanentEvidenceFailureReason(uploadOutcomes);
-    if (!permanentReason)
-        return;
-    await recordUploadBlocked(paths, {
-        attemptedAt,
-        reason: permanentReason,
-    });
-    logPermanentlyRejectedEvidence({
-        reason: permanentReason,
-        outcomes: uploadOutcomes,
-        attemptedAt,
-    });
-}
-/**
- * The receipt that proves ingest persisted exactly what was submitted.
+ * The whole file is built around one rule from the fleet contract: a sync that
+ * did not collect everything must never read green. Objects held by backoff are
+ * reported as named failures, permanent rejections are logged and blocked
+ * rather than silently retried forever, and the ledger row always says why.
  *
- * Anything short of a 202 whose counts match the envelope's own totals is
- * treated as no receipt at all — a proxy's 200, a truncated body, or a
- * dashboard that accepted fewer rows than were sent all land here. They used to
- * land here under the SAME sentence, which is the BLI-3483 complaint: a
- * network appliance answering on the dashboard's behalf and a dashboard that
- * genuinely persisted three of four rows are opposite repairs. They are now two
- * reasons, and the second one names the field that disagreed and by how much.
- */
-function assertAmbientIngestReceipt(response, value, envelope) {
-    if (response.status !== 202 || !value || typeof value !== "object") {
-        const detail = response.status !== 202
-            ? `http ${response.status}; the ingest route answers 202, so something in front of it replied`
-            : "202 with a body that is not an object";
-        console.error("[cockpit-sync] ingest answered without a durable receipt", JSON.stringify({
-            reason: "ingest_receipt_not_202",
-            http_status: response.status,
-            body_is_object: Boolean(value) && typeof value === "object",
-        }));
-        throw new SyncDeliveryError("ingest_receipt_not_202", {
-            httpStatus: response.status,
-            detail,
-        });
-    }
-    const record = value;
-    const ingest = record["ingest"] && typeof record["ingest"] === "object"
-        ? record["ingest"]
-        : null;
-    const expectedFactCount = envelope.events.length;
-    const expectedRiskFlagCount = envelope.events.reduce((total, event) => total + event.risk_flags.length, 0);
-    const expectedEvidenceRefCount = envelope.events.reduce((total, event) => total + event.raw_evidence_pointers.length, 0);
-    const mismatches = [];
-    if (record["ok"] !== true)
-        mismatches.push("ok not true");
-    if (!ingest)
-        mismatches.push("no ingest block");
-    if (ingest && !isNonEmptyString(ingest["work_session_id"])) {
-        mismatches.push("blank work_session_id");
-    }
-    if (ingest) {
-        mismatches.push(...countMismatch("fact_count", ingest["fact_count"], expectedFactCount), ...countMismatch("risk_flag_count", ingest["risk_flag_count"], expectedRiskFlagCount), ...countMismatch("evidence_ref_count", ingest["evidence_ref_count"], expectedEvidenceRefCount));
-    }
-    if (mismatches.length === 0)
-        return;
-    console.error("[cockpit-sync] ingest receipt does not match what was submitted", JSON.stringify({
-        reason: "ingest_receipt_incomplete",
-        http_status: response.status,
-        mismatch_count: mismatches.length,
-        mismatches,
-    }));
-    throw new SyncDeliveryError("ingest_receipt_incomplete", {
-        httpStatus: response.status,
-        detail: mismatches.join("; "),
-    });
-}
-/**
- * `submitted 4, receipt 3` — the sentence that tells partial persistence from a
- * receipt that never carried the field at all. Counts are numbers this
- * collector computed and numbers the server echoed; neither is content.
+ * The public surface of the collector's upload path is re-exported below, so
+ * every consumer still imports from `./upload.js` regardless of which module a
+ * symbol now lives in.
  */
-function countMismatch(field, received, expected) {
-    if (received === expected)
-        return [];
-    return [
-        `${field} submitted ${expected}, receipt ${typeof received === "number" ? received : "absent"}`,
-    ];
-}
+export { LocalUploadBlockedError, buildLocalAmbientEnvelope, } from "./upload-envelope.js";
+export { partitionHeldEvidenceFiles } from "./upload-evidence-delivery.js";
+export { reportAgentImageArtifacts } from "./upload-agent-artifacts.js";
+export { flushPendingCodexSessionReports, postCodexSessionReport, queueCodexSessionReport, reportCodexSessionAttributions, } from "./upload-session-reports.js";
+export { syncLocalAmbientEnvelope } from "./upload-sync.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.55",
+  "version": "0.2.57",
   "private": false,
   "type": "module",
   "bin": {
@@ -27,7 +27,8 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.3",
+    "@bli-cockpit/memory-mcp": "0.1.5",
+    "@bli-cockpit/mcp": "0.1.0",
     "@bli-cockpit/telemetry-core": "0.1.28"
   }
 }