@bli-cockpit/cli 0.2.54 → 0.2.56

package/dist/evidence-upload-terminal.js

@@ -0,0 +1,309 @@
+import { RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
+import { readBeginDispositions } from "./evidence-upload-plan.js";
+import { failedOutcome } from "./evidence-upload-object.js";
+import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
+import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
+/**
+ * Re-key and terminal outcomes: settle a re-keyable `begin` conflict
+ * (BLI-3552), tell the server about a delivery this run gave up on
+ * (BLI-2539), fold a duplicate's fate into its primary's, run the legacy
+ * single-shot fallback for a dashboard without the chunk routes, and
+ * summarize a run's outcomes into the shape every caller reads.
+ */
+/**
+ * Settle a re-keyable conflict by asking for a second name (BLI-3552).
+ *
+ * `hash_mismatch_committed_object` means the key already holds different bytes
+ * that are already durable. Re-sending is the loop; deleting the durable object
+ * to make room is destroying evidence. The third answer is to offer this
+ * content under a name derived from its own hash, which is what this does, once
+ * per object per sync.
+ *
+ * Every path that cannot get there returns the ORIGINAL pair, so the object
+ * fails on the conflict reason `begin` actually gave rather than on a label
+ * invented here. Nothing is written off permanently either way: the upload
+ * cursor only remembers successes, so the next eligible sync offers the bytes
+ * again — at delivery-backoff cadence now instead of every 15 minutes.
+ */
+export async function rekeyConflictedEntry(options, entry, disposition, chunkSizeBytes) {
+    if (disposition.disposition !== "conflict" ||
+        !isRekeyableUploadConflict(disposition.reason)) {
+        return { entry, disposition };
+    }
+    const contentHashPrefix = (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12);
+    const rekeyedPointer = rekeyedEvidencePointer(entry.file.pointer);
+    if (!rekeyedPointer) {
+        // The key already names this content and the server still says it holds
+        // something else. A rename cannot answer that; a person has to.
+        console.error("[evidence-rekey] the key already carries this content hash; leaving the conflict for a person", JSON.stringify({
+            reason: disposition.reason,
+            kind: entry.file.kind ?? "unknown",
+            content_hash_prefix: contentHashPrefix,
+            byte_size: entry.bytes.byteLength,
+        }));
+        return { entry, disposition };
+    }
+    const rekeyedEntry = {
+        ...entry,
+        file: { ...entry.file, pointer: rekeyedPointer },
+        duplicates: entry.duplicates.map((duplicate) => ({
+            ...duplicate,
+            pointer: rekeyedEvidencePointer(duplicate.pointer) ?? duplicate.pointer,
+        })),
+    };
+    const rekeyedDisposition = await beginOneObject(options, rekeyedEntry, chunkSizeBytes);
+    if (!rekeyedDisposition) {
+        return { entry, disposition };
+    }
+    console.error("[evidence-rekey] committed object holds other content; offering these bytes under their own hash", JSON.stringify({
+        reason: disposition.reason,
+        kind: entry.file.kind ?? "unknown",
+        content_hash_prefix: contentHashPrefix,
+        byte_size: entry.bytes.byteLength,
+        chunk_count: entry.chunkCount,
+        rekeyed_disposition: rekeyedDisposition.disposition,
+        rekeyed_reason: rekeyedDisposition.reason ?? "none",
+        duplicate_count: entry.duplicates.length,
+    }));
+    return { entry: rekeyedEntry, disposition: rekeyedDisposition };
+}
+/**
+ * `begin` for a single object. Null whenever the answer cannot be trusted —
+ * transport failure, an unparseable body, a duplicate or missing key, or a
+ * pointer id that is not the one we sent — and the caller then keeps the
+ * original conflict rather than acting on a guess.
+ */
+async function beginOneObject(options, entry, chunkSizeBytes) {
+    const objectKey = entry.file.pointer.object_key ?? "";
+    const response = await requestJson(options, "/api/ambient/evidence/upload/begin", {
+        schema_version: "ambient-raw-evidence-upload-begin.v1",
+        generated_at: options.generatedAt,
+        provenance: options.provenance,
+        objects: [
+            {
+                pointer: entry.file.pointer,
+                chunk_size_bytes: chunkSizeBytes,
+                chunk_count: entry.chunkCount,
+            },
+        ],
+    });
+    if (!response.ok) {
+        console.error("[evidence-rekey] begin refused the re-keyed object; keeping the original conflict", JSON.stringify({
+            reason: "rekey_begin_rejected",
+            http_status: response.status,
+            server_reason: safeFailureDetail(response.body) ?? "none",
+            kind: entry.file.kind ?? "unknown",
+        }));
+        return null;
+    }
+    const parsed = RawEvidenceUploadBeginResponseSchema.safeParse(response.body);
+    if (!parsed.success)
+        return null;
+    const dispositions = readBeginDispositions(parsed.data);
+    const disposition = dispositions?.get(objectKey);
+    if (!disposition)
+        return null;
+    if (disposition.raw_evidence_pointer_id !==
+        entry.file.pointer.raw_evidence_pointer_id) {
+        return null;
+    }
+    return disposition;
+}
+/**
+ * A duplicate file shares its primary's fate, but a primary that physically
+ * uploaded leaves the duplicate as reuse of now-durable content — counting it
+ * as a second upload would inflate object counts.
+ */
+export function duplicateOutcome(primary, duplicate) {
+    return {
+        ...primary,
+        pointer: pointerWithUploadedMetadata(duplicate.pointer, primary.pointer),
+        codex_session_id: duplicate.codex_session_id ?? null,
+        kind: duplicate.kind ?? "unknown",
+        artifact_metadata: duplicate.artifact_metadata,
+        upload_state: primary.upload_state === "uploaded"
+            ? "reused_existing"
+            : primary.upload_state,
+        reason: primary.upload_state === "uploaded" ? "duplicate_in_batch" : primary.reason,
+        uploaded_chunk_count: 0,
+    };
+}
+/**
+ * Tell the server why we gave up on a row `begin` already opened (BLI-2539).
+ *
+ * `begin` opens a ledger row per object and the chunks then go one object at a
+ * time, so one object failing leaves its siblings untouched — the production
+ * shape is a lone failure among committed rows. This function is the only thing
+ * standing between that failure and a row that sits `pending` with a null
+ * reason until a drain relabels it `staging_incomplete` weeks later, which
+ * names the shape and not the cause. 275 rows reached that state by 2026-08-14.
+ *
+ * The server records the reason on the still-open row (for a reason classified
+ * permanent it fails the row closed), so reporting a give-up never costs the
+ * staged chunks the next sync resumes from.
+ *
+ * Deliberately best-effort: the upload has already failed and the caller's
+ * outcome is the answer that matters. Losing the abort as well leaves exactly
+ * the row we had before this existed, so a delivery failure must not throw —
+ * but it is named on stderr rather than swallowed, because a reason the server
+ * never received is invisible precisely where BLI-2539 needed it visible.
+ */
+export async function reportAbandonedUpload(options, disposition, outcome) {
+    if (outcome.upload_state !== "upload_failed")
+        return;
+    if (!outcome.reason)
+        return;
+    // Only a row this call actually opened or resumed. `conflict` never allocated
+    // one for us, and `already_committed` names durable content an abort must not
+    // touch.
+    if (disposition.disposition !== "new" && disposition.disposition !== "resume") {
+        return;
+    }
+    if (!disposition.upload_id)
+        return;
+    const reason = conformReasonLabel(outcome.reason);
+    const response = await requestJson(options, "/api/ambient/evidence/upload/abort", {
+        schema_version: "ambient-raw-evidence-upload-abort.v1",
+        generated_at: options.generatedAt,
+        provenance: options.provenance,
+        upload_id: disposition.upload_id,
+        object_key: outcome.object_key,
+        reason,
+        uploaded_chunk_count: outcome.uploaded_chunk_count,
+    });
+    if (!response.ok) {
+        // status 404 is an old dashboard without the abort route; status 0 means
+        // the request itself never completed. Metadata only — ids, status, labels.
+        console.error("[evidence-abort] delivery failed; the server did not record the reason", JSON.stringify({
+            upload_id: disposition.upload_id,
+            reason,
+            http_status: response.status,
+        }));
+    }
+}
+/**
+ * Force a reason into the label shape the abort route accepts.
+ *
+ * Every reason this client composes already fits, and this exists so that stays
+ * true without anyone having to remember it. A label the server rejects comes
+ * back 400 and the reason is lost — which is the precise failure BLI-2539 is
+ * about, arriving through the code meant to fix it. A mangled label that lands
+ * beats a perfect one that does not.
+ */
+function conformReasonLabel(reason) {
+    const conformed = reason.replace(/[^a-z0-9_:.-]/gi, "_").slice(0, 120);
+    return conformed.length > 0 ? conformed : "upload_failed_unlabelled";
+}
+/**
+ * Older dashboards predate the chunk endpoints. Fall back to one legacy
+ * request per file so an oversized or already-existing object fails (or
+ * reuses) individually instead of dragging the whole batch down. Large files
+ * are attempted rather than pre-failed: a non-serverless dashboard (the
+ * default localhost deployment) accepts them like the old CLI could, and a
+ * platform body-limit rejection comes back as a labeled per-file failure.
+ */
+export async function uploadWithLegacyFallback(options, loaded, outcomes) {
+    for (const entry of loaded) {
+        let outcome;
+        const response = await requestJson(options, "/api/ambient/evidence/upload", {
+            schema_version: "ambient-raw-evidence-upload.v1",
+            generated_at: options.generatedAt,
+            provenance: options.provenance,
+            files: [
+                {
+                    pointer: entry.file.pointer,
+                    content_base64: entry.bytes.toString("base64"),
+                },
+            ],
+        });
+        if (response.ok) {
+            const parsedLegacy = RawEvidenceLegacyUploadResponseSchema.safeParse(response.body);
+            if (!parsedLegacy.success) {
+                outcome = failedOutcome(entry.file, "legacy_upload_invalid_response");
+            }
+            else {
+                const receipt = parsedLegacy.data.uploaded.find((candidate) => candidate.raw_evidence_pointer_id ===
+                    entry.file.pointer.raw_evidence_pointer_id);
+                if (!receipt ||
+                    parsedLegacy.data.bucket !== entry.file.pointer.storage_bucket ||
+                    !legacyUploadReceiptMatchesPointer(entry.file.pointer, receipt)) {
+                    outcome = failedOutcome(entry.file, "legacy_upload_receipt_mismatch");
+                }
+                else {
+                    const pointer = pointerWithLegacyUploadResponse(entry.file.pointer, parsedLegacy.data);
+                    outcome = {
+                        pointer,
+                        object_key: entry.file.pointer.object_key ?? "",
+                        codex_session_id: entry.file.codex_session_id ?? null,
+                        kind: entry.file.kind ?? "unknown",
+                        artifact_metadata: entry.file.artifact_metadata,
+                        upload_state: "uploaded",
+                        reason: "legacy_single_shot_upload",
+                        uploaded_chunk_count: 1,
+                    };
+                }
+            }
+        }
+        else if (response.status === 409) {
+            // The pre-ledger route cannot prove that a conflicting object contains
+            // these exact bytes. Fail closed instead of seeding the cursor with an
+            // unverified pointer.
+            outcome = failedOutcome(entry.file, "legacy_upload_conflict_http_409");
+        }
+        else {
+            outcome = failedOutcome(entry.file, `legacy_upload_failed_http_${response.status}`);
+        }
+        outcomes.push(outcome);
+        for (const duplicate of entry.duplicates) {
+            outcomes.push(duplicateOutcome(outcome, duplicate));
+        }
+    }
+    return summarizeOutcomes(outcomes, true);
+}
+function pointerWithLegacyUploadResponse(pointer, body) {
+    const entry = body.uploaded.find((candidate) => candidate.raw_evidence_pointer_id === pointer.raw_evidence_pointer_id);
+    if (!entry)
+        return pointer;
+    return pointerWithUploadedMetadata(pointer, {
+        content_hash_sha256: entry.content_hash_sha256,
+        byte_size: entry.byte_size,
+        redaction: entry.redaction,
+    });
+}
+function legacyUploadReceiptMatchesPointer(pointer, receipt) {
+    if (receipt.object_key !== pointer.object_key)
+        return false;
+    if (receipt.redaction) {
+        return (receipt.redaction.original_content_hash_sha256 ===
+            pointer.content_hash_sha256 &&
+            receipt.redaction.original_byte_size === pointer.byte_size &&
+            receipt.redaction.sanitized_content_hash_sha256 ===
+                receipt.content_hash_sha256 &&
+            receipt.redaction.sanitized_byte_size === receipt.byte_size);
+    }
+    return (receipt.content_hash_sha256 === pointer.content_hash_sha256 &&
+        receipt.byte_size === pointer.byte_size);
+}
+function pointerWithUploadedMetadata(pointer, metadata) {
+    const redaction = RawEvidenceRedactionMetadataSchema.safeParse(metadata.redaction);
+    return {
+        ...pointer,
+        content_hash_sha256: typeof metadata.content_hash_sha256 === "string"
+            ? metadata.content_hash_sha256
+            : pointer.content_hash_sha256,
+        byte_size: typeof metadata.byte_size === "number" ? metadata.byte_size : pointer.byte_size,
+        redaction: redaction.success ? redaction.data : pointer.redaction,
+    };
+}
+export function summarizeOutcomes(outcomes, usedLegacyFallback) {
+    const uploaded = outcomes.filter((outcome) => outcome.upload_state === "uploaded");
+    return {
+        outcomes,
+        uploaded_object_keys: uploaded.map((outcome) => outcome.object_key),
+        uploaded_object_count: uploaded.length,
+        uploaded_chunk_count: outcomes.reduce((sum, outcome) => sum + outcome.uploaded_chunk_count, 0),
+        reused_count: outcomes.filter((outcome) => outcome.upload_state === "reused_existing").length,
+        failed_count: outcomes.filter((outcome) => outcome.upload_state === "upload_failed").length,
+        used_legacy_fallback: usedLegacyFallback,
+    };
+}

package/dist/evidence-upload-transport.js

@@ -0,0 +1,104 @@
+import crypto from "node:crypto";
+import { describeError } from "./health-detail.js";
+/**
+ * The `send` stage: one retrying POST to an evidence-upload route, plus the
+ * response-body plumbing every later stage reads a reason off of.
+ *
+ * Raw content is never printed, diffed, or summarized here — only hashes,
+ * sizes, keys, and reason labels leave this module.
+ */
+const DEFAULT_MAX_ATTEMPTS = 3;
+const RETRY_DELAY_MS = 250;
+export async function requestJson(options, routePath, body) {
+    const maxAttempts = options.maxAttemptsPerRequest ?? DEFAULT_MAX_ATTEMPTS;
+    const sleep = options.sleep ?? defaultSleep;
+    let lastStatus = 0;
+    let lastBody = null;
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+        try {
+            const response = await options.fetchImpl(`${options.dashboardUrl}${routePath}`, {
+                method: "POST",
+                headers: {
+                    "Authorization": `Bearer ${options.deviceToken}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify(body),
+            });
+            lastStatus = response.status;
+            lastBody = await readResponseJson(response);
+            if (response.ok || (response.status < 500 && response.status !== 429)) {
+                return { ok: response.ok, status: response.status, body: lastBody };
+            }
+        }
+        catch (error) {
+            // `status: 0` is this function's word for "no answer came back", and it
+            // travels all the way to the upload ledger without ever saying whether
+            // the request left the machine. This is the exact silence BLI-2528 sat
+            // behind for 57 days; the route is named, the body never is.
+            console.error("[evidence-upload] request failed before a status came back", JSON.stringify({
+                reason: "upload_request_transport_error",
+                route: routePath,
+                attempt,
+                max_attempts: maxAttempts,
+                ...describeError(error),
+            }));
+            lastStatus = 0;
+            lastBody = null;
+        }
+        if (attempt < maxAttempts)
+            await sleep(RETRY_DELAY_MS * attempt);
+    }
+    return { ok: false, status: lastStatus, body: lastBody };
+}
+/**
+ * Marks a body the server did not produce as JSON.
+ *
+ * A dashboard route always answers with a JSON envelope, so an HTML body on a
+ * 500 means the response came from the platform's error page and not from the
+ * route — the process died before any handler ran. That distinction is the
+ * whole difference between "the commit rejected these bytes" and "the commit
+ * never got to decide", and it was invisible for the nine days of BLI-3067
+ * because both collapsed into `commit_failed_http_500`.
+ */
+export const NON_JSON_RESPONSE_BODY_KEY = "__cockpit_response_body_format";
+export function isNonJsonResponseBody(body) {
+    return (!!body &&
+        typeof body === "object" &&
+        body[NON_JSON_RESPONSE_BODY_KEY] === "non_json");
+}
+async function readResponseJson(response) {
+    const text = await response.text();
+    if (!text)
+        return {};
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        // The raw text is kept for the caller that wants to show it, never for a
+        // label or a log — it is an unbounded HTML page. Its SHAPE is safe and is
+        // the part that matters: a non-JSON body means something in front of the
+        // dashboard answered instead of it (BLI-3067, BLI-3238).
+        console.error("[evidence-upload] reply was not JSON", JSON.stringify({
+            reason: "response_body_not_json",
+            http_status: response.status,
+            byte_size: text.length,
+            content_type: response.headers.get("content-type") ?? "none",
+        }));
+        return { message: text, [NON_JSON_RESPONSE_BODY_KEY]: "non_json" };
+    }
+}
+/** The server's `{ reason }` field, sanity-checked before it is trusted as a label. */
+export function safeFailureDetail(body) {
+    if (!body || typeof body !== "object")
+        return null;
+    const reason = body.reason;
+    return typeof reason === "string" && /^[a-z0-9_:-]{1,80}$/i.test(reason)
+        ? reason
+        : null;
+}
+function defaultSleep(milliseconds) {
+    return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+export function sha256(value) {
+    return crypto.createHash("sha256").update(value).digest("hex");
+}

package/dist/spool/local-spool-io.js

@@ -0,0 +1,122 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { parseUploadSpoolState } from "./local-spool-parse.js";
+import { SPOOL_STATE_FILENAME, emptyUploadSpoolState, } from "./local-spool-types.js";
+/**
+ * Reads, writes, and summarizes the spool file on disk. The write path is
+ * durable-write-then-atomic-rename (temp file, fsync, rename, best-effort
+ * directory fsync) so a crash mid-write never leaves a half-written spool
+ * that a later read would have to guess about.
+ */
+export async function readLocalUploadSpoolState(paths) {
+    const filePath = uploadSpoolStatePath(paths);
+    let serialized;
+    try {
+        serialized = await fs.readFile(filePath, "utf8");
+    }
+    catch (error) {
+        if (isErrnoException(error, "ENOENT"))
+            return emptyUploadSpoolState();
+        throw uploadSpoolReadError(filePath, "unreadable", error);
+    }
+    let raw;
+    try {
+        raw = JSON.parse(serialized);
+    }
+    catch (error) {
+        throw uploadSpoolReadError(filePath, "invalid JSON", error);
+    }
+    try {
+        return parseUploadSpoolState(raw);
+    }
+    catch (error) {
+        throw uploadSpoolReadError(filePath, "invalid schema", error);
+    }
+}
+export async function summarizeLocalUploadSpool(paths) {
+    const state = await readLocalUploadSpoolState(paths);
+    return {
+        last_upload_attempt_at: state.last_upload_attempt_at,
+        last_upload_success_at: state.last_upload_success_at,
+        last_upload_failure_reason: state.last_upload_failure_reason,
+        pending_upload_count: state.pending_uploads.length +
+            state.pending_source_retries.length +
+            state.pending_session_reports.length,
+        retry_command: state.pending_uploads[0]?.retry_command ??
+            (state.pending_source_retries.length > 0 ||
+                state.pending_session_reports.length > 0
+                ? "cockpit sync"
+                : null),
+    };
+}
+export async function writeUploadSpoolState(paths, state) {
+    const filePath = uploadSpoolStatePath(paths);
+    const directoryPath = path.dirname(filePath);
+    const serialized = serializeUploadSpoolState(state);
+    const tempPath = path.join(directoryPath, `.${path.basename(filePath)}.${process.pid}.${crypto.randomUUID()}.tmp`);
+    await fs.mkdir(directoryPath, { recursive: true, mode: 0o700 });
+    let handle = null;
+    let tempCreated = false;
+    let renamed = false;
+    try {
+        handle = await fs.open(tempPath, "wx", 0o600);
+        tempCreated = true;
+        if (process.platform !== "win32") {
+            await handle.chmod(0o600);
+        }
+        await handle.writeFile(serialized, "utf8");
+        await handle.sync();
+        await handle.close();
+        handle = null;
+        await fs.rename(tempPath, filePath);
+        renamed = true;
+        await fsyncDirectoryBestEffort(directoryPath);
+    }
+    catch (error) {
+        await handle?.close().catch(() => undefined);
+        if (tempCreated && !renamed) {
+            await fs.unlink(tempPath).catch(() => undefined);
+        }
+        throw error;
+    }
+}
+function serializeUploadSpoolState(state) {
+    const json = JSON.stringify(state);
+    const canonical = parseUploadSpoolState(JSON.parse(json));
+    return `${JSON.stringify(canonical, null, 2)}\n`;
+}
+function uploadSpoolStatePath(paths) {
+    return path.join(paths.spool_dir, SPOOL_STATE_FILENAME);
+}
+function isErrnoException(error, code) {
+    return (error instanceof Error &&
+        "code" in error &&
+        error.code === code);
+}
+function uploadSpoolReadError(filePath, classification, cause) {
+    const detail = cause &&
+        typeof cause === "object" &&
+        "code" in cause &&
+        typeof cause.code === "string"
+        ? ` (${cause.code})`
+        : "";
+    return new Error(`Upload spool state at ${filePath} is ${classification}; refusing to treat pending delivery state as empty.${detail}`);
+}
+async function fsyncDirectoryBestEffort(directoryPath) {
+    let handle = null;
+    try {
+        handle = await fs.open(directoryPath, "r");
+        await handle.sync();
+    }
+    catch {
+        // Directory fsync is not supported by every host/filesystem (notably some
+        // Windows versions). The file itself was fsynced before the atomic rename.
+        // Deliberately silent (BLI-3238): on those hosts this fails on every
+        // single write, so a line here would be pure noise on exactly the
+        // platform the collector most needs readable logs on.
+    }
+    finally {
+        await handle?.close().catch(() => undefined);
+    }
+}