@bli-cockpit/cli 0.2.53 → 0.2.55

package/dist/evidence-upload-object.js

@@ -0,0 +1,181 @@
+import { COMMIT_CRASHED_PLATFORM, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
+import { isNonJsonResponseBody, requestJson, safeFailureDetail, sha256, } from "./evidence-upload-transport.js";
+/**
+ * The `chunk` and `commit` stages for one already-`begin`-accepted object:
+ * send every chunk the server does not already have, then commit and turn
+ * the receipt into a `ChunkedUploadFileOutcome`.
+ */
+export async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
+    const objectKey = entry.file.pointer.object_key ?? "";
+    if (disposition.disposition === "already_committed") {
+        if (!disposition.upload_id) {
+            return failedOutcome(entry.file, "begin_committed_receipt_unavailable");
+        }
+    }
+    if (disposition.disposition === "conflict" || !disposition.upload_id) {
+        return failedOutcome(entry.file, disposition.reason ?? "upload_conflict");
+    }
+    const chunked = await uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey);
+    if (!chunked.ok)
+        return chunked.outcome;
+    return commitUploadedObject(options, entry, disposition, objectKey, chunked.uploadedChunks);
+}
+async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey) {
+    const received = new Set(disposition.received_chunk_indexes);
+    let uploadedChunks = 0;
+    if (disposition.disposition === "already_committed" || disposition.commit_ready) {
+        return { ok: true, uploadedChunks };
+    }
+    for (let index = 0; index < entry.chunkCount; index += 1) {
+        if (received.has(index))
+            continue;
+        const chunk = entry.bytes.subarray(index * chunkSizeBytes, Math.min((index + 1) * chunkSizeBytes, entry.bytes.byteLength));
+        const chunkResponse = await requestJson(options, "/api/ambient/evidence/upload/chunk", {
+            schema_version: "ambient-raw-evidence-upload-chunk.v1",
+            generated_at: options.generatedAt,
+            provenance: options.provenance,
+            upload_id: disposition.upload_id,
+            object_key: objectKey,
+            chunk_index: index,
+            chunk_count: entry.chunkCount,
+            chunk_hash_sha256: sha256(chunk),
+            content_base64: chunk.toString("base64"),
+        });
+        if (!chunkResponse.ok) {
+            // Same treatment the commit path has had since BLI-2528: the server's
+            // own reason rides on the label, so `chunk_3_failed_http_413` becomes
+            // `chunk_3_failed_http_413_object_too_large` and the ledger row names
+            // the cause instead of the transport (BLI-3483).
+            const chunkDetail = safeFailureDetail(chunkResponse.body);
+            console.error("[evidence-upload] chunk rejected", JSON.stringify({
+                reason: "chunk_rejected",
+                upload_id: disposition.upload_id,
+                http_status: chunkResponse.status,
+                server_reason: chunkDetail ?? "none",
+                chunk_index: index,
+                chunk_count: entry.chunkCount,
+                uploaded_chunk_count: uploadedChunks,
+            }));
+            return {
+                ok: false,
+                outcome: failedOutcome(entry.file, `chunk_${index}_failed_http_${chunkResponse.status}${chunkDetail ? `_${chunkDetail}` : ""}`, uploadedChunks),
+            };
+        }
+        uploadedChunks += 1;
+    }
+    return { ok: true, uploadedChunks };
+}
+async function commitUploadedObject(options, entry, disposition, objectKey, uploadedChunks) {
+    const commit = await requestJson(options, "/api/ambient/evidence/upload/commit", {
+        schema_version: "ambient-raw-evidence-upload-commit.v1",
+        generated_at: options.generatedAt,
+        provenance: options.provenance,
+        upload_id: disposition.upload_id,
+        object_key: objectKey,
+    });
+    if (!commit.ok) {
+        // The server can tell us this object will be refused again. Take its reason
+        // verbatim so the label names the cause ("storage rejected 116 MB") instead
+        // of the transport ("commit_failed_http_500"), which is all the fleet could
+        // say for the 57 days of BLI-2528.
+        const permanent = permanentCommitRejection(commit.body);
+        if (permanent) {
+            return failedOutcome(entry.file, permanent, uploadedChunks);
+        }
+        // A 5xx whose body is not JSON did not come from the route. The commit
+        // handler always answers `{ code, message, ... }`; an HTML error page means
+        // the serverless process was killed — an out-of-memory on a large
+        // assembly, or a hard timeout — so nothing server-side ran a catch, wrote a
+        // ledger reason, or logged a line. Naming it separately is the only way an
+        // operator reading upload reasons can tell "the server refused these bytes"
+        // from "the server never survived them".
+        //
+        // The stem comes from telemetry-core so the label the collector writes and
+        // the label `classifyUploadFailure` reads are the same string by
+        // construction; the status rides on the end so a 502 gateway timeout can
+        // still be told from a 500 process kill, and core strips it back off.
+        if (commit.status >= 500 && isNonJsonResponseBody(commit.body)) {
+            console.error("[evidence-commit] the server died before it could answer; the object is still pending", JSON.stringify({
+                upload_id: disposition.upload_id,
+                http_status: commit.status,
+                byte_size: entry.bytes.byteLength,
+                chunk_count: entry.chunkCount,
+                uploaded_chunk_count: uploadedChunks,
+                reason: COMMIT_CRASHED_PLATFORM,
+            }));
+            return failedOutcome(entry.file, `${COMMIT_CRASHED_PLATFORM}_http_${commit.status}`, uploadedChunks);
+        }
+        const detail = safeFailureDetail(commit.body);
+        return failedOutcome(entry.file, `commit_failed_http_${commit.status}${detail ? `_${detail}` : ""}`, uploadedChunks);
+    }
+    // "already_committed" means a concurrent or earlier sync made these bytes
+    // durable; this sync does not own them, so they must be reported as reuse —
+    // a later ingest failure here must not clean up an object another sync's
+    // indexed refs already point at.
+    const parsedCommit = RawEvidenceUploadCommitResponseSchema.safeParse(commit.body);
+    if (!parsedCommit.success) {
+        return failedOutcome(entry.file, "commit_invalid_response", uploadedChunks);
+    }
+    if (!chunkCommitReceiptMatchesPointer(entry.file.pointer, parsedCommit.data)) {
+        return failedOutcome(entry.file, "commit_receipt_mismatch", uploadedChunks);
+    }
+    const committedPointer = entry.file.pointer;
+    if (parsedCommit.data.status === "already_committed") {
+        return {
+            pointer: committedPointer,
+            object_key: objectKey,
+            codex_session_id: entry.file.codex_session_id ?? null,
+            kind: entry.file.kind ?? "unknown",
+            artifact_metadata: entry.file.artifact_metadata,
+            upload_state: "reused_existing",
+            reason: "already_committed",
+            uploaded_chunk_count: uploadedChunks,
+        };
+    }
+    return {
+        pointer: committedPointer,
+        object_key: objectKey,
+        codex_session_id: entry.file.codex_session_id ?? null,
+        kind: entry.file.kind ?? "unknown",
+        artifact_metadata: entry.file.artifact_metadata,
+        upload_state: "uploaded",
+        reason: null,
+        uploaded_chunk_count: uploadedChunks,
+    };
+}
+/**
+ * The server's verdict that repeating this commit is pointless, or null.
+ *
+ * Two conditions, both required: the response says `retryable: false`, and the
+ * reason is one this collector has classified as permanent. The second check is
+ * the important one. A newer dashboard could declare a reason this CLI has never
+ * heard of, and quietly abandoning an object on a word we cannot interpret is
+ * exactly the silent drop the fleet contract forbids — so an unclassified reason
+ * falls through to the ordinary retry path and stays visible.
+ */
+function permanentCommitRejection(body) {
+    if (!body || typeof body !== "object")
+        return null;
+    if (body.retryable !== false)
+        return null;
+    const reason = safeFailureDetail(body);
+    return reason && isPermanentUploadFailure(reason) ? reason : null;
+}
+function chunkCommitReceiptMatchesPointer(pointer, receipt) {
+    return (receipt.object_key === pointer.object_key &&
+        receipt.content_hash_sha256 === pointer.content_hash_sha256 &&
+        receipt.byte_size === pointer.byte_size &&
+        receipt.redaction === undefined);
+}
+export function failedOutcome(file, reason, uploadedChunks = 0) {
+    return {
+        pointer: file.pointer,
+        object_key: file.pointer.object_key ?? "",
+        codex_session_id: file.codex_session_id ?? null,
+        kind: file.kind ?? "unknown",
+        artifact_metadata: file.artifact_metadata,
+        upload_state: "upload_failed",
+        reason,
+        uploaded_chunk_count: uploadedChunks,
+    };
+}

package/dist/evidence-upload-plan.js

@@ -0,0 +1,233 @@
+import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RawEvidenceUploadBeginResponseSchema, } from "@bli-cockpit/telemetry-core";
+import fs from "node:fs/promises";
+import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
+import { failedOutcome, uploadOneObject } from "./evidence-upload-object.js";
+import { duplicateOutcome, rekeyConflictedEntry, reportAbandonedUpload, } from "./evidence-upload-terminal.js";
+import { describeError } from "./health-detail.js";
+export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
+    const loaded = [];
+    const loadedByObjectKey = new Map();
+    for (const file of options.files) {
+        let stat;
+        try {
+            stat = await fs.stat(file.local_path);
+        }
+        catch (error) {
+            // `file_read_failed` is a durable ledger label and stays. Beside it: a
+            // staged object that vanished (the GC won a race) and one the process
+            // has no permission to open are the same word and opposite repairs
+            // (BLI-3238).
+            console.error("[evidence-upload] staged object could not be stat'd", JSON.stringify({
+                reason: "file_read_failed",
+                stage: "stat",
+                ...describeError(error),
+            }));
+            outcomes.push(failedOutcome(file, "file_read_failed"));
+            continue;
+        }
+        if (stat.size === 0) {
+            outcomes.push(failedOutcome(file, "empty_file"));
+            continue;
+        }
+        if (stat.size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
+            outcomes.push(failedOutcome(file, "file_too_large"));
+            continue;
+        }
+        let bytes;
+        try {
+            bytes = await fs.readFile(file.local_path);
+        }
+        catch (error) {
+            // Stat succeeded and the read did not — narrower than the stat failure
+            // above and worth telling apart, so it carries its own stage label.
+            console.error("[evidence-upload] staged object could not be read after a successful stat", JSON.stringify({
+                reason: "file_read_failed",
+                stage: "read",
+                byte_size: stat.size,
+                ...describeError(error),
+            }));
+            outcomes.push(failedOutcome(file, "file_read_failed"));
+            continue;
+        }
+        if (bytes.byteLength === 0) {
+            outcomes.push(failedOutcome(file, "empty_file"));
+            continue;
+        }
+        if (bytes.byteLength > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
+            outcomes.push(failedOutcome(file, "file_too_large"));
+            continue;
+        }
+        // Content-addressed keys collide for byte-identical files (e.g. a resumed
+        // session copied twice); upload once and share the outcome instead of
+        // racing a second upload against a committed ledger row.
+        const existing = file.pointer.object_key
+            ? loadedByObjectKey.get(file.pointer.object_key)
+            : undefined;
+        if (existing) {
+            existing.duplicates.push(file);
+            continue;
+        }
+        const entry = {
+            file,
+            bytes,
+            chunkCount: Math.ceil(bytes.byteLength / chunkSizeBytes),
+            duplicates: [],
+        };
+        loaded.push(entry);
+        if (file.pointer.object_key) {
+            loadedByObjectKey.set(file.pointer.object_key, entry);
+        }
+    }
+    return loaded;
+}
+export function batches(items, size) {
+    const out = [];
+    for (let offset = 0; offset < items.length; offset += size) {
+        out.push(items.slice(offset, offset + size));
+    }
+    return out;
+}
+/**
+ * One `begin` round-trip for one batch, then delivery of everything it
+ * accepted. Returns `true` when the route itself cannot be used (old
+ * dashboard, or a ledger migration not yet applied) so the caller can fall
+ * back to the legacy single-shot route for what is left unresolved.
+ *
+ * A decision table over `begin`'s possible answers, in the order they are
+ * checked — transport unavailable, refused, unparseable, then delivered —
+ * so each branch stays a guard clause rather than nested conditionals.
+ */
+export async function resolveBeginBatch(options, totalLoadedCount, batch, chunkSizeBytes, outcomes, resolvedEntries) {
+    const begin = await requestJson(options, "/api/ambient/evidence/upload/begin", {
+        schema_version: "ambient-raw-evidence-upload-begin.v1",
+        generated_at: options.generatedAt,
+        provenance: options.provenance,
+        objects: batch.map((entry) => ({
+            pointer: entry.file.pointer,
+            chunk_size_bytes: chunkSizeBytes,
+            chunk_count: entry.chunkCount,
+        })),
+    });
+    // 404 means an old dashboard without the chunk routes; a persistent 5xx
+    // (after retries) covers a new dashboard whose ledger migration has not
+    // been applied yet. Both still serve the legacy v1 route.
+    if (begin.status === 404 || begin.status >= 500) {
+        // Until BLI-3483 this downgrade abandoned the entire chunked path
+        // without a word, and the comment above named two causes that the fleet
+        // had no way to tell apart — an old dashboard versus an unapplied ledger
+        // migration. This is the BLI-2528 shape exactly: the code knew, the
+        // operator did not. Once per run, because `break` leaves the loop.
+        console.error("[evidence-upload] chunked upload unavailable; falling back to the legacy single-shot route", JSON.stringify({
+            reason: begin.status === 404
+                ? "begin_route_absent"
+                : "begin_server_error_after_retries",
+            http_status: begin.status,
+            server_reason: safeFailureDetail(begin.body) ?? "none",
+            objects_in_batch: batch.length,
+            objects_unresolved: totalLoadedCount - resolvedEntries.size,
+        }));
+        return true;
+    }
+    if (!begin.ok) {
+        // The server's own `{ reason }` sat unread in this body while the ledger
+        // recorded the transport status and nothing else (BLI-3483); the commit
+        // path has read it since BLI-2528 and this one now does the same.
+        const beginDetail = safeFailureDetail(begin.body);
+        const beginReason = `begin_failed_http_${begin.status}${beginDetail ? `_${beginDetail}` : ""}`;
+        console.error("[evidence-upload] begin refused these objects", JSON.stringify({
+            reason: "begin_rejected",
+            http_status: begin.status,
+            server_reason: beginDetail ?? "none",
+            objects_in_batch: batch.length,
+        }));
+        for (const entry of batch) {
+            outcomes.push(failedOutcome(entry.file, beginReason));
+            for (const duplicate of entry.duplicates) {
+                outcomes.push(failedOutcome(duplicate, beginReason));
+            }
+            resolvedEntries.add(entry);
+        }
+        return false;
+    }
+    const parsedBegin = RawEvidenceUploadBeginResponseSchema.safeParse(begin.body);
+    if (!parsedBegin.success) {
+        for (const entry of batch) {
+            outcomes.push(failedOutcome(entry.file, "begin_invalid_response"));
+            for (const duplicate of entry.duplicates) {
+                outcomes.push(failedOutcome(duplicate, "begin_invalid_response"));
+            }
+            resolvedEntries.add(entry);
+        }
+        return false;
+    }
+    const dispositions = readBeginDispositions(parsedBegin.data);
+    if (!dispositions) {
+        for (const entry of batch) {
+            outcomes.push(failedOutcome(entry.file, "begin_invalid_response"));
+            for (const duplicate of entry.duplicates) {
+                outcomes.push(failedOutcome(duplicate, "begin_invalid_response"));
+            }
+            resolvedEntries.add(entry);
+        }
+        return false;
+    }
+    for (const entry of batch) {
+        await deliverBatchEntry(options, entry, dispositions, chunkSizeBytes, outcomes);
+        resolvedEntries.add(entry);
+    }
+    return false;
+}
+/**
+ * One accepted object: settle a re-keyable conflict if `begin` reported one
+ * (BLI-3552), then chunk/commit it, then tell the server if the attempt was
+ * abandoned, then record the outcome for it and every byte-identical
+ * duplicate that rode along.
+ */
+async function deliverBatchEntry(options, entry, dispositions, chunkSizeBytes, outcomes) {
+    const objectKey = entry.file.pointer.object_key ?? "";
+    const disposition = dispositions.get(objectKey);
+    if (!disposition) {
+        outcomes.push(failedOutcome(entry.file, "begin_missing_disposition"));
+        for (const duplicate of entry.duplicates) {
+            outcomes.push(failedOutcome(duplicate, "begin_missing_disposition"));
+        }
+        return;
+    }
+    if (disposition.raw_evidence_pointer_id !== entry.file.pointer.raw_evidence_pointer_id) {
+        outcomes.push(failedOutcome(entry.file, "begin_disposition_mismatch"));
+        for (const duplicate of entry.duplicates) {
+            outcomes.push(failedOutcome(duplicate, "begin_disposition_mismatch"));
+        }
+        return;
+    }
+    // A conflict a different name can settle gets one, here, before the
+    // upload is attempted (BLI-3552). Anything else comes back unchanged and
+    // fails on its own reason inside `uploadOneObject`.
+    const resolved = await rekeyConflictedEntry(options, entry, disposition, chunkSizeBytes);
+    const outcome = await uploadOneObject(options, resolved.entry, resolved.disposition, chunkSizeBytes);
+    await reportAbandonedUpload(options, resolved.disposition, outcome);
+    outcomes.push(outcome);
+    // The duplicates of a re-keyed primary were re-keyed with it: a duplicate
+    // is byte-identical and shared the primary's key, so it shares the new
+    // one too. Leaving them on the old key would point their evidence refs at
+    // somebody else's durable object.
+    for (const duplicate of resolved.entry.duplicates) {
+        outcomes.push(duplicateOutcome(outcome, duplicate));
+    }
+}
+export function readBeginDispositions(body) {
+    const map = new Map();
+    for (const entry of body.objects) {
+        if (map.has(entry.object_key))
+            return null;
+        map.set(entry.object_key, {
+            disposition: entry.disposition,
+            upload_id: entry.upload_id ?? null,
+            received_chunk_indexes: entry.received_chunk_indexes,
+            commit_ready: entry.commit_ready === true,
+            reason: entry.reason ?? null,
+            raw_evidence_pointer_id: entry.raw_evidence_pointer_id,
+        });
+    }
+    return map;
+}