@bli-cockpit/cli 0.2.262 → 0.2.263

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,160 @@
1
+ import { RAW_EVIDENCE_UPLOAD_MAX_CHUNK_BYTES, RawEvidenceLegacyUploadResponseSchema, } from "@bli-cockpit/telemetry-core";
2
+ import { isLocalDashboardUrl } from "./dev-build.js";
3
+ import { failedOutcome } from "./evidence-upload-object.js";
4
+ import { StagedFileError, readWholeStagedFile } from "./evidence-upload-staged.js";
5
+ import { duplicateOutcome, pointerWithUploadedMetadata, summarizeOutcomes } from "./evidence-upload-terminal.js";
6
+ import { requestJson } from "./evidence-upload-transport.js";
7
+ /**
8
+ * The legacy single-shot fallback: one request per file for a dashboard that
9
+ * predates the chunk routes. Split out of `evidence-upload-terminal.ts`.
10
+ */
11
+ /**
12
+ * The most raw bytes the legacy single-shot route is offered on a hosted dashboard (BLI-3936).
13
+ *
14
+ * The route carries the file as base64 inside one JSON body, and Vercel refuses a body over about
15
+ * 4.5 MB before our code runs (a 93-byte `text/plain` 413). 3 MiB of raw bytes is 4 MiB of base64
16
+ * plus a few KB of pointer: the exact size the chunk route already sends in every chunk body, so
17
+ * it is proven to pass. Above it the request cannot succeed, only cost a full upload of the object
18
+ * to be told so (14 MB, every backoff window, on the reference Mac).
19
+ */
20
+ export const LEGACY_SINGLE_SHOT_MAX_BYTES = RAW_EVIDENCE_UPLOAD_MAX_CHUNK_BYTES;
21
+ /** A 413 from the legacy route: the host's body limit refused bytes we expected to fit. Terminal for those bytes. */
22
+ export const EVIDENCE_TOO_LARGE_HTTP_413_REASON = "evidence_too_large_http_413";
23
+ /** The reason an object too big for the legacy route carries while the chunk route is down. Retryable: `begin` recovering delivers it. */
24
+ export const LEGACY_SINGLE_SHOT_TOO_LARGE_REASON = "legacy_single_shot_too_large_chunk_route_unavailable";
25
+ /**
26
+ * Older dashboards predate the chunk endpoints. Fall back to one legacy
27
+ * request per file so an oversized or already-existing object fails (or
28
+ * reuses) individually instead of dragging the whole batch down. On a hosted
29
+ * dashboard an object over `LEGACY_SINGLE_SHOT_MAX_BYTES` is not sent at all
30
+ * (BLI-3936): the host's body limit refuses it for certain. A non-serverless
31
+ * dashboard (the default localhost deployment) is still attempted at any size,
32
+ * like the old CLI, and a platform refusal comes back as a labeled per-file
33
+ * failure; a 413 there is `evidence_too_large_http_413`, terminal (`terminalDeliveryReason`).
34
+ */
35
+ export async function uploadWithLegacyFallback(options, loaded, outcomes) {
36
+ for (const entry of loaded) {
37
+ let outcome;
38
+ if (legacyRouteCannotCarry(options, entry.byteSize)) {
39
+ // Said out loud, with the numbers: this object was NOT sent, because the route would refuse it.
40
+ console.error("[evidence-upload] legacy single-shot upload not attempted; the object is too large for it and the chunk route is unavailable", JSON.stringify({
41
+ reason: LEGACY_SINGLE_SHOT_TOO_LARGE_REASON,
42
+ kind: entry.file.kind ?? "unknown",
43
+ byte_size: entry.byteSize,
44
+ limit_bytes: LEGACY_SINGLE_SHOT_MAX_BYTES,
45
+ content_hash_prefix: (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12),
46
+ next: "retried through the chunk route once `begin` answers again",
47
+ }));
48
+ outcome = failedOutcome(entry.file, LEGACY_SINGLE_SHOT_TOO_LARGE_REASON);
49
+ outcomes.push(outcome);
50
+ for (const duplicate of entry.duplicates)
51
+ outcomes.push(duplicateOutcome(outcome, duplicate));
52
+ continue;
53
+ }
54
+ let wholeBytes;
55
+ try {
56
+ // Legacy single-shot needs the whole body; read it now, drop it after the send.
57
+ wholeBytes = await readWholeStagedFile(entry.file.local_path, entry.byteSize);
58
+ }
59
+ catch (error) {
60
+ const reason = error instanceof StagedFileError ? error.reason : "file_read_failed";
61
+ console.error("[evidence-upload] legacy single-shot upload not attempted; staged file unreadable", JSON.stringify({ reason, byte_size: entry.byteSize, kind: entry.file.kind ?? "unknown" }));
62
+ outcome = failedOutcome(entry.file, reason);
63
+ outcomes.push(outcome);
64
+ for (const duplicate of entry.duplicates)
65
+ outcomes.push(duplicateOutcome(outcome, duplicate));
66
+ continue;
67
+ }
68
+ const response = await requestJson(options, "/api/ambient/evidence/upload", {
69
+ schema_version: "ambient-raw-evidence-upload.v1",
70
+ generated_at: options.generatedAt,
71
+ provenance: options.provenance,
72
+ files: [
73
+ {
74
+ pointer: entry.file.pointer,
75
+ content_base64: wholeBytes.toString("base64"),
76
+ },
77
+ ],
78
+ });
79
+ outcome = legacyUploadOutcome(entry.file, response);
80
+ outcomes.push(outcome);
81
+ for (const duplicate of entry.duplicates) {
82
+ outcomes.push(duplicateOutcome(outcome, duplicate));
83
+ }
84
+ }
85
+ return summarizeOutcomes(outcomes, true);
86
+ }
87
+ /** True when sending this many raw bytes to the legacy route is certain to be refused by the host's body limit. */
88
+ function legacyRouteCannotCarry(options, byteSize) {
89
+ return byteSize > LEGACY_SINGLE_SHOT_MAX_BYTES && !isLocalDashboardUrl(options.dashboardUrl);
90
+ }
91
+ function legacyUploadOutcome(file, response) {
92
+ let outcome;
93
+ if (response.ok) {
94
+ const parsedLegacy = RawEvidenceLegacyUploadResponseSchema.safeParse(response.body);
95
+ if (!parsedLegacy.success) {
96
+ outcome = failedOutcome(file, "legacy_upload_invalid_response");
97
+ }
98
+ else {
99
+ const receipt = parsedLegacy.data.uploaded.find((candidate) => candidate.raw_evidence_pointer_id ===
100
+ file.pointer.raw_evidence_pointer_id);
101
+ if (!receipt ||
102
+ parsedLegacy.data.bucket !== file.pointer.storage_bucket ||
103
+ !legacyUploadReceiptMatchesPointer(file.pointer, receipt)) {
104
+ outcome = failedOutcome(file, "legacy_upload_receipt_mismatch");
105
+ }
106
+ else {
107
+ const pointer = pointerWithLegacyUploadResponse(file.pointer, parsedLegacy.data);
108
+ outcome = {
109
+ pointer,
110
+ object_key: file.pointer.object_key ?? "",
111
+ codex_session_id: file.codex_session_id ?? null,
112
+ kind: file.kind ?? "unknown",
113
+ artifact_metadata: file.artifact_metadata,
114
+ upload_state: "uploaded",
115
+ reason: "legacy_single_shot_upload",
116
+ uploaded_chunk_count: 1,
117
+ };
118
+ }
119
+ }
120
+ }
121
+ else if (response.status === 409) {
122
+ // The pre-ledger route cannot prove that a conflicting object contains
123
+ // these exact bytes. Fail closed instead of seeding the cursor with an
124
+ // unverified pointer.
125
+ outcome = failedOutcome(file, "legacy_upload_conflict_http_409");
126
+ }
127
+ else if (response.status === 413) {
128
+ // BLI-3936: the host refused this body for its size. The same bytes meet the same limit on
129
+ // every retry, so this is terminal (`terminalDeliveryReason`) and says what it is.
130
+ outcome = failedOutcome(file, EVIDENCE_TOO_LARGE_HTTP_413_REASON);
131
+ }
132
+ else {
133
+ outcome = failedOutcome(file, `legacy_upload_failed_http_${response.status}`);
134
+ }
135
+ return outcome;
136
+ }
137
+ function pointerWithLegacyUploadResponse(pointer, body) {
138
+ const entry = body.uploaded.find((candidate) => candidate.raw_evidence_pointer_id === pointer.raw_evidence_pointer_id);
139
+ if (!entry)
140
+ return pointer;
141
+ return pointerWithUploadedMetadata(pointer, {
142
+ content_hash_sha256: entry.content_hash_sha256,
143
+ byte_size: entry.byte_size,
144
+ redaction: entry.redaction,
145
+ });
146
+ }
147
+ function legacyUploadReceiptMatchesPointer(pointer, receipt) {
148
+ if (receipt.object_key !== pointer.object_key)
149
+ return false;
150
+ if (receipt.redaction) {
151
+ return (receipt.redaction.original_content_hash_sha256 ===
152
+ pointer.content_hash_sha256 &&
153
+ receipt.redaction.original_byte_size === pointer.byte_size &&
154
+ receipt.redaction.sanitized_content_hash_sha256 ===
155
+ receipt.content_hash_sha256 &&
156
+ receipt.redaction.sanitized_byte_size === receipt.byte_size);
157
+ }
158
+ return (receipt.content_hash_sha256 === pointer.content_hash_sha256 &&
159
+ receipt.byte_size === pointer.byte_size);
160
+ }
@@ -1,13 +1,9 @@
1
1
  import { putSignedChunk } from "./evidence-upload-direct.js";
2
2
  import { gzipSync } from "node:zlib";
3
3
  import { COMMIT_CRASHED_PLATFORM, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
4
+ import { StagedFileError, readVerifiedChunk } from "./evidence-upload-staged.js";
4
5
  import { beginOneObject } from "./evidence-upload-terminal.js";
5
- import { isNonJsonResponseBody, requestJson, safeFailureDetail, sha256, } from "./evidence-upload-transport.js";
6
- /**
7
- * The `chunk` and `commit` stages for one already-`begin`-accepted object:
8
- * send every chunk the server does not already have, then commit and turn
9
- * the receipt into a `ChunkedUploadFileOutcome`.
10
- */
6
+ import { isNonJsonResponseBody, requestJson, safeFailureDetail, } from "./evidence-upload-transport.js";
11
7
  export async function uploadOneObject(options, entry, disposition, chunkSizeBytes, restageAttempted = false) {
12
8
  const objectKey = entry.file.pointer.object_key ?? "";
13
9
  if (disposition.disposition === "already_committed") {
@@ -48,7 +44,20 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
48
44
  for (let index = 0; index < entry.chunkCount; index += 1) {
49
45
  if (received.has(index))
50
46
  continue;
51
- const chunk = entry.bytes.subarray(index * chunkSizeBytes, Math.min((index + 1) * chunkSizeBytes, entry.bytes.byteLength));
47
+ let chunk;
48
+ try {
49
+ // Re-read from disk by offset and check against the plan-time hash; the
50
+ // buffer lives only for this iteration (BLI-23317).
51
+ chunk = await readVerifiedChunk(entry.file.local_path, index * chunkSizeBytes, Math.min(chunkSizeBytes, entry.byteSize - index * chunkSizeBytes), entry.chunkHashes[index]);
52
+ }
53
+ catch (error) {
54
+ const reason = error instanceof StagedFileError ? error.reason : "file_read_failed";
55
+ console.error("[evidence-upload] staged chunk could not be re-read; left for the next tick", JSON.stringify({
56
+ reason, upload_id: disposition.upload_id, chunk_index: index,
57
+ detail: error instanceof StagedFileError ? error.detail : "read_error",
58
+ }));
59
+ return { ok: false, outcome: failedOutcome(entry.file, reason, uploadedChunks) };
60
+ }
52
61
  const offer = disposition.signed_uploads?.find(upload => upload.chunk_index === index);
53
62
  const direct = offer
54
63
  ? await putSignedChunk(options, offer.signed_url, chunk, disposition.upload_id, index)
@@ -62,7 +71,7 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
62
71
  }
63
72
  if (direct === "uploaded") {
64
73
  directChunks.push({ chunk_index: index, byte_size: chunk.length,
65
- chunk_hash_sha256: sha256(chunk) });
74
+ chunk_hash_sha256: entry.chunkHashes[index] });
66
75
  uploadedChunks += 1;
67
76
  continue;
68
77
  }
@@ -92,7 +101,7 @@ async function uploadChunkThroughEndpoint(options, entry, disposition, objectKey
92
101
  object_key: objectKey,
93
102
  chunk_index: index,
94
103
  chunk_count: entry.chunkCount,
95
- chunk_hash_sha256: sha256(chunk),
104
+ chunk_hash_sha256: entry.chunkHashes[index],
96
105
  content_base64: encodedChunk.toString("base64"),
97
106
  });
98
107
  if (!chunkResponse.ok)
@@ -186,7 +195,7 @@ function rejectCommit(commit, entry, disposition, uploadedChunks) {
186
195
  console.error("[evidence-commit] the server died before it could answer; the object is still pending", JSON.stringify({
187
196
  upload_id: disposition.upload_id,
188
197
  http_status: commit.status,
189
- byte_size: entry.bytes.byteLength,
198
+ byte_size: entry.byteSize,
190
199
  chunk_count: entry.chunkCount,
191
200
  uploaded_chunk_count: uploadedChunks,
192
201
  reason: COMMIT_CRASHED_PLATFORM,
@@ -4,12 +4,13 @@ import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
4
4
  import { failedOutcome, uploadOneObject } from "./evidence-upload-object.js";
5
5
  import { duplicateOutcome, rekeyConflictedEntry, reportAbandonedUpload, } from "./evidence-upload-terminal.js";
6
6
  import { describeError } from "./health-detail.js";
7
+ import { StagedFileError, fingerprintStagedFile } from "./evidence-upload-staged.js";
7
8
  export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
8
9
  const loaded = [];
9
10
  const loadedByObjectKey = new Map();
10
11
  for (const file of options.files) {
11
- const bytes = await readStagedUploadBytes(file, outcomes);
12
- if (!bytes)
12
+ const measured = await measureStagedUpload(file, chunkSizeBytes, outcomes);
13
+ if (!measured)
13
14
  continue;
14
15
  // Content-addressed keys collide for byte-identical files (e.g. a resumed
15
16
  // session copied twice); upload once and share the outcome instead of
@@ -23,8 +24,10 @@ export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
23
24
  }
24
25
  const entry = {
25
26
  file,
26
- bytes,
27
- chunkCount: Math.ceil(bytes.byteLength / chunkSizeBytes),
27
+ byteSize: measured.byteSize,
28
+ objectSha256: measured.objectSha256,
29
+ chunkHashes: measured.chunkHashes,
30
+ chunkCount: measured.chunkHashes.length,
28
31
  duplicates: [],
29
32
  };
30
33
  loaded.push(entry);
@@ -34,7 +37,7 @@ export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
34
37
  }
35
38
  return loaded;
36
39
  }
37
- async function readStagedUploadBytes(file, outcomes) {
40
+ async function measureStagedUpload(file, chunkSizeBytes, outcomes) {
38
41
  let stat;
39
42
  try {
40
43
  stat = await fs.stat(file.local_path);
@@ -60,11 +63,21 @@ async function readStagedUploadBytes(file, outcomes) {
60
63
  outcomes.push(failedOutcome(file, "file_too_large"));
61
64
  return null;
62
65
  }
63
- let bytes;
64
66
  try {
65
- bytes = await fs.readFile(file.local_path);
67
+ const fingerprint = await fingerprintStagedFile(file.local_path, stat.size, chunkSizeBytes);
68
+ console.error("[evidence-upload] staged object measured", JSON.stringify({
69
+ byte_size: fingerprint.byteSize,
70
+ chunk_count: fingerprint.chunkHashes.length,
71
+ kind: file.kind ?? "unknown",
72
+ }));
73
+ return fingerprint;
66
74
  }
67
75
  catch (error) {
76
+ if (error instanceof StagedFileError) {
77
+ console.error("[evidence-upload] staged object changed or vanished while planning; left for the next tick", JSON.stringify({ reason: error.reason, stage: "plan", byte_size: stat.size, detail: error.detail }));
78
+ outcomes.push(failedOutcome(file, error.reason));
79
+ return null;
80
+ }
68
81
  // Stat succeeded and the read did not — narrower than the stat failure
69
82
  // above and worth telling apart, so it carries its own stage label.
70
83
  console.error("[evidence-upload] staged object could not be read after a successful stat", JSON.stringify({
@@ -76,15 +89,6 @@ async function readStagedUploadBytes(file, outcomes) {
76
89
  outcomes.push(failedOutcome(file, "file_read_failed"));
77
90
  return null;
78
91
  }
79
- if (bytes.byteLength === 0) {
80
- outcomes.push(failedOutcome(file, "empty_file"));
81
- return null;
82
- }
83
- if (bytes.byteLength > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
84
- outcomes.push(failedOutcome(file, "file_too_large"));
85
- return null;
86
- }
87
- return bytes;
88
92
  }
89
93
  export function batches(items, size) {
90
94
  const out = [];
@@ -0,0 +1,115 @@
1
+ import crypto from "node:crypto";
2
+ import { createReadStream } from "node:fs";
3
+ import fs from "node:fs/promises";
4
+ /** A staged file that no longer matches its plan-time fingerprint. */
5
+ export class StagedFileError extends Error {
6
+ reason;
7
+ detail;
8
+ constructor(reason, detail) {
9
+ super(`${reason}: ${detail}`);
10
+ this.reason = reason;
11
+ this.detail = detail;
12
+ }
13
+ }
14
+ /** Read exactly `length` bytes at `position`, looping over short reads. */
15
+ export async function readExactly(handle, length, position) {
16
+ const buffer = Buffer.allocUnsafe(length);
17
+ let filled = 0;
18
+ while (filled < length) {
19
+ const { bytesRead } = await handle.read(buffer, filled, length - filled, position + filled);
20
+ if (bytesRead === 0)
21
+ break;
22
+ filled += bytesRead;
23
+ }
24
+ return filled === length ? buffer : buffer.subarray(0, filled);
25
+ }
26
+ /**
27
+ * Stream the file once, hashing each chunk and the whole object. Memory is
28
+ * one chunk. Throws StagedFileError when the file vanishes or its length
29
+ * moves while it is being read.
30
+ */
31
+ export async function fingerprintStagedFile(localPath, expectedSize, chunkSizeBytes, openFile = fs.open) {
32
+ const handle = await openStaged(localPath, openFile);
33
+ try {
34
+ const whole = crypto.createHash("sha256");
35
+ const chunkHashes = [];
36
+ for (let offset = 0; offset < expectedSize; offset += chunkSizeBytes) {
37
+ const want = Math.min(chunkSizeBytes, expectedSize - offset);
38
+ const chunk = await readExactly(handle, want, offset);
39
+ if (chunk.length !== want) {
40
+ throw new StagedFileError("staged_file_changed", "file shrank while planning");
41
+ }
42
+ whole.update(chunk);
43
+ chunkHashes.push(crypto.createHash("sha256").update(chunk).digest("hex"));
44
+ }
45
+ const probe = await readExactly(handle, 1, expectedSize);
46
+ if (probe.length !== 0) {
47
+ throw new StagedFileError("staged_file_changed", "file grew while planning");
48
+ }
49
+ return { byteSize: expectedSize, objectSha256: whole.digest("hex"), chunkHashes };
50
+ }
51
+ finally {
52
+ await handle.close();
53
+ }
54
+ }
55
+ /** Read one chunk at send time and check it against the plan-time hash. */
56
+ export async function readVerifiedChunk(localPath, offset, length, expectedSha256, openFile = fs.open) {
57
+ const handle = await openStaged(localPath, openFile);
58
+ try {
59
+ const chunk = await readExactly(handle, length, offset);
60
+ if (chunk.length !== length) {
61
+ throw new StagedFileError("staged_file_changed", "file shorter than planned");
62
+ }
63
+ const actual = crypto.createHash("sha256").update(chunk).digest("hex");
64
+ if (actual !== expectedSha256) {
65
+ throw new StagedFileError("staged_file_changed", "chunk hash differs from plan");
66
+ }
67
+ return chunk;
68
+ }
69
+ finally {
70
+ await handle.close();
71
+ }
72
+ }
73
+ /** Whole staged file, for the legacy single-shot route only (small objects on hosted dashboards). */
74
+ export async function readWholeStagedFile(localPath, byteSize) {
75
+ const handle = await openStaged(localPath, fs.open);
76
+ try {
77
+ const bytes = await readExactly(handle, byteSize, 0);
78
+ if (bytes.length !== byteSize) {
79
+ throw new StagedFileError("staged_file_changed", "file shorter than planned");
80
+ }
81
+ return bytes;
82
+ }
83
+ finally {
84
+ await handle.close();
85
+ }
86
+ }
87
+ /**
88
+ * sha256 and byte count of a file, streamed so memory is one read buffer.
89
+ * Null when the file cannot be read (callers treat that as "unknown").
90
+ */
91
+ export async function hashFileStreaming(localPath) {
92
+ const hash = crypto.createHash("sha256");
93
+ let byteSize = 0;
94
+ try {
95
+ for await (const piece of createReadStream(localPath)) {
96
+ hash.update(piece);
97
+ byteSize += piece.byteLength;
98
+ }
99
+ }
100
+ catch {
101
+ return null;
102
+ }
103
+ return { digest: hash.digest("hex"), byteSize };
104
+ }
105
+ async function openStaged(localPath, openFile) {
106
+ try {
107
+ return await openFile(localPath, "r");
108
+ }
109
+ catch (error) {
110
+ if (error.code === "ENOENT") {
111
+ throw new StagedFileError("staged_file_missing", "ENOENT");
112
+ }
113
+ throw error;
114
+ }
115
+ }
@@ -1,7 +1,5 @@
1
- import { RAW_EVIDENCE_UPLOAD_MAX_CHUNK_BYTES, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
2
- import { isLocalDashboardUrl } from "./dev-build.js";
1
+ import { RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
3
2
  import { readBeginDispositions } from "./evidence-upload-plan.js";
4
- import { failedOutcome } from "./evidence-upload-object.js";
5
3
  import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
6
4
  import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
7
5
  /**
@@ -30,7 +28,7 @@ export async function rekeyConflictedEntry(options, entry, disposition, chunkSiz
30
28
  reason: disposition.reason,
31
29
  kind: entry.file.kind ?? "unknown",
32
30
  content_hash_prefix: contentHashPrefix,
33
- byte_size: entry.bytes.byteLength,
31
+ byte_size: entry.byteSize,
34
32
  }));
35
33
  return { entry, disposition };
36
34
  }
@@ -50,7 +48,7 @@ export async function rekeyConflictedEntry(options, entry, disposition, chunkSiz
50
48
  reason: disposition.reason,
51
49
  kind: entry.file.kind ?? "unknown",
52
50
  content_hash_prefix: contentHashPrefix,
53
- byte_size: entry.bytes.byteLength,
51
+ byte_size: entry.byteSize,
54
52
  chunk_count: entry.chunkCount,
55
53
  rekeyed_disposition: rekeyedDisposition.disposition,
56
54
  rekeyed_reason: rekeyedDisposition.reason ?? "none",
@@ -185,153 +183,6 @@ function conformReasonLabel(reason) {
185
183
  const conformed = reason.replace(/[^a-z0-9_:.-]/gi, "_").slice(0, 120);
186
184
  return conformed.length > 0 ? conformed : "upload_failed_unlabelled";
187
185
  }
188
- /**
189
- * The most raw bytes the legacy single-shot route is offered on a hosted dashboard (BLI-3936).
190
- *
191
- * The route carries the file as base64 inside one JSON body, and Vercel refuses a body over about
192
- * 4.5 MB before our code runs (a 93-byte `text/plain` 413). 3 MiB of raw bytes is 4 MiB of base64
193
- * plus a few KB of pointer: the exact size the chunk route already sends in every chunk body, so
194
- * it is proven to pass. Above it the request cannot succeed, only cost a full upload of the object
195
- * to be told so (14 MB, every backoff window, on the reference Mac).
196
- */
197
- export const LEGACY_SINGLE_SHOT_MAX_BYTES = RAW_EVIDENCE_UPLOAD_MAX_CHUNK_BYTES;
198
- /** A 413 from the legacy route: the host's body limit refused bytes we expected to fit. Terminal for those bytes. */
199
- export const EVIDENCE_TOO_LARGE_HTTP_413_REASON = "evidence_too_large_http_413";
200
- /** The reason an object too big for the legacy route carries while the chunk route is down. Retryable: `begin` recovering delivers it. */
201
- export const LEGACY_SINGLE_SHOT_TOO_LARGE_REASON = "legacy_single_shot_too_large_chunk_route_unavailable";
202
- /**
203
- * Older dashboards predate the chunk endpoints. Fall back to one legacy
204
- * request per file so an oversized or already-existing object fails (or
205
- * reuses) individually instead of dragging the whole batch down. On a hosted
206
- * dashboard an object over `LEGACY_SINGLE_SHOT_MAX_BYTES` is not sent at all
207
- * (BLI-3936): the host's body limit refuses it for certain. A non-serverless
208
- * dashboard (the default localhost deployment) is still attempted at any size,
209
- * like the old CLI, and a platform refusal comes back as a labeled per-file
210
- * failure; a 413 there is `evidence_too_large_http_413`, terminal (`terminalDeliveryReason`).
211
- */
212
- export async function uploadWithLegacyFallback(options, loaded, outcomes) {
213
- for (const entry of loaded) {
214
- let outcome;
215
- if (legacyRouteCannotCarry(options, entry.bytes.byteLength)) {
216
- // Said out loud, with the numbers: this object was NOT sent, because the route would refuse it.
217
- console.error("[evidence-upload] legacy single-shot upload not attempted; the object is too large for it and the chunk route is unavailable", JSON.stringify({
218
- reason: LEGACY_SINGLE_SHOT_TOO_LARGE_REASON,
219
- kind: entry.file.kind ?? "unknown",
220
- byte_size: entry.bytes.byteLength,
221
- limit_bytes: LEGACY_SINGLE_SHOT_MAX_BYTES,
222
- content_hash_prefix: (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12),
223
- next: "retried through the chunk route once `begin` answers again",
224
- }));
225
- outcome = failedOutcome(entry.file, LEGACY_SINGLE_SHOT_TOO_LARGE_REASON);
226
- outcomes.push(outcome);
227
- for (const duplicate of entry.duplicates)
228
- outcomes.push(duplicateOutcome(outcome, duplicate));
229
- continue;
230
- }
231
- const response = await requestJson(options, "/api/ambient/evidence/upload", {
232
- schema_version: "ambient-raw-evidence-upload.v1",
233
- generated_at: options.generatedAt,
234
- provenance: options.provenance,
235
- files: [
236
- {
237
- pointer: entry.file.pointer,
238
- content_base64: entry.bytes.toString("base64"),
239
- },
240
- ],
241
- });
242
- outcome = legacyUploadOutcome(entry.file, response);
243
- outcomes.push(outcome);
244
- for (const duplicate of entry.duplicates) {
245
- outcomes.push(duplicateOutcome(outcome, duplicate));
246
- }
247
- }
248
- return summarizeOutcomes(outcomes, true);
249
- }
250
- /** True when sending this many raw bytes to the legacy route is certain to be refused by the host's body limit. */
251
- function legacyRouteCannotCarry(options, byteSize) {
252
- return byteSize > LEGACY_SINGLE_SHOT_MAX_BYTES && !isLocalDashboardUrl(options.dashboardUrl);
253
- }
254
- function legacyUploadOutcome(file, response) {
255
- let outcome;
256
- if (response.ok) {
257
- const parsedLegacy = RawEvidenceLegacyUploadResponseSchema.safeParse(response.body);
258
- if (!parsedLegacy.success) {
259
- outcome = failedOutcome(file, "legacy_upload_invalid_response");
260
- }
261
- else {
262
- const receipt = parsedLegacy.data.uploaded.find((candidate) => candidate.raw_evidence_pointer_id ===
263
- file.pointer.raw_evidence_pointer_id);
264
- if (!receipt ||
265
- parsedLegacy.data.bucket !== file.pointer.storage_bucket ||
266
- !legacyUploadReceiptMatchesPointer(file.pointer, receipt)) {
267
- outcome = failedOutcome(file, "legacy_upload_receipt_mismatch");
268
- }
269
- else {
270
- const pointer = pointerWithLegacyUploadResponse(file.pointer, parsedLegacy.data);
271
- outcome = {
272
- pointer,
273
- object_key: file.pointer.object_key ?? "",
274
- codex_session_id: file.codex_session_id ?? null,
275
- kind: file.kind ?? "unknown",
276
- artifact_metadata: file.artifact_metadata,
277
- upload_state: "uploaded",
278
- reason: "legacy_single_shot_upload",
279
- uploaded_chunk_count: 1,
280
- };
281
- }
282
- }
283
- }
284
- else if (response.status === 409) {
285
- // The pre-ledger route cannot prove that a conflicting object contains
286
- // these exact bytes. Fail closed instead of seeding the cursor with an
287
- // unverified pointer.
288
- outcome = failedOutcome(file, "legacy_upload_conflict_http_409");
289
- }
290
- else if (response.status === 413) {
291
- // BLI-3936: the host refused this body for its size. The same bytes meet the same limit on
292
- // every retry, so this is terminal (`terminalDeliveryReason`) and says what it is.
293
- outcome = failedOutcome(file, EVIDENCE_TOO_LARGE_HTTP_413_REASON);
294
- }
295
- else {
296
- outcome = failedOutcome(file, `legacy_upload_failed_http_${response.status}`);
297
- }
298
- return outcome;
299
- }
300
- function pointerWithLegacyUploadResponse(pointer, body) {
301
- const entry = body.uploaded.find((candidate) => candidate.raw_evidence_pointer_id === pointer.raw_evidence_pointer_id);
302
- if (!entry)
303
- return pointer;
304
- return pointerWithUploadedMetadata(pointer, {
305
- content_hash_sha256: entry.content_hash_sha256,
306
- byte_size: entry.byte_size,
307
- redaction: entry.redaction,
308
- });
309
- }
310
- function legacyUploadReceiptMatchesPointer(pointer, receipt) {
311
- if (receipt.object_key !== pointer.object_key)
312
- return false;
313
- if (receipt.redaction) {
314
- return (receipt.redaction.original_content_hash_sha256 ===
315
- pointer.content_hash_sha256 &&
316
- receipt.redaction.original_byte_size === pointer.byte_size &&
317
- receipt.redaction.sanitized_content_hash_sha256 ===
318
- receipt.content_hash_sha256 &&
319
- receipt.redaction.sanitized_byte_size === receipt.byte_size);
320
- }
321
- return (receipt.content_hash_sha256 === pointer.content_hash_sha256 &&
322
- receipt.byte_size === pointer.byte_size);
323
- }
324
- function pointerWithUploadedMetadata(pointer, metadata) {
325
- const redaction = RawEvidenceRedactionMetadataSchema.safeParse(metadata.redaction);
326
- return {
327
- ...pointer,
328
- content_hash_sha256: typeof metadata.content_hash_sha256 === "string"
329
- ? metadata.content_hash_sha256
330
- : pointer.content_hash_sha256,
331
- byte_size: typeof metadata.byte_size === "number" ? metadata.byte_size : pointer.byte_size,
332
- redaction: redaction.success ? redaction.data : pointer.redaction,
333
- };
334
- }
335
186
  export function summarizeOutcomes(outcomes, usedLegacyFallback) {
336
187
  const uploaded = outcomes.filter((outcome) => outcome.upload_state === "uploaded");
337
188
  return {
@@ -343,4 +194,15 @@ export function summarizeOutcomes(outcomes, usedLegacyFallback) {
343
194
  failed_count: outcomes.filter((outcome) => outcome.upload_state === "upload_failed").length,
344
195
  used_legacy_fallback: usedLegacyFallback,
345
196
  };
197
+ }
198
+ export function pointerWithUploadedMetadata(pointer, metadata) {
199
+ const redaction = RawEvidenceRedactionMetadataSchema.safeParse(metadata.redaction);
200
+ return {
201
+ ...pointer,
202
+ content_hash_sha256: typeof metadata.content_hash_sha256 === "string"
203
+ ? metadata.content_hash_sha256
204
+ : pointer.content_hash_sha256,
205
+ byte_size: typeof metadata.byte_size === "number" ? metadata.byte_size : pointer.byte_size,
206
+ redaction: redaction.success ? redaction.data : pointer.redaction,
207
+ };
346
208
  }