@bli-cockpit/cli 0.2.262 → 0.2.264
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/agent-image-evidence.js +4 -4
- package/dist/adapters/raw-evidence-collection-state.js +14 -2
- package/dist/adapters/raw-evidence-sanitize-held.js +191 -0
- package/dist/adapters/raw-evidence-sanitize.js +10 -10
- package/dist/adapters/raw-evidence-transcript-body.js +69 -0
- package/dist/adapters/raw-evidence-transcript-file.js +11 -54
- package/dist/adapters/transcript-line-stream.js +47 -0
- package/dist/adapters/transcript-read-memo.js +1 -1
- package/dist/command-guard.js +202 -0
- package/dist/commands/backfill.js +6 -1
- package/dist/commands/cli-io.js +3 -1
- package/dist/commands/doctor-disk-words.js +5 -1
- package/dist/commands/doctor.js +9 -2
- package/dist/commands/heartbeat.js +40 -0
- package/dist/commands/local-args-collector-status.js +4 -0
- package/dist/commands/local-help-commands.js +8 -0
- package/dist/commands/local.js +2 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sessions-forget.js +62 -0
- package/dist/commands/sync-heartbeat.js +5 -1
- package/dist/commands/sync.js +9 -0
- package/dist/cursors/raw-evidence-cursor.js +1 -1
- package/dist/disk-usage-files.js +5 -5
- package/dist/evidence-upload-client.js +2 -1
- package/dist/evidence-upload-legacy.js +160 -0
- package/dist/evidence-upload-object.js +19 -10
- package/dist/evidence-upload-plan.js +20 -16
- package/dist/evidence-upload-staged.js +115 -0
- package/dist/evidence-upload-terminal.js +14 -152
- package/dist/http1-fetch.js +243 -0
- package/dist/image-description-session.js +27 -9
- package/dist/local-state-files.js +4 -2
- package/dist/raw-evidence-gc.js +5 -5
- package/dist/slim/slim-transcript.js +99 -22
- package/dist/spool/install-event-outbox.js +30 -0
- package/dist/staging-gate.js +6 -3
- package/dist/sync-tick-timing.js +4 -0
- package/package.json +4 -4
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { RAW_EVIDENCE_UPLOAD_MAX_CHUNK_BYTES, RawEvidenceLegacyUploadResponseSchema, } from "@bli-cockpit/telemetry-core";
|
|
2
|
+
import { isLocalDashboardUrl } from "./dev-build.js";
|
|
3
|
+
import { failedOutcome } from "./evidence-upload-object.js";
|
|
4
|
+
import { StagedFileError, readWholeStagedFile } from "./evidence-upload-staged.js";
|
|
5
|
+
import { duplicateOutcome, pointerWithUploadedMetadata, summarizeOutcomes } from "./evidence-upload-terminal.js";
|
|
6
|
+
import { requestJson } from "./evidence-upload-transport.js";
|
|
7
|
+
/**
|
|
8
|
+
* The legacy single-shot fallback: one request per file for a dashboard that
|
|
9
|
+
* predates the chunk routes. Split out of `evidence-upload-terminal.ts`.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* The most raw bytes the legacy single-shot route is offered on a hosted dashboard (BLI-3936).
|
|
13
|
+
*
|
|
14
|
+
* The route carries the file as base64 inside one JSON body, and Vercel refuses a body over about
|
|
15
|
+
* 4.5 MB before our code runs (a 93-byte `text/plain` 413). 3 MiB of raw bytes is 4 MiB of base64
|
|
16
|
+
* plus a few KB of pointer: the exact size the chunk route already sends in every chunk body, so
|
|
17
|
+
* it is proven to pass. Above it the request cannot succeed, only cost a full upload of the object
|
|
18
|
+
* to be told so (14 MB, every backoff window, on the reference Mac).
|
|
19
|
+
*/
|
|
20
|
+
export const LEGACY_SINGLE_SHOT_MAX_BYTES = RAW_EVIDENCE_UPLOAD_MAX_CHUNK_BYTES;
|
|
21
|
+
/** A 413 from the legacy route: the host's body limit refused bytes we expected to fit. Terminal for those bytes. */
|
|
22
|
+
export const EVIDENCE_TOO_LARGE_HTTP_413_REASON = "evidence_too_large_http_413";
|
|
23
|
+
/** The reason an object too big for the legacy route carries while the chunk route is down. Retryable: `begin` recovering delivers it. */
|
|
24
|
+
export const LEGACY_SINGLE_SHOT_TOO_LARGE_REASON = "legacy_single_shot_too_large_chunk_route_unavailable";
|
|
25
|
+
/**
|
|
26
|
+
* Older dashboards predate the chunk endpoints. Fall back to one legacy
|
|
27
|
+
* request per file so an oversized or already-existing object fails (or
|
|
28
|
+
* reuses) individually instead of dragging the whole batch down. On a hosted
|
|
29
|
+
* dashboard an object over `LEGACY_SINGLE_SHOT_MAX_BYTES` is not sent at all
|
|
30
|
+
* (BLI-3936): the host's body limit refuses it for certain. A non-serverless
|
|
31
|
+
* dashboard (the default localhost deployment) is still attempted at any size,
|
|
32
|
+
* like the old CLI, and a platform refusal comes back as a labeled per-file
|
|
33
|
+
* failure; a 413 there is `evidence_too_large_http_413`, terminal (`terminalDeliveryReason`).
|
|
34
|
+
*/
|
|
35
|
+
export async function uploadWithLegacyFallback(options, loaded, outcomes) {
|
|
36
|
+
for (const entry of loaded) {
|
|
37
|
+
let outcome;
|
|
38
|
+
if (legacyRouteCannotCarry(options, entry.byteSize)) {
|
|
39
|
+
// Said out loud, with the numbers: this object was NOT sent, because the route would refuse it.
|
|
40
|
+
console.error("[evidence-upload] legacy single-shot upload not attempted; the object is too large for it and the chunk route is unavailable", JSON.stringify({
|
|
41
|
+
reason: LEGACY_SINGLE_SHOT_TOO_LARGE_REASON,
|
|
42
|
+
kind: entry.file.kind ?? "unknown",
|
|
43
|
+
byte_size: entry.byteSize,
|
|
44
|
+
limit_bytes: LEGACY_SINGLE_SHOT_MAX_BYTES,
|
|
45
|
+
content_hash_prefix: (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12),
|
|
46
|
+
next: "retried through the chunk route once `begin` answers again",
|
|
47
|
+
}));
|
|
48
|
+
outcome = failedOutcome(entry.file, LEGACY_SINGLE_SHOT_TOO_LARGE_REASON);
|
|
49
|
+
outcomes.push(outcome);
|
|
50
|
+
for (const duplicate of entry.duplicates)
|
|
51
|
+
outcomes.push(duplicateOutcome(outcome, duplicate));
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
let wholeBytes;
|
|
55
|
+
try {
|
|
56
|
+
// Legacy single-shot needs the whole body; read it now, drop it after the send.
|
|
57
|
+
wholeBytes = await readWholeStagedFile(entry.file.local_path, entry.byteSize);
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
const reason = error instanceof StagedFileError ? error.reason : "file_read_failed";
|
|
61
|
+
console.error("[evidence-upload] legacy single-shot upload not attempted; staged file unreadable", JSON.stringify({ reason, byte_size: entry.byteSize, kind: entry.file.kind ?? "unknown" }));
|
|
62
|
+
outcome = failedOutcome(entry.file, reason);
|
|
63
|
+
outcomes.push(outcome);
|
|
64
|
+
for (const duplicate of entry.duplicates)
|
|
65
|
+
outcomes.push(duplicateOutcome(outcome, duplicate));
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
const response = await requestJson(options, "/api/ambient/evidence/upload", {
|
|
69
|
+
schema_version: "ambient-raw-evidence-upload.v1",
|
|
70
|
+
generated_at: options.generatedAt,
|
|
71
|
+
provenance: options.provenance,
|
|
72
|
+
files: [
|
|
73
|
+
{
|
|
74
|
+
pointer: entry.file.pointer,
|
|
75
|
+
content_base64: wholeBytes.toString("base64"),
|
|
76
|
+
},
|
|
77
|
+
],
|
|
78
|
+
});
|
|
79
|
+
outcome = legacyUploadOutcome(entry.file, response);
|
|
80
|
+
outcomes.push(outcome);
|
|
81
|
+
for (const duplicate of entry.duplicates) {
|
|
82
|
+
outcomes.push(duplicateOutcome(outcome, duplicate));
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return summarizeOutcomes(outcomes, true);
|
|
86
|
+
}
|
|
87
|
+
/** True when sending this many raw bytes to the legacy route is certain to be refused by the host's body limit. */
|
|
88
|
+
function legacyRouteCannotCarry(options, byteSize) {
|
|
89
|
+
return byteSize > LEGACY_SINGLE_SHOT_MAX_BYTES && !isLocalDashboardUrl(options.dashboardUrl);
|
|
90
|
+
}
|
|
91
|
+
function legacyUploadOutcome(file, response) {
|
|
92
|
+
let outcome;
|
|
93
|
+
if (response.ok) {
|
|
94
|
+
const parsedLegacy = RawEvidenceLegacyUploadResponseSchema.safeParse(response.body);
|
|
95
|
+
if (!parsedLegacy.success) {
|
|
96
|
+
outcome = failedOutcome(file, "legacy_upload_invalid_response");
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
const receipt = parsedLegacy.data.uploaded.find((candidate) => candidate.raw_evidence_pointer_id ===
|
|
100
|
+
file.pointer.raw_evidence_pointer_id);
|
|
101
|
+
if (!receipt ||
|
|
102
|
+
parsedLegacy.data.bucket !== file.pointer.storage_bucket ||
|
|
103
|
+
!legacyUploadReceiptMatchesPointer(file.pointer, receipt)) {
|
|
104
|
+
outcome = failedOutcome(file, "legacy_upload_receipt_mismatch");
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
const pointer = pointerWithLegacyUploadResponse(file.pointer, parsedLegacy.data);
|
|
108
|
+
outcome = {
|
|
109
|
+
pointer,
|
|
110
|
+
object_key: file.pointer.object_key ?? "",
|
|
111
|
+
codex_session_id: file.codex_session_id ?? null,
|
|
112
|
+
kind: file.kind ?? "unknown",
|
|
113
|
+
artifact_metadata: file.artifact_metadata,
|
|
114
|
+
upload_state: "uploaded",
|
|
115
|
+
reason: "legacy_single_shot_upload",
|
|
116
|
+
uploaded_chunk_count: 1,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
else if (response.status === 409) {
|
|
122
|
+
// The pre-ledger route cannot prove that a conflicting object contains
|
|
123
|
+
// these exact bytes. Fail closed instead of seeding the cursor with an
|
|
124
|
+
// unverified pointer.
|
|
125
|
+
outcome = failedOutcome(file, "legacy_upload_conflict_http_409");
|
|
126
|
+
}
|
|
127
|
+
else if (response.status === 413) {
|
|
128
|
+
// BLI-3936: the host refused this body for its size. The same bytes meet the same limit on
|
|
129
|
+
// every retry, so this is terminal (`terminalDeliveryReason`) and says what it is.
|
|
130
|
+
outcome = failedOutcome(file, EVIDENCE_TOO_LARGE_HTTP_413_REASON);
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
outcome = failedOutcome(file, `legacy_upload_failed_http_${response.status}`);
|
|
134
|
+
}
|
|
135
|
+
return outcome;
|
|
136
|
+
}
|
|
137
|
+
function pointerWithLegacyUploadResponse(pointer, body) {
|
|
138
|
+
const entry = body.uploaded.find((candidate) => candidate.raw_evidence_pointer_id === pointer.raw_evidence_pointer_id);
|
|
139
|
+
if (!entry)
|
|
140
|
+
return pointer;
|
|
141
|
+
return pointerWithUploadedMetadata(pointer, {
|
|
142
|
+
content_hash_sha256: entry.content_hash_sha256,
|
|
143
|
+
byte_size: entry.byte_size,
|
|
144
|
+
redaction: entry.redaction,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
function legacyUploadReceiptMatchesPointer(pointer, receipt) {
|
|
148
|
+
if (receipt.object_key !== pointer.object_key)
|
|
149
|
+
return false;
|
|
150
|
+
if (receipt.redaction) {
|
|
151
|
+
return (receipt.redaction.original_content_hash_sha256 ===
|
|
152
|
+
pointer.content_hash_sha256 &&
|
|
153
|
+
receipt.redaction.original_byte_size === pointer.byte_size &&
|
|
154
|
+
receipt.redaction.sanitized_content_hash_sha256 ===
|
|
155
|
+
receipt.content_hash_sha256 &&
|
|
156
|
+
receipt.redaction.sanitized_byte_size === receipt.byte_size);
|
|
157
|
+
}
|
|
158
|
+
return (receipt.content_hash_sha256 === pointer.content_hash_sha256 &&
|
|
159
|
+
receipt.byte_size === pointer.byte_size);
|
|
160
|
+
}
|
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
import { putSignedChunk } from "./evidence-upload-direct.js";
|
|
2
2
|
import { gzipSync } from "node:zlib";
|
|
3
3
|
import { COMMIT_CRASHED_PLATFORM, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
4
|
+
import { StagedFileError, readVerifiedChunk } from "./evidence-upload-staged.js";
|
|
4
5
|
import { beginOneObject } from "./evidence-upload-terminal.js";
|
|
5
|
-
import { isNonJsonResponseBody, requestJson, safeFailureDetail,
|
|
6
|
-
/**
|
|
7
|
-
* The `chunk` and `commit` stages for one already-`begin`-accepted object:
|
|
8
|
-
* send every chunk the server does not already have, then commit and turn
|
|
9
|
-
* the receipt into a `ChunkedUploadFileOutcome`.
|
|
10
|
-
*/
|
|
6
|
+
import { isNonJsonResponseBody, requestJson, safeFailureDetail, } from "./evidence-upload-transport.js";
|
|
11
7
|
export async function uploadOneObject(options, entry, disposition, chunkSizeBytes, restageAttempted = false) {
|
|
12
8
|
const objectKey = entry.file.pointer.object_key ?? "";
|
|
13
9
|
if (disposition.disposition === "already_committed") {
|
|
@@ -48,7 +44,20 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
|
|
|
48
44
|
for (let index = 0; index < entry.chunkCount; index += 1) {
|
|
49
45
|
if (received.has(index))
|
|
50
46
|
continue;
|
|
51
|
-
|
|
47
|
+
let chunk;
|
|
48
|
+
try {
|
|
49
|
+
// Re-read from disk by offset and check against the plan-time hash; the
|
|
50
|
+
// buffer lives only for this iteration (BLI-23317).
|
|
51
|
+
chunk = await readVerifiedChunk(entry.file.local_path, index * chunkSizeBytes, Math.min(chunkSizeBytes, entry.byteSize - index * chunkSizeBytes), entry.chunkHashes[index]);
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
const reason = error instanceof StagedFileError ? error.reason : "file_read_failed";
|
|
55
|
+
console.error("[evidence-upload] staged chunk could not be re-read; left for the next tick", JSON.stringify({
|
|
56
|
+
reason, upload_id: disposition.upload_id, chunk_index: index,
|
|
57
|
+
detail: error instanceof StagedFileError ? error.detail : "read_error",
|
|
58
|
+
}));
|
|
59
|
+
return { ok: false, outcome: failedOutcome(entry.file, reason, uploadedChunks) };
|
|
60
|
+
}
|
|
52
61
|
const offer = disposition.signed_uploads?.find(upload => upload.chunk_index === index);
|
|
53
62
|
const direct = offer
|
|
54
63
|
? await putSignedChunk(options, offer.signed_url, chunk, disposition.upload_id, index)
|
|
@@ -62,7 +71,7 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
|
|
|
62
71
|
}
|
|
63
72
|
if (direct === "uploaded") {
|
|
64
73
|
directChunks.push({ chunk_index: index, byte_size: chunk.length,
|
|
65
|
-
chunk_hash_sha256:
|
|
74
|
+
chunk_hash_sha256: entry.chunkHashes[index] });
|
|
66
75
|
uploadedChunks += 1;
|
|
67
76
|
continue;
|
|
68
77
|
}
|
|
@@ -92,7 +101,7 @@ async function uploadChunkThroughEndpoint(options, entry, disposition, objectKey
|
|
|
92
101
|
object_key: objectKey,
|
|
93
102
|
chunk_index: index,
|
|
94
103
|
chunk_count: entry.chunkCount,
|
|
95
|
-
chunk_hash_sha256:
|
|
104
|
+
chunk_hash_sha256: entry.chunkHashes[index],
|
|
96
105
|
content_base64: encodedChunk.toString("base64"),
|
|
97
106
|
});
|
|
98
107
|
if (!chunkResponse.ok)
|
|
@@ -186,7 +195,7 @@ function rejectCommit(commit, entry, disposition, uploadedChunks) {
|
|
|
186
195
|
console.error("[evidence-commit] the server died before it could answer; the object is still pending", JSON.stringify({
|
|
187
196
|
upload_id: disposition.upload_id,
|
|
188
197
|
http_status: commit.status,
|
|
189
|
-
byte_size: entry.
|
|
198
|
+
byte_size: entry.byteSize,
|
|
190
199
|
chunk_count: entry.chunkCount,
|
|
191
200
|
uploaded_chunk_count: uploadedChunks,
|
|
192
201
|
reason: COMMIT_CRASHED_PLATFORM,
|
|
@@ -4,12 +4,13 @@ import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
|
|
|
4
4
|
import { failedOutcome, uploadOneObject } from "./evidence-upload-object.js";
|
|
5
5
|
import { duplicateOutcome, rekeyConflictedEntry, reportAbandonedUpload, } from "./evidence-upload-terminal.js";
|
|
6
6
|
import { describeError } from "./health-detail.js";
|
|
7
|
+
import { StagedFileError, fingerprintStagedFile } from "./evidence-upload-staged.js";
|
|
7
8
|
export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
|
|
8
9
|
const loaded = [];
|
|
9
10
|
const loadedByObjectKey = new Map();
|
|
10
11
|
for (const file of options.files) {
|
|
11
|
-
const
|
|
12
|
-
if (!
|
|
12
|
+
const measured = await measureStagedUpload(file, chunkSizeBytes, outcomes);
|
|
13
|
+
if (!measured)
|
|
13
14
|
continue;
|
|
14
15
|
// Content-addressed keys collide for byte-identical files (e.g. a resumed
|
|
15
16
|
// session copied twice); upload once and share the outcome instead of
|
|
@@ -23,8 +24,10 @@ export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
|
|
|
23
24
|
}
|
|
24
25
|
const entry = {
|
|
25
26
|
file,
|
|
26
|
-
|
|
27
|
-
|
|
27
|
+
byteSize: measured.byteSize,
|
|
28
|
+
objectSha256: measured.objectSha256,
|
|
29
|
+
chunkHashes: measured.chunkHashes,
|
|
30
|
+
chunkCount: measured.chunkHashes.length,
|
|
28
31
|
duplicates: [],
|
|
29
32
|
};
|
|
30
33
|
loaded.push(entry);
|
|
@@ -34,7 +37,7 @@ export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
|
|
|
34
37
|
}
|
|
35
38
|
return loaded;
|
|
36
39
|
}
|
|
37
|
-
async function
|
|
40
|
+
async function measureStagedUpload(file, chunkSizeBytes, outcomes) {
|
|
38
41
|
let stat;
|
|
39
42
|
try {
|
|
40
43
|
stat = await fs.stat(file.local_path);
|
|
@@ -60,11 +63,21 @@ async function readStagedUploadBytes(file, outcomes) {
|
|
|
60
63
|
outcomes.push(failedOutcome(file, "file_too_large"));
|
|
61
64
|
return null;
|
|
62
65
|
}
|
|
63
|
-
let bytes;
|
|
64
66
|
try {
|
|
65
|
-
|
|
67
|
+
const fingerprint = await fingerprintStagedFile(file.local_path, stat.size, chunkSizeBytes);
|
|
68
|
+
console.error("[evidence-upload] staged object measured", JSON.stringify({
|
|
69
|
+
byte_size: fingerprint.byteSize,
|
|
70
|
+
chunk_count: fingerprint.chunkHashes.length,
|
|
71
|
+
kind: file.kind ?? "unknown",
|
|
72
|
+
}));
|
|
73
|
+
return fingerprint;
|
|
66
74
|
}
|
|
67
75
|
catch (error) {
|
|
76
|
+
if (error instanceof StagedFileError) {
|
|
77
|
+
console.error("[evidence-upload] staged object changed or vanished while planning; left for the next tick", JSON.stringify({ reason: error.reason, stage: "plan", byte_size: stat.size, detail: error.detail }));
|
|
78
|
+
outcomes.push(failedOutcome(file, error.reason));
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
68
81
|
// Stat succeeded and the read did not — narrower than the stat failure
|
|
69
82
|
// above and worth telling apart, so it carries its own stage label.
|
|
70
83
|
console.error("[evidence-upload] staged object could not be read after a successful stat", JSON.stringify({
|
|
@@ -76,15 +89,6 @@ async function readStagedUploadBytes(file, outcomes) {
|
|
|
76
89
|
outcomes.push(failedOutcome(file, "file_read_failed"));
|
|
77
90
|
return null;
|
|
78
91
|
}
|
|
79
|
-
if (bytes.byteLength === 0) {
|
|
80
|
-
outcomes.push(failedOutcome(file, "empty_file"));
|
|
81
|
-
return null;
|
|
82
|
-
}
|
|
83
|
-
if (bytes.byteLength > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
|
|
84
|
-
outcomes.push(failedOutcome(file, "file_too_large"));
|
|
85
|
-
return null;
|
|
86
|
-
}
|
|
87
|
-
return bytes;
|
|
88
92
|
}
|
|
89
93
|
export function batches(items, size) {
|
|
90
94
|
const out = [];
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import { createReadStream } from "node:fs";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
/** A staged file that no longer matches its plan-time fingerprint. */
|
|
5
|
+
export class StagedFileError extends Error {
|
|
6
|
+
reason;
|
|
7
|
+
detail;
|
|
8
|
+
constructor(reason, detail) {
|
|
9
|
+
super(`${reason}: ${detail}`);
|
|
10
|
+
this.reason = reason;
|
|
11
|
+
this.detail = detail;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
/** Read exactly `length` bytes at `position`, looping over short reads. */
|
|
15
|
+
export async function readExactly(handle, length, position) {
|
|
16
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
17
|
+
let filled = 0;
|
|
18
|
+
while (filled < length) {
|
|
19
|
+
const { bytesRead } = await handle.read(buffer, filled, length - filled, position + filled);
|
|
20
|
+
if (bytesRead === 0)
|
|
21
|
+
break;
|
|
22
|
+
filled += bytesRead;
|
|
23
|
+
}
|
|
24
|
+
return filled === length ? buffer : buffer.subarray(0, filled);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Stream the file once, hashing each chunk and the whole object. Memory is
|
|
28
|
+
* one chunk. Throws StagedFileError when the file vanishes or its length
|
|
29
|
+
* moves while it is being read.
|
|
30
|
+
*/
|
|
31
|
+
export async function fingerprintStagedFile(localPath, expectedSize, chunkSizeBytes, openFile = fs.open) {
|
|
32
|
+
const handle = await openStaged(localPath, openFile);
|
|
33
|
+
try {
|
|
34
|
+
const whole = crypto.createHash("sha256");
|
|
35
|
+
const chunkHashes = [];
|
|
36
|
+
for (let offset = 0; offset < expectedSize; offset += chunkSizeBytes) {
|
|
37
|
+
const want = Math.min(chunkSizeBytes, expectedSize - offset);
|
|
38
|
+
const chunk = await readExactly(handle, want, offset);
|
|
39
|
+
if (chunk.length !== want) {
|
|
40
|
+
throw new StagedFileError("staged_file_changed", "file shrank while planning");
|
|
41
|
+
}
|
|
42
|
+
whole.update(chunk);
|
|
43
|
+
chunkHashes.push(crypto.createHash("sha256").update(chunk).digest("hex"));
|
|
44
|
+
}
|
|
45
|
+
const probe = await readExactly(handle, 1, expectedSize);
|
|
46
|
+
if (probe.length !== 0) {
|
|
47
|
+
throw new StagedFileError("staged_file_changed", "file grew while planning");
|
|
48
|
+
}
|
|
49
|
+
return { byteSize: expectedSize, objectSha256: whole.digest("hex"), chunkHashes };
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
await handle.close();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** Read one chunk at send time and check it against the plan-time hash. */
|
|
56
|
+
export async function readVerifiedChunk(localPath, offset, length, expectedSha256, openFile = fs.open) {
|
|
57
|
+
const handle = await openStaged(localPath, openFile);
|
|
58
|
+
try {
|
|
59
|
+
const chunk = await readExactly(handle, length, offset);
|
|
60
|
+
if (chunk.length !== length) {
|
|
61
|
+
throw new StagedFileError("staged_file_changed", "file shorter than planned");
|
|
62
|
+
}
|
|
63
|
+
const actual = crypto.createHash("sha256").update(chunk).digest("hex");
|
|
64
|
+
if (actual !== expectedSha256) {
|
|
65
|
+
throw new StagedFileError("staged_file_changed", "chunk hash differs from plan");
|
|
66
|
+
}
|
|
67
|
+
return chunk;
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
await handle.close();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** Whole staged file, for the legacy single-shot route only (small objects on hosted dashboards). */
|
|
74
|
+
export async function readWholeStagedFile(localPath, byteSize) {
|
|
75
|
+
const handle = await openStaged(localPath, fs.open);
|
|
76
|
+
try {
|
|
77
|
+
const bytes = await readExactly(handle, byteSize, 0);
|
|
78
|
+
if (bytes.length !== byteSize) {
|
|
79
|
+
throw new StagedFileError("staged_file_changed", "file shorter than planned");
|
|
80
|
+
}
|
|
81
|
+
return bytes;
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
await handle.close();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* sha256 and byte count of a file, streamed so memory is one read buffer.
|
|
89
|
+
* Null when the file cannot be read (callers treat that as "unknown").
|
|
90
|
+
*/
|
|
91
|
+
export async function hashFileStreaming(localPath) {
|
|
92
|
+
const hash = crypto.createHash("sha256");
|
|
93
|
+
let byteSize = 0;
|
|
94
|
+
try {
|
|
95
|
+
for await (const piece of createReadStream(localPath)) {
|
|
96
|
+
hash.update(piece);
|
|
97
|
+
byteSize += piece.byteLength;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
return { digest: hash.digest("hex"), byteSize };
|
|
104
|
+
}
|
|
105
|
+
async function openStaged(localPath, openFile) {
|
|
106
|
+
try {
|
|
107
|
+
return await openFile(localPath, "r");
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
if (error.code === "ENOENT") {
|
|
111
|
+
throw new StagedFileError("staged_file_missing", "ENOENT");
|
|
112
|
+
}
|
|
113
|
+
throw error;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { isLocalDashboardUrl } from "./dev-build.js";
|
|
1
|
+
import { RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
|
|
3
2
|
import { readBeginDispositions } from "./evidence-upload-plan.js";
|
|
4
|
-
import { failedOutcome } from "./evidence-upload-object.js";
|
|
5
3
|
import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
|
|
6
4
|
import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
|
|
7
5
|
/**
|
|
@@ -30,7 +28,7 @@ export async function rekeyConflictedEntry(options, entry, disposition, chunkSiz
|
|
|
30
28
|
reason: disposition.reason,
|
|
31
29
|
kind: entry.file.kind ?? "unknown",
|
|
32
30
|
content_hash_prefix: contentHashPrefix,
|
|
33
|
-
byte_size: entry.
|
|
31
|
+
byte_size: entry.byteSize,
|
|
34
32
|
}));
|
|
35
33
|
return { entry, disposition };
|
|
36
34
|
}
|
|
@@ -50,7 +48,7 @@ export async function rekeyConflictedEntry(options, entry, disposition, chunkSiz
|
|
|
50
48
|
reason: disposition.reason,
|
|
51
49
|
kind: entry.file.kind ?? "unknown",
|
|
52
50
|
content_hash_prefix: contentHashPrefix,
|
|
53
|
-
byte_size: entry.
|
|
51
|
+
byte_size: entry.byteSize,
|
|
54
52
|
chunk_count: entry.chunkCount,
|
|
55
53
|
rekeyed_disposition: rekeyedDisposition.disposition,
|
|
56
54
|
rekeyed_reason: rekeyedDisposition.reason ?? "none",
|
|
@@ -185,153 +183,6 @@ function conformReasonLabel(reason) {
|
|
|
185
183
|
const conformed = reason.replace(/[^a-z0-9_:.-]/gi, "_").slice(0, 120);
|
|
186
184
|
return conformed.length > 0 ? conformed : "upload_failed_unlabelled";
|
|
187
185
|
}
|
|
188
|
-
/**
|
|
189
|
-
* The most raw bytes the legacy single-shot route is offered on a hosted dashboard (BLI-3936).
|
|
190
|
-
*
|
|
191
|
-
* The route carries the file as base64 inside one JSON body, and Vercel refuses a body over about
|
|
192
|
-
* 4.5 MB before our code runs (a 93-byte `text/plain` 413). 3 MiB of raw bytes is 4 MiB of base64
|
|
193
|
-
* plus a few KB of pointer: the exact size the chunk route already sends in every chunk body, so
|
|
194
|
-
* it is proven to pass. Above it the request cannot succeed, only cost a full upload of the object
|
|
195
|
-
* to be told so (14 MB, every backoff window, on the reference Mac).
|
|
196
|
-
*/
|
|
197
|
-
export const LEGACY_SINGLE_SHOT_MAX_BYTES = RAW_EVIDENCE_UPLOAD_MAX_CHUNK_BYTES;
|
|
198
|
-
/** A 413 from the legacy route: the host's body limit refused bytes we expected to fit. Terminal for those bytes. */
|
|
199
|
-
export const EVIDENCE_TOO_LARGE_HTTP_413_REASON = "evidence_too_large_http_413";
|
|
200
|
-
/** The reason an object too big for the legacy route carries while the chunk route is down. Retryable: `begin` recovering delivers it. */
|
|
201
|
-
export const LEGACY_SINGLE_SHOT_TOO_LARGE_REASON = "legacy_single_shot_too_large_chunk_route_unavailable";
|
|
202
|
-
/**
|
|
203
|
-
* Older dashboards predate the chunk endpoints. Fall back to one legacy
|
|
204
|
-
* request per file so an oversized or already-existing object fails (or
|
|
205
|
-
* reuses) individually instead of dragging the whole batch down. On a hosted
|
|
206
|
-
* dashboard an object over `LEGACY_SINGLE_SHOT_MAX_BYTES` is not sent at all
|
|
207
|
-
* (BLI-3936): the host's body limit refuses it for certain. A non-serverless
|
|
208
|
-
* dashboard (the default localhost deployment) is still attempted at any size,
|
|
209
|
-
* like the old CLI, and a platform refusal comes back as a labeled per-file
|
|
210
|
-
* failure; a 413 there is `evidence_too_large_http_413`, terminal (`terminalDeliveryReason`).
|
|
211
|
-
*/
|
|
212
|
-
export async function uploadWithLegacyFallback(options, loaded, outcomes) {
|
|
213
|
-
for (const entry of loaded) {
|
|
214
|
-
let outcome;
|
|
215
|
-
if (legacyRouteCannotCarry(options, entry.bytes.byteLength)) {
|
|
216
|
-
// Said out loud, with the numbers: this object was NOT sent, because the route would refuse it.
|
|
217
|
-
console.error("[evidence-upload] legacy single-shot upload not attempted; the object is too large for it and the chunk route is unavailable", JSON.stringify({
|
|
218
|
-
reason: LEGACY_SINGLE_SHOT_TOO_LARGE_REASON,
|
|
219
|
-
kind: entry.file.kind ?? "unknown",
|
|
220
|
-
byte_size: entry.bytes.byteLength,
|
|
221
|
-
limit_bytes: LEGACY_SINGLE_SHOT_MAX_BYTES,
|
|
222
|
-
content_hash_prefix: (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12),
|
|
223
|
-
next: "retried through the chunk route once `begin` answers again",
|
|
224
|
-
}));
|
|
225
|
-
outcome = failedOutcome(entry.file, LEGACY_SINGLE_SHOT_TOO_LARGE_REASON);
|
|
226
|
-
outcomes.push(outcome);
|
|
227
|
-
for (const duplicate of entry.duplicates)
|
|
228
|
-
outcomes.push(duplicateOutcome(outcome, duplicate));
|
|
229
|
-
continue;
|
|
230
|
-
}
|
|
231
|
-
const response = await requestJson(options, "/api/ambient/evidence/upload", {
|
|
232
|
-
schema_version: "ambient-raw-evidence-upload.v1",
|
|
233
|
-
generated_at: options.generatedAt,
|
|
234
|
-
provenance: options.provenance,
|
|
235
|
-
files: [
|
|
236
|
-
{
|
|
237
|
-
pointer: entry.file.pointer,
|
|
238
|
-
content_base64: entry.bytes.toString("base64"),
|
|
239
|
-
},
|
|
240
|
-
],
|
|
241
|
-
});
|
|
242
|
-
outcome = legacyUploadOutcome(entry.file, response);
|
|
243
|
-
outcomes.push(outcome);
|
|
244
|
-
for (const duplicate of entry.duplicates) {
|
|
245
|
-
outcomes.push(duplicateOutcome(outcome, duplicate));
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
return summarizeOutcomes(outcomes, true);
|
|
249
|
-
}
|
|
250
|
-
/** True when sending this many raw bytes to the legacy route is certain to be refused by the host's body limit. */
|
|
251
|
-
function legacyRouteCannotCarry(options, byteSize) {
|
|
252
|
-
return byteSize > LEGACY_SINGLE_SHOT_MAX_BYTES && !isLocalDashboardUrl(options.dashboardUrl);
|
|
253
|
-
}
|
|
254
|
-
function legacyUploadOutcome(file, response) {
|
|
255
|
-
let outcome;
|
|
256
|
-
if (response.ok) {
|
|
257
|
-
const parsedLegacy = RawEvidenceLegacyUploadResponseSchema.safeParse(response.body);
|
|
258
|
-
if (!parsedLegacy.success) {
|
|
259
|
-
outcome = failedOutcome(file, "legacy_upload_invalid_response");
|
|
260
|
-
}
|
|
261
|
-
else {
|
|
262
|
-
const receipt = parsedLegacy.data.uploaded.find((candidate) => candidate.raw_evidence_pointer_id ===
|
|
263
|
-
file.pointer.raw_evidence_pointer_id);
|
|
264
|
-
if (!receipt ||
|
|
265
|
-
parsedLegacy.data.bucket !== file.pointer.storage_bucket ||
|
|
266
|
-
!legacyUploadReceiptMatchesPointer(file.pointer, receipt)) {
|
|
267
|
-
outcome = failedOutcome(file, "legacy_upload_receipt_mismatch");
|
|
268
|
-
}
|
|
269
|
-
else {
|
|
270
|
-
const pointer = pointerWithLegacyUploadResponse(file.pointer, parsedLegacy.data);
|
|
271
|
-
outcome = {
|
|
272
|
-
pointer,
|
|
273
|
-
object_key: file.pointer.object_key ?? "",
|
|
274
|
-
codex_session_id: file.codex_session_id ?? null,
|
|
275
|
-
kind: file.kind ?? "unknown",
|
|
276
|
-
artifact_metadata: file.artifact_metadata,
|
|
277
|
-
upload_state: "uploaded",
|
|
278
|
-
reason: "legacy_single_shot_upload",
|
|
279
|
-
uploaded_chunk_count: 1,
|
|
280
|
-
};
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
else if (response.status === 409) {
|
|
285
|
-
// The pre-ledger route cannot prove that a conflicting object contains
|
|
286
|
-
// these exact bytes. Fail closed instead of seeding the cursor with an
|
|
287
|
-
// unverified pointer.
|
|
288
|
-
outcome = failedOutcome(file, "legacy_upload_conflict_http_409");
|
|
289
|
-
}
|
|
290
|
-
else if (response.status === 413) {
|
|
291
|
-
// BLI-3936: the host refused this body for its size. The same bytes meet the same limit on
|
|
292
|
-
// every retry, so this is terminal (`terminalDeliveryReason`) and says what it is.
|
|
293
|
-
outcome = failedOutcome(file, EVIDENCE_TOO_LARGE_HTTP_413_REASON);
|
|
294
|
-
}
|
|
295
|
-
else {
|
|
296
|
-
outcome = failedOutcome(file, `legacy_upload_failed_http_${response.status}`);
|
|
297
|
-
}
|
|
298
|
-
return outcome;
|
|
299
|
-
}
|
|
300
|
-
function pointerWithLegacyUploadResponse(pointer, body) {
|
|
301
|
-
const entry = body.uploaded.find((candidate) => candidate.raw_evidence_pointer_id === pointer.raw_evidence_pointer_id);
|
|
302
|
-
if (!entry)
|
|
303
|
-
return pointer;
|
|
304
|
-
return pointerWithUploadedMetadata(pointer, {
|
|
305
|
-
content_hash_sha256: entry.content_hash_sha256,
|
|
306
|
-
byte_size: entry.byte_size,
|
|
307
|
-
redaction: entry.redaction,
|
|
308
|
-
});
|
|
309
|
-
}
|
|
310
|
-
function legacyUploadReceiptMatchesPointer(pointer, receipt) {
|
|
311
|
-
if (receipt.object_key !== pointer.object_key)
|
|
312
|
-
return false;
|
|
313
|
-
if (receipt.redaction) {
|
|
314
|
-
return (receipt.redaction.original_content_hash_sha256 ===
|
|
315
|
-
pointer.content_hash_sha256 &&
|
|
316
|
-
receipt.redaction.original_byte_size === pointer.byte_size &&
|
|
317
|
-
receipt.redaction.sanitized_content_hash_sha256 ===
|
|
318
|
-
receipt.content_hash_sha256 &&
|
|
319
|
-
receipt.redaction.sanitized_byte_size === receipt.byte_size);
|
|
320
|
-
}
|
|
321
|
-
return (receipt.content_hash_sha256 === pointer.content_hash_sha256 &&
|
|
322
|
-
receipt.byte_size === pointer.byte_size);
|
|
323
|
-
}
|
|
324
|
-
function pointerWithUploadedMetadata(pointer, metadata) {
|
|
325
|
-
const redaction = RawEvidenceRedactionMetadataSchema.safeParse(metadata.redaction);
|
|
326
|
-
return {
|
|
327
|
-
...pointer,
|
|
328
|
-
content_hash_sha256: typeof metadata.content_hash_sha256 === "string"
|
|
329
|
-
? metadata.content_hash_sha256
|
|
330
|
-
: pointer.content_hash_sha256,
|
|
331
|
-
byte_size: typeof metadata.byte_size === "number" ? metadata.byte_size : pointer.byte_size,
|
|
332
|
-
redaction: redaction.success ? redaction.data : pointer.redaction,
|
|
333
|
-
};
|
|
334
|
-
}
|
|
335
186
|
export function summarizeOutcomes(outcomes, usedLegacyFallback) {
|
|
336
187
|
const uploaded = outcomes.filter((outcome) => outcome.upload_state === "uploaded");
|
|
337
188
|
return {
|
|
@@ -343,4 +194,15 @@ export function summarizeOutcomes(outcomes, usedLegacyFallback) {
|
|
|
343
194
|
failed_count: outcomes.filter((outcome) => outcome.upload_state === "upload_failed").length,
|
|
344
195
|
used_legacy_fallback: usedLegacyFallback,
|
|
345
196
|
};
|
|
197
|
+
}
|
|
198
|
+
export function pointerWithUploadedMetadata(pointer, metadata) {
|
|
199
|
+
const redaction = RawEvidenceRedactionMetadataSchema.safeParse(metadata.redaction);
|
|
200
|
+
return {
|
|
201
|
+
...pointer,
|
|
202
|
+
content_hash_sha256: typeof metadata.content_hash_sha256 === "string"
|
|
203
|
+
? metadata.content_hash_sha256
|
|
204
|
+
: pointer.content_hash_sha256,
|
|
205
|
+
byte_size: typeof metadata.byte_size === "number" ? metadata.byte_size : pointer.byte_size,
|
|
206
|
+
redaction: redaction.success ? redaction.data : pointer.redaction,
|
|
207
|
+
};
|
|
346
208
|
}
|