@bli-cockpit/cli 0.2.4 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -7
- package/dist/adapters/attribution-core.js +152 -31
- package/dist/adapters/claude-attribution.js +14 -7
- package/dist/adapters/codex-attribution.js +8 -2
- package/dist/adapters/raw-evidence.js +33 -12
- package/dist/autostart.js +460 -20
- package/dist/commands/backfill.js +582 -87
- package/dist/commands/doctor.js +72 -60
- package/dist/commands/local-args.js +19 -2
- package/dist/commands/local.js +518 -247
- package/dist/commands/public-root.js +6 -1
- package/dist/commands/session-sync.js +259 -54
- package/dist/cursors/backfill-cursor.js +169 -1
- package/dist/cursors/raw-evidence-cursor.js +16 -2
- package/dist/evidence-upload-client.js +139 -107
- package/dist/local-state.js +82 -6
- package/dist/onboarding-roots.js +42 -17
- package/dist/process-runner.js +103 -0
- package/dist/raw-evidence-attribution-policy.js +25 -0
- package/dist/repo-identity.js +142 -26
- package/dist/root-normalization.js +29 -16
- package/dist/spool/install-event-outbox.js +191 -0
- package/dist/spool/local-spool.js +335 -58
- package/dist/upload.js +309 -66
- package/package.json +2 -2
|
@@ -3,6 +3,8 @@ import fs from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
export const BACKFILL_CURSOR_FILENAME = "backfill.json";
|
|
5
5
|
export const BACKFILL_COMPLETION_MARKER_FILENAME = "backfill-complete.json";
|
|
6
|
+
export const BACKFILL_COVERAGE_VERSION = "redacted-session-backfill.v3";
|
|
7
|
+
export const BACKFILL_COMPLETION_RECHECK_MS = 24 * 60 * 60 * 1_000;
|
|
6
8
|
export function emptyBackfillCursorState() {
|
|
7
9
|
return {
|
|
8
10
|
schema_version: "cockpit-backfill-cursor.v1",
|
|
@@ -29,14 +31,98 @@ export async function writeBackfillCursor(paths, state) {
|
|
|
29
31
|
export async function writeBackfillCompletionMarker(paths, marker) {
|
|
30
32
|
await writePrivateJson(backfillCompletionMarkerPath(paths), marker);
|
|
31
33
|
}
|
|
34
|
+
export async function readBackfillCompletionMarker(paths) {
|
|
35
|
+
try {
|
|
36
|
+
const raw = JSON.parse(await fs.readFile(backfillCompletionMarkerPath(paths), "utf8"));
|
|
37
|
+
return parseBackfillCompletionMarker(raw);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The cursor is only reusable inside the exact approved-root scope that
|
|
45
|
+
* produced it. Without this binding, adding a root after an earlier completed
|
|
46
|
+
* backfill can make the global oldest-mtime cursor skip every session in the
|
|
47
|
+
* new root.
|
|
48
|
+
*/
|
|
49
|
+
export function prepareBackfillCursorForScope(cursor, collectionRoots, sources) {
|
|
50
|
+
const collectionScopeId = backfillCollectionScopeId(collectionRoots);
|
|
51
|
+
let reset = false;
|
|
52
|
+
for (const source of sources) {
|
|
53
|
+
if (cursor.sources[source].collection_scope_id === collectionScopeId) {
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
cursor.sources[source] = emptySourceCursor(collectionScopeId);
|
|
57
|
+
reset = true;
|
|
58
|
+
}
|
|
59
|
+
return {
|
|
60
|
+
cursor,
|
|
61
|
+
collection_scope_id: collectionScopeId,
|
|
62
|
+
reset,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
export function backfillCompletionCovers(marker, collectionRoots, requiredSources, now = new Date()) {
|
|
66
|
+
if (!marker)
|
|
67
|
+
return false;
|
|
68
|
+
if (marker.coverage_version !== BACKFILL_COVERAGE_VERSION)
|
|
69
|
+
return false;
|
|
70
|
+
if (marker.collection_scope_id !== backfillCollectionScopeId(collectionRoots)) {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
const revalidateAfter = Date.parse(marker.revalidate_after);
|
|
74
|
+
if (!Number.isFinite(revalidateAfter) || now.getTime() >= revalidateAfter) {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
const completedSources = new Set(marker.sources);
|
|
78
|
+
return requiredSources.every((source) => completedSources.has(source));
|
|
79
|
+
}
|
|
80
|
+
export function backfillCollectionScopeId(collectionRoots) {
|
|
81
|
+
const normalized = [...new Set(collectionRoots.map(normalizeScopeRoot))].sort();
|
|
82
|
+
return `scope-${crypto
|
|
83
|
+
.createHash("sha256")
|
|
84
|
+
.update(JSON.stringify(normalized))
|
|
85
|
+
.digest("hex")
|
|
86
|
+
.slice(0, 24)}`;
|
|
87
|
+
}
|
|
32
88
|
export function recordBackfillCursorObservations(cursor, observations, now) {
|
|
33
89
|
for (const observation of observations) {
|
|
34
90
|
const source = cursor.sources[observation.source] ?? emptySourceCursor();
|
|
91
|
+
const newest = source.newest_mtime_ms_covered;
|
|
92
|
+
if (newest === null ||
|
|
93
|
+
observation.session_file_mtime_ms > newest) {
|
|
94
|
+
source.newest_mtime_ms_covered = observation.session_file_mtime_ms;
|
|
95
|
+
source.newest_mtime_covered = observation.session_file_mtime;
|
|
96
|
+
source.processed_keys_at_newest_mtime = observation.cursor_key
|
|
97
|
+
? [observation.cursor_key]
|
|
98
|
+
: [];
|
|
99
|
+
}
|
|
100
|
+
else if (observation.session_file_mtime_ms === newest &&
|
|
101
|
+
observation.cursor_key) {
|
|
102
|
+
source.processed_keys_at_newest_mtime = [
|
|
103
|
+
...new Set([
|
|
104
|
+
...source.processed_keys_at_newest_mtime,
|
|
105
|
+
observation.cursor_key,
|
|
106
|
+
]),
|
|
107
|
+
].sort();
|
|
108
|
+
}
|
|
35
109
|
const oldest = source.oldest_mtime_ms_processed;
|
|
36
110
|
if (oldest === null ||
|
|
37
111
|
observation.session_file_mtime_ms < oldest) {
|
|
38
112
|
source.oldest_mtime_ms_processed = observation.session_file_mtime_ms;
|
|
39
113
|
source.oldest_mtime_processed = observation.session_file_mtime;
|
|
114
|
+
source.processed_keys_at_oldest_mtime = observation.cursor_key
|
|
115
|
+
? [observation.cursor_key]
|
|
116
|
+
: [];
|
|
117
|
+
}
|
|
118
|
+
else if (observation.session_file_mtime_ms === oldest &&
|
|
119
|
+
observation.cursor_key) {
|
|
120
|
+
source.processed_keys_at_oldest_mtime = [
|
|
121
|
+
...new Set([
|
|
122
|
+
...source.processed_keys_at_oldest_mtime,
|
|
123
|
+
observation.cursor_key,
|
|
124
|
+
]),
|
|
125
|
+
].sort();
|
|
40
126
|
}
|
|
41
127
|
source.state_counts[observation.state] =
|
|
42
128
|
(source.state_counts[observation.state] ?? 0) + 1;
|
|
@@ -46,16 +132,43 @@ export function recordBackfillCursorObservations(cursor, observations, now) {
|
|
|
46
132
|
}
|
|
47
133
|
cursor.updated_at = now.toISOString();
|
|
48
134
|
}
|
|
135
|
+
/**
|
|
136
|
+
* Marks the upper edge of an exhaustive, issue-free scan. This is deliberately
|
|
137
|
+
* separate from candidate observations: an empty session store is still a
|
|
138
|
+
* valid completed scan, and a future run must only revisit files newer than
|
|
139
|
+
* this edge plus any unfinished historical tail.
|
|
140
|
+
*/
|
|
141
|
+
export function recordBackfillScanCoverage(cursor, sources, coveredThrough, now) {
|
|
142
|
+
const coveredThroughMs = coveredThrough.getTime();
|
|
143
|
+
for (const sourceKey of sources) {
|
|
144
|
+
const source = cursor.sources[sourceKey] ?? emptySourceCursor();
|
|
145
|
+
if (source.newest_mtime_ms_covered === null ||
|
|
146
|
+
coveredThroughMs > source.newest_mtime_ms_covered) {
|
|
147
|
+
source.newest_mtime_ms_covered = coveredThroughMs;
|
|
148
|
+
source.newest_mtime_covered = coveredThrough.toISOString();
|
|
149
|
+
// No file identity is implied by a wall-clock scan boundary. A file
|
|
150
|
+
// later observed at this exact timestamp remains eligible.
|
|
151
|
+
source.processed_keys_at_newest_mtime = [];
|
|
152
|
+
}
|
|
153
|
+
cursor.sources[sourceKey] = source;
|
|
154
|
+
}
|
|
155
|
+
cursor.updated_at = now.toISOString();
|
|
156
|
+
}
|
|
49
157
|
export function backfillCursorPath(paths) {
|
|
50
158
|
return path.join(paths.cursors_dir, BACKFILL_CURSOR_FILENAME);
|
|
51
159
|
}
|
|
52
160
|
export function backfillCompletionMarkerPath(paths) {
|
|
53
161
|
return path.join(paths.cursors_dir, BACKFILL_COMPLETION_MARKER_FILENAME);
|
|
54
162
|
}
|
|
55
|
-
function emptySourceCursor() {
|
|
163
|
+
function emptySourceCursor(collectionScopeId = null) {
|
|
56
164
|
return {
|
|
165
|
+
collection_scope_id: collectionScopeId,
|
|
166
|
+
newest_mtime_ms_covered: null,
|
|
167
|
+
newest_mtime_covered: null,
|
|
168
|
+
processed_keys_at_newest_mtime: [],
|
|
57
169
|
oldest_mtime_ms_processed: null,
|
|
58
170
|
oldest_mtime_processed: null,
|
|
171
|
+
processed_keys_at_oldest_mtime: [],
|
|
59
172
|
state_counts: {},
|
|
60
173
|
reason_counts: {},
|
|
61
174
|
};
|
|
@@ -78,15 +191,63 @@ function parseSourceCursor(value) {
|
|
|
78
191
|
if (!value || typeof value !== "object")
|
|
79
192
|
return emptySourceCursor();
|
|
80
193
|
const record = value;
|
|
194
|
+
const newest = optionalNumber(record["newest_mtime_ms_covered"]);
|
|
81
195
|
const oldest = optionalNumber(record["oldest_mtime_ms_processed"]);
|
|
82
196
|
return {
|
|
197
|
+
collection_scope_id: optionalString(record["collection_scope_id"]),
|
|
198
|
+
newest_mtime_ms_covered: newest,
|
|
199
|
+
newest_mtime_covered: optionalString(record["newest_mtime_covered"]) ??
|
|
200
|
+
(newest === null ? null : new Date(newest).toISOString()),
|
|
201
|
+
processed_keys_at_newest_mtime: parseStringArray(record["processed_keys_at_newest_mtime"]),
|
|
83
202
|
oldest_mtime_ms_processed: oldest,
|
|
84
203
|
oldest_mtime_processed: optionalString(record["oldest_mtime_processed"]) ??
|
|
85
204
|
(oldest === null ? null : new Date(oldest).toISOString()),
|
|
205
|
+
processed_keys_at_oldest_mtime: parseStringArray(record["processed_keys_at_oldest_mtime"]),
|
|
86
206
|
state_counts: parseNumberRecord(record["state_counts"]),
|
|
87
207
|
reason_counts: parseNumberRecord(record["reason_counts"]),
|
|
88
208
|
};
|
|
89
209
|
}
|
|
210
|
+
function parseBackfillCompletionMarker(value) {
|
|
211
|
+
if (!value || typeof value !== "object")
|
|
212
|
+
return null;
|
|
213
|
+
const record = value;
|
|
214
|
+
if (record["schema_version"] !== "cockpit-backfill-complete.v2" ||
|
|
215
|
+
record["coverage_version"] !== BACKFILL_COVERAGE_VERSION) {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
const collectionScopeId = optionalString(record["collection_scope_id"]);
|
|
219
|
+
const completedAt = optionalString(record["completed_at"]);
|
|
220
|
+
const revalidateAfter = optionalString(record["revalidate_after"]);
|
|
221
|
+
const rawSources = record["sources"];
|
|
222
|
+
if (!collectionScopeId ||
|
|
223
|
+
!completedAt ||
|
|
224
|
+
!revalidateAfter ||
|
|
225
|
+
!Number.isFinite(Date.parse(revalidateAfter)) ||
|
|
226
|
+
!Array.isArray(rawSources)) {
|
|
227
|
+
return null;
|
|
228
|
+
}
|
|
229
|
+
const sources = [
|
|
230
|
+
...new Set(rawSources.filter((source) => source === "codex" || source === "claude_code")),
|
|
231
|
+
];
|
|
232
|
+
if (sources.length === 0)
|
|
233
|
+
return null;
|
|
234
|
+
return {
|
|
235
|
+
schema_version: "cockpit-backfill-complete.v2",
|
|
236
|
+
coverage_version: BACKFILL_COVERAGE_VERSION,
|
|
237
|
+
collection_scope_id: collectionScopeId,
|
|
238
|
+
sources,
|
|
239
|
+
completed_at: completedAt,
|
|
240
|
+
revalidate_after: revalidateAfter,
|
|
241
|
+
cursor: parseBackfillCursor(record["cursor"]),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
function normalizeScopeRoot(value) {
|
|
245
|
+
const windowsStyle = path.win32.isAbsolute(value) && !path.posix.isAbsolute(value);
|
|
246
|
+
if (windowsStyle)
|
|
247
|
+
return path.win32.normalize(value).toLowerCase();
|
|
248
|
+
const resolved = path.resolve(value);
|
|
249
|
+
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
250
|
+
}
|
|
90
251
|
function parseNumberRecord(value) {
|
|
91
252
|
if (!value || typeof value !== "object")
|
|
92
253
|
return {};
|
|
@@ -98,6 +259,13 @@ function parseNumberRecord(value) {
|
|
|
98
259
|
}
|
|
99
260
|
return out;
|
|
100
261
|
}
|
|
262
|
+
function parseStringArray(value) {
|
|
263
|
+
if (!Array.isArray(value))
|
|
264
|
+
return [];
|
|
265
|
+
return [
|
|
266
|
+
...new Set(value.filter((entry) => typeof entry === "string" && entry.trim().length > 0)),
|
|
267
|
+
].sort();
|
|
268
|
+
}
|
|
101
269
|
async function writePrivateJson(filePath, value) {
|
|
102
270
|
await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
103
271
|
const tempPath = `${filePath}.tmp-${process.pid}-${crypto.randomUUID()}`;
|
|
@@ -24,7 +24,8 @@ export const PRIMARY_CURSOR_FILENAME = "raw-evidence.json";
|
|
|
24
24
|
export const CLAUDE_CURSOR_FILENAME = "claude-raw-evidence.json";
|
|
25
25
|
// Sidecars can push a single sync past 500 objects; pruning at 500 would force
|
|
26
26
|
// re-read + re-begin of durable objects every sync thereafter. Entries are
|
|
27
|
-
// ~150 B, so 5,000 is ~750 KB on disk (D4).
|
|
27
|
+
// ~150 B, so 5,000 is ~750 KB on disk (D4). Undurable sessions are exempt from
|
|
28
|
+
// the session cap below because dropping retry state can strand evidence.
|
|
28
29
|
const MAX_TRACKED_OBJECTS = 5_000;
|
|
29
30
|
const MAX_TRACKED_SESSIONS = 5_000;
|
|
30
31
|
export function emptyRawEvidenceCursorState() {
|
|
@@ -98,9 +99,22 @@ function pruneCursorState(state) {
|
|
|
98
99
|
return {
|
|
99
100
|
...state,
|
|
100
101
|
objects: pruneNewest(state.objects, MAX_TRACKED_OBJECTS, (entry) => entry.committed_at),
|
|
101
|
-
sessions:
|
|
102
|
+
sessions: pruneSessionEntries(state.sessions),
|
|
102
103
|
};
|
|
103
104
|
}
|
|
105
|
+
function pruneSessionEntries(sessions) {
|
|
106
|
+
const entries = Object.entries(sessions);
|
|
107
|
+
if (entries.length <= MAX_TRACKED_SESSIONS)
|
|
108
|
+
return sessions;
|
|
109
|
+
const undurable = entries.filter(([, entry]) => !entry.uploaded_object_key);
|
|
110
|
+
const durable = entries.filter(([, entry]) => entry.uploaded_object_key);
|
|
111
|
+
durable.sort((a, b) => b[1].last_seen_at.localeCompare(a[1].last_seen_at));
|
|
112
|
+
const durableSlots = Math.max(0, MAX_TRACKED_SESSIONS - undurable.length);
|
|
113
|
+
return Object.fromEntries([
|
|
114
|
+
...undurable,
|
|
115
|
+
...durable.slice(0, durableSlots),
|
|
116
|
+
]);
|
|
117
|
+
}
|
|
104
118
|
function pruneNewest(record, max, sortKey) {
|
|
105
119
|
const entries = Object.entries(record);
|
|
106
120
|
if (entries.length <= max)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadCommitResponseSchema, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
@@ -97,7 +97,28 @@ export async function uploadRawEvidenceFilesChunked(options) {
|
|
|
97
97
|
}
|
|
98
98
|
continue;
|
|
99
99
|
}
|
|
100
|
-
const
|
|
100
|
+
const parsedBegin = RawEvidenceUploadBeginResponseSchema.safeParse(begin.body);
|
|
101
|
+
if (!parsedBegin.success) {
|
|
102
|
+
for (const entry of batch) {
|
|
103
|
+
outcomes.push(failedOutcome(entry.file, "begin_invalid_response"));
|
|
104
|
+
for (const duplicate of entry.duplicates) {
|
|
105
|
+
outcomes.push(failedOutcome(duplicate, "begin_invalid_response"));
|
|
106
|
+
}
|
|
107
|
+
resolvedEntries.add(entry);
|
|
108
|
+
}
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const dispositions = readBeginDispositions(parsedBegin.data);
|
|
112
|
+
if (!dispositions) {
|
|
113
|
+
for (const entry of batch) {
|
|
114
|
+
outcomes.push(failedOutcome(entry.file, "begin_invalid_response"));
|
|
115
|
+
for (const duplicate of entry.duplicates) {
|
|
116
|
+
outcomes.push(failedOutcome(duplicate, "begin_invalid_response"));
|
|
117
|
+
}
|
|
118
|
+
resolvedEntries.add(entry);
|
|
119
|
+
}
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
101
122
|
for (const entry of batch) {
|
|
102
123
|
const objectKey = entry.file.pointer.object_key ?? "";
|
|
103
124
|
const disposition = dispositions.get(objectKey);
|
|
@@ -109,6 +130,15 @@ export async function uploadRawEvidenceFilesChunked(options) {
|
|
|
109
130
|
resolvedEntries.add(entry);
|
|
110
131
|
continue;
|
|
111
132
|
}
|
|
133
|
+
if (disposition.raw_evidence_pointer_id !==
|
|
134
|
+
entry.file.pointer.raw_evidence_pointer_id) {
|
|
135
|
+
outcomes.push(failedOutcome(entry.file, "begin_disposition_mismatch"));
|
|
136
|
+
for (const duplicate of entry.duplicates) {
|
|
137
|
+
outcomes.push(failedOutcome(duplicate, "begin_disposition_mismatch"));
|
|
138
|
+
}
|
|
139
|
+
resolvedEntries.add(entry);
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
112
142
|
const outcome = await uploadOneObject(options, entry, disposition, chunkSizeBytes);
|
|
113
143
|
outcomes.push(outcome);
|
|
114
144
|
for (const duplicate of entry.duplicates) {
|
|
@@ -144,41 +174,37 @@ function duplicateOutcome(primary, duplicate) {
|
|
|
144
174
|
async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
145
175
|
const objectKey = entry.file.pointer.object_key ?? "";
|
|
146
176
|
if (disposition.disposition === "already_committed") {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
codex_session_id: entry.file.codex_session_id ?? null,
|
|
151
|
-
kind: entry.file.kind ?? "unknown",
|
|
152
|
-
artifact_metadata: entry.file.artifact_metadata,
|
|
153
|
-
upload_state: "reused_existing",
|
|
154
|
-
reason: "already_committed",
|
|
155
|
-
uploaded_chunk_count: 0,
|
|
156
|
-
};
|
|
177
|
+
if (!disposition.upload_id) {
|
|
178
|
+
return failedOutcome(entry.file, "begin_committed_receipt_unavailable");
|
|
179
|
+
}
|
|
157
180
|
}
|
|
158
181
|
if (disposition.disposition === "conflict" || !disposition.upload_id) {
|
|
159
182
|
return failedOutcome(entry.file, disposition.reason ?? "upload_conflict");
|
|
160
183
|
}
|
|
161
184
|
const received = new Set(disposition.received_chunk_indexes);
|
|
162
185
|
let uploadedChunks = 0;
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
186
|
+
if (disposition.disposition !== "already_committed" &&
|
|
187
|
+
!disposition.commit_ready) {
|
|
188
|
+
for (let index = 0; index < entry.chunkCount; index += 1) {
|
|
189
|
+
if (received.has(index))
|
|
190
|
+
continue;
|
|
191
|
+
const chunk = entry.bytes.subarray(index * chunkSizeBytes, Math.min((index + 1) * chunkSizeBytes, entry.bytes.byteLength));
|
|
192
|
+
const chunkResponse = await requestJson(options, "/api/ambient/evidence/upload/chunk", {
|
|
193
|
+
schema_version: "ambient-raw-evidence-upload-chunk.v1",
|
|
194
|
+
generated_at: options.generatedAt,
|
|
195
|
+
provenance: options.provenance,
|
|
196
|
+
upload_id: disposition.upload_id,
|
|
197
|
+
object_key: objectKey,
|
|
198
|
+
chunk_index: index,
|
|
199
|
+
chunk_count: entry.chunkCount,
|
|
200
|
+
chunk_hash_sha256: sha256(chunk),
|
|
201
|
+
content_base64: chunk.toString("base64"),
|
|
202
|
+
});
|
|
203
|
+
if (!chunkResponse.ok) {
|
|
204
|
+
return failedOutcome(entry.file, `chunk_${index}_failed_http_${chunkResponse.status}`, uploadedChunks);
|
|
205
|
+
}
|
|
206
|
+
uploadedChunks += 1;
|
|
180
207
|
}
|
|
181
|
-
uploadedChunks += 1;
|
|
182
208
|
}
|
|
183
209
|
const commit = await requestJson(options, "/api/ambient/evidence/upload/commit", {
|
|
184
210
|
schema_version: "ambient-raw-evidence-upload-commit.v1",
|
|
@@ -188,17 +214,22 @@ async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
|
188
214
|
object_key: objectKey,
|
|
189
215
|
});
|
|
190
216
|
if (!commit.ok) {
|
|
191
|
-
|
|
217
|
+
const detail = safeFailureDetail(commit.body);
|
|
218
|
+
return failedOutcome(entry.file, `commit_failed_http_${commit.status}${detail ? `_${detail}` : ""}`, uploadedChunks);
|
|
192
219
|
}
|
|
193
220
|
// "already_committed" means a concurrent or earlier sync made these bytes
|
|
194
221
|
// durable; this sync does not own them, so they must be reported as reuse —
|
|
195
222
|
// a later ingest failure here must not clean up an object another sync's
|
|
196
223
|
// indexed refs already point at.
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
if (
|
|
224
|
+
const parsedCommit = RawEvidenceUploadCommitResponseSchema.safeParse(commit.body);
|
|
225
|
+
if (!parsedCommit.success) {
|
|
226
|
+
return failedOutcome(entry.file, "commit_invalid_response", uploadedChunks);
|
|
227
|
+
}
|
|
228
|
+
if (!chunkCommitReceiptMatchesPointer(entry.file.pointer, parsedCommit.data)) {
|
|
229
|
+
return failedOutcome(entry.file, "commit_receipt_mismatch", uploadedChunks);
|
|
230
|
+
}
|
|
231
|
+
const committedPointer = entry.file.pointer;
|
|
232
|
+
if (parsedCommit.data.status === "already_committed") {
|
|
202
233
|
return {
|
|
203
234
|
pointer: committedPointer,
|
|
204
235
|
object_key: objectKey,
|
|
@@ -244,31 +275,38 @@ async function uploadWithLegacyFallback(options, loaded, outcomes) {
|
|
|
244
275
|
],
|
|
245
276
|
});
|
|
246
277
|
if (response.ok) {
|
|
247
|
-
const
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
278
|
+
const parsedLegacy = RawEvidenceLegacyUploadResponseSchema.safeParse(response.body);
|
|
279
|
+
if (!parsedLegacy.success) {
|
|
280
|
+
outcome = failedOutcome(entry.file, "legacy_upload_invalid_response");
|
|
281
|
+
}
|
|
282
|
+
else {
|
|
283
|
+
const receipt = parsedLegacy.data.uploaded.find((candidate) => candidate.raw_evidence_pointer_id ===
|
|
284
|
+
entry.file.pointer.raw_evidence_pointer_id);
|
|
285
|
+
if (!receipt ||
|
|
286
|
+
parsedLegacy.data.bucket !== entry.file.pointer.storage_bucket ||
|
|
287
|
+
!legacyUploadReceiptMatchesPointer(entry.file.pointer, receipt)) {
|
|
288
|
+
outcome = failedOutcome(entry.file, "legacy_upload_receipt_mismatch");
|
|
289
|
+
}
|
|
290
|
+
else {
|
|
291
|
+
const pointer = pointerWithLegacyUploadResponse(entry.file.pointer, parsedLegacy.data);
|
|
292
|
+
outcome = {
|
|
293
|
+
pointer,
|
|
294
|
+
object_key: entry.file.pointer.object_key ?? "",
|
|
295
|
+
codex_session_id: entry.file.codex_session_id ?? null,
|
|
296
|
+
kind: entry.file.kind ?? "unknown",
|
|
297
|
+
artifact_metadata: entry.file.artifact_metadata,
|
|
298
|
+
upload_state: "uploaded",
|
|
299
|
+
reason: "legacy_single_shot_upload",
|
|
300
|
+
uploaded_chunk_count: 1,
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
}
|
|
258
304
|
}
|
|
259
305
|
else if (response.status === 409) {
|
|
260
|
-
//
|
|
261
|
-
//
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
object_key: entry.file.pointer.object_key ?? "",
|
|
265
|
-
codex_session_id: entry.file.codex_session_id ?? null,
|
|
266
|
-
kind: entry.file.kind ?? "unknown",
|
|
267
|
-
artifact_metadata: entry.file.artifact_metadata,
|
|
268
|
-
upload_state: "reused_existing",
|
|
269
|
-
reason: "legacy_already_exists",
|
|
270
|
-
uploaded_chunk_count: 0,
|
|
271
|
-
};
|
|
306
|
+
// The pre-ledger route cannot prove that a conflicting object contains
|
|
307
|
+
// these exact bytes. Fail closed instead of seeding the cursor with an
|
|
308
|
+
// unverified pointer.
|
|
309
|
+
outcome = failedOutcome(entry.file, "legacy_upload_conflict_http_409");
|
|
272
310
|
}
|
|
273
311
|
else {
|
|
274
312
|
outcome = failedOutcome(entry.file, `legacy_upload_failed_http_${response.status}`);
|
|
@@ -312,27 +350,16 @@ async function requestJson(options, routePath, body) {
|
|
|
312
350
|
}
|
|
313
351
|
function readBeginDispositions(body) {
|
|
314
352
|
const map = new Map();
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
const disposition = record["disposition"];
|
|
326
|
-
if (typeof objectKey !== "string" || typeof disposition !== "string") {
|
|
327
|
-
continue;
|
|
328
|
-
}
|
|
329
|
-
map.set(objectKey, {
|
|
330
|
-
disposition,
|
|
331
|
-
upload_id: typeof record["upload_id"] === "string" ? record["upload_id"] : null,
|
|
332
|
-
received_chunk_indexes: Array.isArray(record["received_chunk_indexes"])
|
|
333
|
-
? record["received_chunk_indexes"].filter((value) => typeof value === "number")
|
|
334
|
-
: [],
|
|
335
|
-
reason: typeof record["reason"] === "string" ? record["reason"] : null,
|
|
353
|
+
for (const entry of body.objects) {
|
|
354
|
+
if (map.has(entry.object_key))
|
|
355
|
+
return null;
|
|
356
|
+
map.set(entry.object_key, {
|
|
357
|
+
disposition: entry.disposition,
|
|
358
|
+
upload_id: entry.upload_id ?? null,
|
|
359
|
+
received_chunk_indexes: entry.received_chunk_indexes,
|
|
360
|
+
commit_ready: entry.commit_ready === true,
|
|
361
|
+
reason: entry.reason ?? null,
|
|
362
|
+
raw_evidence_pointer_id: entry.raw_evidence_pointer_id,
|
|
336
363
|
});
|
|
337
364
|
}
|
|
338
365
|
return map;
|
|
@@ -349,39 +376,44 @@ function failedOutcome(file, reason, uploadedChunks = 0) {
|
|
|
349
376
|
uploaded_chunk_count: uploadedChunks,
|
|
350
377
|
};
|
|
351
378
|
}
|
|
352
|
-
function
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
return
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
redaction: parsed.data.redaction,
|
|
360
|
-
});
|
|
379
|
+
function safeFailureDetail(body) {
|
|
380
|
+
if (!body || typeof body !== "object")
|
|
381
|
+
return null;
|
|
382
|
+
const reason = body.reason;
|
|
383
|
+
return typeof reason === "string" && /^[a-z0-9_:-]{1,80}$/i.test(reason)
|
|
384
|
+
? reason
|
|
385
|
+
: null;
|
|
361
386
|
}
|
|
362
387
|
function pointerWithLegacyUploadResponse(pointer, body) {
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
const uploaded = body.uploaded;
|
|
366
|
-
if (!Array.isArray(uploaded))
|
|
367
|
-
return pointer;
|
|
368
|
-
const entry = uploaded.find((candidate) => {
|
|
369
|
-
if (!candidate || typeof candidate !== "object")
|
|
370
|
-
return false;
|
|
371
|
-
return (candidate
|
|
372
|
-
.raw_evidence_pointer_id === pointer.raw_evidence_pointer_id);
|
|
373
|
-
});
|
|
374
|
-
if (!entry || typeof entry !== "object")
|
|
388
|
+
const entry = body.uploaded.find((candidate) => candidate.raw_evidence_pointer_id === pointer.raw_evidence_pointer_id);
|
|
389
|
+
if (!entry)
|
|
375
390
|
return pointer;
|
|
376
|
-
const record = entry;
|
|
377
391
|
return pointerWithUploadedMetadata(pointer, {
|
|
378
|
-
content_hash_sha256:
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
byte_size: typeof record["byte_size"] === "number" ? record["byte_size"] : undefined,
|
|
382
|
-
redaction: record["redaction"],
|
|
392
|
+
content_hash_sha256: entry.content_hash_sha256,
|
|
393
|
+
byte_size: entry.byte_size,
|
|
394
|
+
redaction: entry.redaction,
|
|
383
395
|
});
|
|
384
396
|
}
|
|
397
|
+
function chunkCommitReceiptMatchesPointer(pointer, receipt) {
|
|
398
|
+
return (receipt.object_key === pointer.object_key &&
|
|
399
|
+
receipt.content_hash_sha256 === pointer.content_hash_sha256 &&
|
|
400
|
+
receipt.byte_size === pointer.byte_size &&
|
|
401
|
+
receipt.redaction === undefined);
|
|
402
|
+
}
|
|
403
|
+
function legacyUploadReceiptMatchesPointer(pointer, receipt) {
|
|
404
|
+
if (receipt.object_key !== pointer.object_key)
|
|
405
|
+
return false;
|
|
406
|
+
if (receipt.redaction) {
|
|
407
|
+
return (receipt.redaction.original_content_hash_sha256 ===
|
|
408
|
+
pointer.content_hash_sha256 &&
|
|
409
|
+
receipt.redaction.original_byte_size === pointer.byte_size &&
|
|
410
|
+
receipt.redaction.sanitized_content_hash_sha256 ===
|
|
411
|
+
receipt.content_hash_sha256 &&
|
|
412
|
+
receipt.redaction.sanitized_byte_size === receipt.byte_size);
|
|
413
|
+
}
|
|
414
|
+
return (receipt.content_hash_sha256 === pointer.content_hash_sha256 &&
|
|
415
|
+
receipt.byte_size === pointer.byte_size);
|
|
416
|
+
}
|
|
385
417
|
function pointerWithUploadedMetadata(pointer, metadata) {
|
|
386
418
|
const redaction = RawEvidenceRedactionMetadataSchema.safeParse(metadata.redaction);
|
|
387
419
|
return {
|