@bli-cockpit/cli 0.2.30 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/agent-image-evidence.js +4 -0
- package/dist/adapters/attribution-core.js +12 -0
- package/dist/adapters/car-state.js +12 -1
- package/dist/adapters/claude-attribution.js +81 -7
- package/dist/adapters/codex-attribution.js +37 -3
- package/dist/adapters/raw-evidence-manifest.js +12 -1
- package/dist/adapters/raw-evidence-pack-store.js +28 -3
- package/dist/adapters/raw-evidence-sanitize.js +46 -2
- package/dist/adapters/raw-evidence.js +51 -6
- package/dist/agent-rules.js +34 -3
- package/dist/autostart.js +3 -0
- package/dist/backfill-lock.js +22 -1
- package/dist/commands/backfill.js +41 -7
- package/dist/commands/cli-io.js +3 -0
- package/dist/commands/doctor.js +35 -4
- package/dist/commands/install-receipts.js +43 -6
- package/dist/commands/install-update.js +3 -1
- package/dist/commands/local-args.js +4 -0
- package/dist/commands/local-auth.js +14 -0
- package/dist/commands/local.js +20 -1
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync.js +35 -6
- package/dist/commands/status.js +17 -1
- package/dist/cursors/backfill-cursor.js +23 -2
- package/dist/cursors/raw-evidence-cursor.js +14 -1
- package/dist/discovery-limits.js +12 -1
- package/dist/evidence-upload-client.js +41 -4
- package/dist/health-detail.js +111 -2
- package/dist/local-state.js +98 -11
- package/dist/onboarding-roots.js +3 -0
- package/dist/raw-evidence-attribution-policy.js +7 -0
- package/dist/raw-evidence-gc.js +5 -0
- package/dist/raw-evidence-staging.js +12 -1
- package/dist/repo-identity.js +15 -1
- package/dist/scheduled-self-update.js +14 -1
- package/dist/spool/install-event-outbox.js +11 -1
- package/dist/spool/local-spool.js +3 -0
- package/dist/sync-lock.js +24 -1
- package/dist/upload-agent-artifacts.js +11 -1
- package/dist/upload-envelope.js +30 -3
- package/dist/upload-http.js +10 -0
- package/dist/upload-session-reports.js +36 -3
- package/dist/upload.js +16 -5
- package/package.json +2 -2
|
@@ -29,6 +29,10 @@ export async function collectAgentImageEvidenceFromJsonlFile(options) {
|
|
|
29
29
|
record = JSON.parse(line);
|
|
30
30
|
}
|
|
31
31
|
catch {
|
|
32
|
+
// Deliberately silent (BLI-3238). Per line of a transcript, and this
|
|
33
|
+
// scan is looking for image records specifically — a line it cannot
|
|
34
|
+
// parse is a line that was not going to be one. The transcript's own
|
|
35
|
+
// adapter already counts parse errors at the session grain.
|
|
32
36
|
continue;
|
|
33
37
|
}
|
|
34
38
|
for (const candidate of imageCandidatesFromRecord(record, {
|
|
@@ -347,6 +347,10 @@ function fallbackToTranscriptOriginForDeletedRepo(options) {
|
|
|
347
347
|
return !options.pathExists?.(value);
|
|
348
348
|
}
|
|
349
349
|
catch {
|
|
350
|
+
// Deliberately silent (BLI-3238). This is an existence PROBE and
|
|
351
|
+
// failure is the answer: a path we cannot test is a path we cannot
|
|
352
|
+
// claim is missing, so the candidate is passed over. The scorer is
|
|
353
|
+
// pure and synchronous by design and has no channel to report on.
|
|
350
354
|
return false;
|
|
351
355
|
}
|
|
352
356
|
}) ?? null;
|
|
@@ -386,6 +390,10 @@ function terminalReasonForRecordedPaths(paths, pathExists) {
|
|
|
386
390
|
return pathExists(value);
|
|
387
391
|
}
|
|
388
392
|
catch {
|
|
393
|
+
// Deliberately silent (BLI-3238), same probe as above: "cannot test"
|
|
394
|
+
// folds into "does not exist", which is the conservative direction —
|
|
395
|
+
// it keeps the older `repo_not_on_disk` label rather than inventing
|
|
396
|
+
// `cwd_not_a_repo`.
|
|
389
397
|
return false;
|
|
390
398
|
}
|
|
391
399
|
});
|
|
@@ -514,6 +522,10 @@ function canonicalExistingAttributionPath(value) {
|
|
|
514
522
|
return attributionPathApi(value).resolve(realpathSync(value));
|
|
515
523
|
}
|
|
516
524
|
catch {
|
|
525
|
+
// Deliberately silent (BLI-3238). `realpathSync` is the test for "does
|
|
526
|
+
// this path resolve to something real?", and `null` — "no canonical form"
|
|
527
|
+
// — is the answer, not a degradation. Callers already treat a null here
|
|
528
|
+
// as one more alias that does not apply.
|
|
517
529
|
return null;
|
|
518
530
|
}
|
|
519
531
|
}
|
|
@@ -2,6 +2,7 @@ import { SourceScanResultSchema, } from "@bli-cockpit/telemetry-core";
|
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { makeSourceAdapterIdentity, parseTicketIdFromText, } from "./common.js";
|
|
5
|
+
import { describeError, isMissingFileFailure } from "../health-detail.js";
|
|
5
6
|
export async function collectCarState(context) {
|
|
6
7
|
const ticketsDir = path.join(context.repoRoot, ".codex-autorunner", "tickets");
|
|
7
8
|
try {
|
|
@@ -41,7 +42,17 @@ export async function collectCarState(context) {
|
|
|
41
42
|
facts,
|
|
42
43
|
};
|
|
43
44
|
}
|
|
44
|
-
catch {
|
|
45
|
+
catch (error) {
|
|
46
|
+
// `car_not_present` is honest for a repo with no `car/` folder, which is
|
|
47
|
+
// most of them, so a missing directory stays quiet. It is also what a
|
|
48
|
+
// malformed ticket file collapses to — and that version silently costs the
|
|
49
|
+
// repo its ticket binding while reporting the ordinary answer (BLI-3238).
|
|
50
|
+
if (!isMissingFileFailure(error)) {
|
|
51
|
+
console.error("[car-state] car folder present but unreadable, reporting it as not present", JSON.stringify({
|
|
52
|
+
reason: "car_not_present",
|
|
53
|
+
...describeError(error),
|
|
54
|
+
}));
|
|
55
|
+
}
|
|
45
56
|
const facts = {
|
|
46
57
|
present: false,
|
|
47
58
|
current_ticket: null,
|
|
@@ -5,6 +5,7 @@ import { createReadStream, existsSync } from "node:fs";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { StringDecoder } from "node:string_decoder";
|
|
7
7
|
import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
|
|
8
|
+
import { describeError } from "../health-detail.js";
|
|
8
9
|
import { SESSION_FILE_UUID_PATTERN, isPathWithin, sanitizeSessionId, scoreSignalsAgainstWorktrees, sessionIdFromFileName, shortHash, } from "./attribution-core.js";
|
|
9
10
|
/**
|
|
10
11
|
* Deterministic Claude Code session JSONL -> repo/worktree attribution.
|
|
@@ -89,11 +90,24 @@ async function discoverClaudeSessions(projectsDir, cutoffMs) {
|
|
|
89
90
|
let sessionStatFailedCount = 0;
|
|
90
91
|
let sidecarDirReadFailedCount = 0;
|
|
91
92
|
let sidecarStatFailedCount = 0;
|
|
93
|
+
// Same shape as the Codex walk: first reason plus counts, once. Per-file
|
|
94
|
+
// lines across a store of thousands would be their own silence (BLI-3238).
|
|
95
|
+
let firstDirectoryFailure = null;
|
|
96
|
+
let firstStatFailure = null;
|
|
92
97
|
let projectEntries;
|
|
93
98
|
try {
|
|
94
99
|
projectEntries = await fs.readdir(projectsDir, { withFileTypes: true });
|
|
95
100
|
}
|
|
96
101
|
catch (error) {
|
|
102
|
+
// The whole projects root. Absent means Claude Code has never run here;
|
|
103
|
+
// anything else means every Claude session on this machine is invisible
|
|
104
|
+
// and the scan still reports a clean zero.
|
|
105
|
+
if (!isMissingPathError(error)) {
|
|
106
|
+
console.error("[claude-attribution] Claude projects root unreadable; no sessions can be seen", JSON.stringify({
|
|
107
|
+
reason: "projects_root_unreadable",
|
|
108
|
+
...describeError(error),
|
|
109
|
+
}));
|
|
110
|
+
}
|
|
97
111
|
return {
|
|
98
112
|
sessions,
|
|
99
113
|
projectDirsSkipped,
|
|
@@ -112,8 +126,10 @@ async function discoverClaudeSessions(projectsDir, cutoffMs) {
|
|
|
112
126
|
sessionEntries = await fs.readdir(projectDir, { withFileTypes: true });
|
|
113
127
|
}
|
|
114
128
|
catch (error) {
|
|
115
|
-
if (!isMissingPathError(error))
|
|
129
|
+
if (!isMissingPathError(error)) {
|
|
116
130
|
projectDirReadFailedCount += 1;
|
|
131
|
+
firstDirectoryFailure ??= describeError(error);
|
|
132
|
+
}
|
|
117
133
|
continue;
|
|
118
134
|
}
|
|
119
135
|
for (const sessionEntry of sessionEntries) {
|
|
@@ -126,8 +142,9 @@ async function discoverClaudeSessions(projectsDir, cutoffMs) {
|
|
|
126
142
|
try {
|
|
127
143
|
mainStat = await fs.stat(mainFile);
|
|
128
144
|
}
|
|
129
|
-
catch {
|
|
145
|
+
catch (error) {
|
|
130
146
|
sessionStatFailedCount += 1;
|
|
147
|
+
firstStatFailure ??= describeError(error);
|
|
131
148
|
continue;
|
|
132
149
|
}
|
|
133
150
|
const sessionUuid = sessionEntry.name.replace(/\.jsonl$/i, "");
|
|
@@ -149,6 +166,18 @@ async function discoverClaudeSessions(projectsDir, cutoffMs) {
|
|
|
149
166
|
});
|
|
150
167
|
}
|
|
151
168
|
}
|
|
169
|
+
if (projectDirReadFailedCount > 0 || sessionStatFailedCount > 0) {
|
|
170
|
+
console.error("[claude-attribution] Claude sessions were invisible to the walk", JSON.stringify({
|
|
171
|
+
reason: "session_discovery_incomplete",
|
|
172
|
+
project_dir_read_failed_count: projectDirReadFailedCount,
|
|
173
|
+
session_stat_failed_count: sessionStatFailedCount,
|
|
174
|
+
found_session_count: sessions.length,
|
|
175
|
+
...(firstDirectoryFailure
|
|
176
|
+
? { first_directory_failure: firstDirectoryFailure }
|
|
177
|
+
: {}),
|
|
178
|
+
...(firstStatFailure ? { first_stat_failure: firstStatFailure } : {}),
|
|
179
|
+
}));
|
|
180
|
+
}
|
|
152
181
|
return {
|
|
153
182
|
sessions,
|
|
154
183
|
projectDirsSkipped,
|
|
@@ -164,6 +193,15 @@ async function discoverSidecars(subagentsDir) {
|
|
|
164
193
|
entries = await fs.readdir(subagentsDir, { withFileTypes: true });
|
|
165
194
|
}
|
|
166
195
|
catch (error) {
|
|
196
|
+
// Most sessions have no subagents at all, so absent is quiet. Anything
|
|
197
|
+
// else means the session is collected WITHOUT its subagent transcripts and
|
|
198
|
+
// nothing downstream can tell that from a session that had none.
|
|
199
|
+
if (!isMissingPathError(error)) {
|
|
200
|
+
console.error("[claude-attribution] subagent folder unreadable; sidecars omitted from this session", JSON.stringify({
|
|
201
|
+
reason: "sidecar_dir_unreadable",
|
|
202
|
+
...describeError(error),
|
|
203
|
+
}));
|
|
204
|
+
}
|
|
167
205
|
return {
|
|
168
206
|
sidecars: [],
|
|
169
207
|
sidecarsCapped: 0,
|
|
@@ -173,6 +211,7 @@ async function discoverSidecars(subagentsDir) {
|
|
|
173
211
|
}
|
|
174
212
|
const discovered = [];
|
|
175
213
|
let statFailedCount = 0;
|
|
214
|
+
let firstSidecarStatFailure = null;
|
|
176
215
|
for (const entry of entries) {
|
|
177
216
|
// Only agent transcripts. `*.meta.json` carry operator-authored
|
|
178
217
|
// descriptions and are never harvested (D3).
|
|
@@ -186,8 +225,9 @@ async function discoverSidecars(subagentsDir) {
|
|
|
186
225
|
try {
|
|
187
226
|
stat = await fs.stat(local_path);
|
|
188
227
|
}
|
|
189
|
-
catch {
|
|
228
|
+
catch (error) {
|
|
190
229
|
statFailedCount += 1;
|
|
230
|
+
firstSidecarStatFailure ??= describeError(error);
|
|
191
231
|
continue;
|
|
192
232
|
}
|
|
193
233
|
discovered.push({
|
|
@@ -197,6 +237,14 @@ async function discoverSidecars(subagentsDir) {
|
|
|
197
237
|
byteSize: stat.size,
|
|
198
238
|
});
|
|
199
239
|
}
|
|
240
|
+
if (statFailedCount > 0) {
|
|
241
|
+
console.error("[claude-attribution] subagent transcripts skipped", JSON.stringify({
|
|
242
|
+
reason: "sidecar_stat_failed",
|
|
243
|
+
stat_failed_count: statFailedCount,
|
|
244
|
+
found_sidecar_count: discovered.length,
|
|
245
|
+
...firstSidecarStatFailure,
|
|
246
|
+
}));
|
|
247
|
+
}
|
|
200
248
|
discovered.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
201
249
|
const capped = Math.max(0, discovered.length - CLAUDE_SESSION_MAX_SIDECAR_FILES);
|
|
202
250
|
return {
|
|
@@ -241,9 +289,16 @@ async function attributeOneSession(session, worktrees, collectionRoots) {
|
|
|
241
289
|
try {
|
|
242
290
|
streamed = await streamMainSignals(session.mainFile);
|
|
243
291
|
}
|
|
244
|
-
catch {
|
|
292
|
+
catch (error) {
|
|
245
293
|
// A read race on one oversized main must not abort the whole scan; honor
|
|
246
|
-
// the per-file skip contract.
|
|
294
|
+
// the per-file skip contract. Per session, so worth a line each time:
|
|
295
|
+
// this is a whole session nobody will ever coach on (BLI-3238).
|
|
296
|
+
console.error("[claude-attribution] oversized session could not be streamed, skipping it", JSON.stringify({
|
|
297
|
+
reason: "file_read_failed",
|
|
298
|
+
stage: "stream_oversized_main",
|
|
299
|
+
byte_size: session.mainByteSize,
|
|
300
|
+
...describeError(error),
|
|
301
|
+
}));
|
|
247
302
|
return skippedResult(base, "file_read_failed");
|
|
248
303
|
}
|
|
249
304
|
signals = streamed.signals;
|
|
@@ -257,7 +312,13 @@ async function attributeOneSession(session, worktrees, collectionRoots) {
|
|
|
257
312
|
try {
|
|
258
313
|
raw = await fs.readFile(session.mainFile);
|
|
259
314
|
}
|
|
260
|
-
catch {
|
|
315
|
+
catch (error) {
|
|
316
|
+
console.error("[claude-attribution] session file unreadable, skipping it", JSON.stringify({
|
|
317
|
+
reason: "file_read_failed",
|
|
318
|
+
stage: "read_main",
|
|
319
|
+
byte_size: session.mainByteSize,
|
|
320
|
+
...describeError(error),
|
|
321
|
+
}));
|
|
261
322
|
return skippedResult(base, "file_read_failed");
|
|
262
323
|
}
|
|
263
324
|
const content = raw.toString("utf8");
|
|
@@ -346,7 +407,16 @@ async function collectSidecarDiagnostics(sidecars, worktree) {
|
|
|
346
407
|
try {
|
|
347
408
|
raw = await fs.readFile(sidecar.local_path);
|
|
348
409
|
}
|
|
349
|
-
catch {
|
|
410
|
+
catch (error) {
|
|
411
|
+
// `file_read_failed` stays on the sidecar row. Beside it: a subagent
|
|
412
|
+
// transcript that was stat'd successfully seconds ago and now will not
|
|
413
|
+
// read is a race worth being able to recognise (BLI-3238).
|
|
414
|
+
console.error("[claude-attribution] subagent transcript unreadable, skipping it", JSON.stringify({
|
|
415
|
+
reason: "file_read_failed",
|
|
416
|
+
stage: "read_sidecar",
|
|
417
|
+
byte_size: sidecar.byteSize,
|
|
418
|
+
...describeError(error),
|
|
419
|
+
}));
|
|
350
420
|
out.push({ ...entry, skipped_reason: "file_read_failed" });
|
|
351
421
|
continue;
|
|
352
422
|
}
|
|
@@ -397,6 +467,10 @@ function createSignalAccumulator() {
|
|
|
397
467
|
record = JSON.parse(line);
|
|
398
468
|
}
|
|
399
469
|
catch {
|
|
470
|
+
// Deliberately silent (BLI-3238), same as the Codex line parser: per
|
|
471
|
+
// LINE across hundreds of thousands, the last line of a live session
|
|
472
|
+
// is routinely half-written, the count travels in `parseErrorCount` at
|
|
473
|
+
// the right grain, and the error would carry transcript text.
|
|
400
474
|
parseErrorCount += 1;
|
|
401
475
|
return;
|
|
402
476
|
}
|
|
@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { StringDecoder } from "node:string_decoder";
|
|
6
6
|
import { normalizeGitOrigin } from "../repo-identity.js";
|
|
7
|
+
import { describeError } from "../health-detail.js";
|
|
7
8
|
import { sanitizeSessionId, scoreSignalsAgainstWorktrees, sessionIdFromFileName, shortHash, } from "./attribution-core.js";
|
|
8
9
|
/**
|
|
9
10
|
* Deterministic Codex session JSONL -> repo/worktree attribution.
|
|
@@ -74,6 +75,11 @@ async function discoverCodexJsonlFiles(dir, cutoffMs) {
|
|
|
74
75
|
const secretPathSkippedCount = 0;
|
|
75
76
|
const stack = Array.isArray(dir) ? [...dir] : [dir];
|
|
76
77
|
const seenFiles = new Set();
|
|
78
|
+
// The counts already travel in the scan result; what never did is WHY, and
|
|
79
|
+
// one line per skipped file in a store of thousands would be its own kind of
|
|
80
|
+
// silence. First reason plus count, logged once (BLI-3238).
|
|
81
|
+
let firstDirectoryFailure = null;
|
|
82
|
+
let firstStatFailure = null;
|
|
77
83
|
while (stack.length > 0) {
|
|
78
84
|
const current = stack.pop();
|
|
79
85
|
if (!current)
|
|
@@ -83,8 +89,10 @@ async function discoverCodexJsonlFiles(dir, cutoffMs) {
|
|
|
83
89
|
entries = await fs.readdir(current, { withFileTypes: true });
|
|
84
90
|
}
|
|
85
91
|
catch (error) {
|
|
86
|
-
if (!isMissingPathError(error))
|
|
92
|
+
if (!isMissingPathError(error)) {
|
|
87
93
|
directoryReadFailedCount += 1;
|
|
94
|
+
firstDirectoryFailure ??= describeError(error);
|
|
95
|
+
}
|
|
88
96
|
continue;
|
|
89
97
|
}
|
|
90
98
|
for (const entry of entries) {
|
|
@@ -99,8 +107,9 @@ async function discoverCodexJsonlFiles(dir, cutoffMs) {
|
|
|
99
107
|
try {
|
|
100
108
|
stat = await fs.stat(full);
|
|
101
109
|
}
|
|
102
|
-
catch {
|
|
110
|
+
catch (error) {
|
|
103
111
|
statFailedCount += 1;
|
|
112
|
+
firstStatFailure ??= describeError(error);
|
|
104
113
|
continue;
|
|
105
114
|
}
|
|
106
115
|
if (stat.mtimeMs >= cutoffMs) {
|
|
@@ -112,6 +121,18 @@ async function discoverCodexJsonlFiles(dir, cutoffMs) {
|
|
|
112
121
|
}
|
|
113
122
|
}
|
|
114
123
|
}
|
|
124
|
+
if (directoryReadFailedCount > 0 || statFailedCount > 0) {
|
|
125
|
+
console.error("[codex-attribution] Codex sessions were invisible to the walk", JSON.stringify({
|
|
126
|
+
reason: "session_discovery_incomplete",
|
|
127
|
+
directory_read_failed_count: directoryReadFailedCount,
|
|
128
|
+
stat_failed_count: statFailedCount,
|
|
129
|
+
found_file_count: out.length,
|
|
130
|
+
...(firstDirectoryFailure
|
|
131
|
+
? { first_directory_failure: firstDirectoryFailure }
|
|
132
|
+
: {}),
|
|
133
|
+
...(firstStatFailure ? { first_stat_failure: firstStatFailure } : {}),
|
|
134
|
+
}));
|
|
135
|
+
}
|
|
115
136
|
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
116
137
|
return {
|
|
117
138
|
files: out,
|
|
@@ -158,7 +179,15 @@ async function attributeOneSession(file, worktrees, collectionRoots) {
|
|
|
158
179
|
try {
|
|
159
180
|
read = await readCodexMetadataSignals(file.file);
|
|
160
181
|
}
|
|
161
|
-
catch {
|
|
182
|
+
catch (error) {
|
|
183
|
+
// One skipped session, with `file_read_failed` on its row. Logged per
|
|
184
|
+
// session because it is per SESSION, not per file in a walk: a session
|
|
185
|
+
// that cannot be read is a session nobody will ever coach on (BLI-3238).
|
|
186
|
+
console.error("[codex-attribution] session file unreadable, skipping it", JSON.stringify({
|
|
187
|
+
reason: "file_read_failed",
|
|
188
|
+
byte_size: file.byteSize,
|
|
189
|
+
...describeError(error),
|
|
190
|
+
}));
|
|
162
191
|
return skippedResult(base, "file_read_failed");
|
|
163
192
|
}
|
|
164
193
|
base.content_hash_sha256 = read.contentHashSha256;
|
|
@@ -268,6 +297,11 @@ function absorbCodexSessionLine(state, line) {
|
|
|
268
297
|
record = JSON.parse(line);
|
|
269
298
|
}
|
|
270
299
|
catch {
|
|
300
|
+
// Deliberately silent (BLI-3238). Per LINE, in files with hundreds of
|
|
301
|
+
// thousands of them, and the last line of a live session is routinely
|
|
302
|
+
// half-written — this is expected, not a failure. The count travels in
|
|
303
|
+
// `parseErrorCount` on the scan result, which is the right grain, and the
|
|
304
|
+
// error object would carry a fragment of the transcript.
|
|
271
305
|
state.parseErrorCount += 1;
|
|
272
306
|
return;
|
|
273
307
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
|
+
import { describeError } from "../health-detail.js";
|
|
2
3
|
import { remoteObjectKey, sha256 } from "./raw-evidence-keys.js";
|
|
3
4
|
export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
|
|
4
5
|
export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
|
|
@@ -126,7 +127,17 @@ export function manifestDescribesEntries(manifestBytes, entries) {
|
|
|
126
127
|
return (recorded.length === expected.length &&
|
|
127
128
|
recorded.every((hash, index) => hash === expected[index]));
|
|
128
129
|
}
|
|
129
|
-
catch {
|
|
130
|
+
catch (error) {
|
|
131
|
+
// `false` is the safe answer and is also what a legitimately-stale
|
|
132
|
+
// manifest returns, so the pack is simply rewritten either way. The
|
|
133
|
+
// difference is that an unparseable manifest means a pack was written
|
|
134
|
+
// half-way, which will repeat until someone knows it is happening.
|
|
135
|
+
console.error("[raw-evidence-manifest] manifest unparseable; treating the pack as not described", JSON.stringify({
|
|
136
|
+
reason: "manifest_unparseable",
|
|
137
|
+
byte_size: manifestBytes.byteLength,
|
|
138
|
+
expected_file_count: entries.length,
|
|
139
|
+
...describeError(error),
|
|
140
|
+
}));
|
|
130
141
|
return false;
|
|
131
142
|
}
|
|
132
143
|
}
|
|
@@ -14,6 +14,7 @@ import fs from "node:fs/promises";
|
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
import { makeManifest, manifestDescribesEntries, } from "./raw-evidence-manifest.js";
|
|
16
16
|
import { writeRawEvidenceStagingState, } from "../raw-evidence-staging.js";
|
|
17
|
+
import { describeError } from "../health-detail.js";
|
|
17
18
|
/**
|
|
18
19
|
* Turn the staging directory into the content-keyed pack directory.
|
|
19
20
|
*
|
|
@@ -41,9 +42,20 @@ export async function promoteStagedPack(options) {
|
|
|
41
42
|
refilledFileCount: 0,
|
|
42
43
|
};
|
|
43
44
|
}
|
|
44
|
-
catch {
|
|
45
|
+
catch (error) {
|
|
45
46
|
// A concurrent sync can win the race to the same content-keyed name.
|
|
46
|
-
// Losing it is fine: the winner staged the identical bytes
|
|
47
|
+
// Losing it is fine: the winner staged the identical bytes — and that
|
|
48
|
+
// race is EEXIST/ENOTEMPTY/EPERM. A rename that fails for any other
|
|
49
|
+
// reason (no space, cross-device, read-only) also lands here and looks
|
|
50
|
+
// exactly like the benign race while the pack is never stored
|
|
51
|
+
// (BLI-3238).
|
|
52
|
+
if (!isPackRenameRaceError(error)) {
|
|
53
|
+
console.error("[raw-evidence-pack-store] could not move the staged pack into place", JSON.stringify({
|
|
54
|
+
reason: "pack_rename_failed",
|
|
55
|
+
pack_id: options.packId,
|
|
56
|
+
...describeError(error),
|
|
57
|
+
}));
|
|
58
|
+
}
|
|
47
59
|
}
|
|
48
60
|
}
|
|
49
61
|
const refilledFileCount = await refillMissingPackFiles(evidenceDir, options.entries);
|
|
@@ -120,11 +132,24 @@ export async function persistStagingState(stateDir, staging, nowIso) {
|
|
|
120
132
|
await writeRawEvidenceStagingState(stateDir, staging).catch((error) => {
|
|
121
133
|
console.error("[raw-evidence] staging state write failed", JSON.stringify({
|
|
122
134
|
reason: "staging_state_write_failed",
|
|
123
|
-
detail: error instanceof Error ? error.name : typeof error,
|
|
124
135
|
staged_count: Object.keys(staging.staged).length,
|
|
136
|
+
// Was `error.name` alone, which is "Error" for every fs failure
|
|
137
|
+
// there is. The code is the part that distinguishes them (BLI-3238).
|
|
138
|
+
...describeError(error),
|
|
125
139
|
}));
|
|
126
140
|
});
|
|
127
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* The codes `rename` produces when the destination already exists — which is
|
|
144
|
+
* the benign "another sync staged these identical bytes first" outcome. EPERM
|
|
145
|
+
* is Windows's version of it.
|
|
146
|
+
*/
|
|
147
|
+
function isPackRenameRaceError(error) {
|
|
148
|
+
if (!error || typeof error !== "object")
|
|
149
|
+
return false;
|
|
150
|
+
const code = error.code;
|
|
151
|
+
return code === "EEXIST" || code === "ENOTEMPTY" || code === "EPERM";
|
|
152
|
+
}
|
|
128
153
|
export async function ensurePrivateDir(dir) {
|
|
129
154
|
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
130
155
|
await chmodPrivate(dir, 0o700);
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { containsSecretLikeContent, redactSecretLikeContent, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import { sha256 } from "./raw-evidence-keys.js";
|
|
3
|
+
import { describeError } from "../health-detail.js";
|
|
4
|
+
/** BLI-3238: a crashed redactor has to say so; see the catch below. */
|
|
3
5
|
export function sanitizeTextEvidenceForUpload(options) {
|
|
4
6
|
const originalBytes = options.originalBytes ?? Buffer.from(options.text, "utf8");
|
|
5
7
|
// Deliberately outside the try: the content guard is the cheap check that
|
|
@@ -35,9 +37,25 @@ export function sanitizeTextEvidenceForUpload(options) {
|
|
|
35
37
|
redactionResult,
|
|
36
38
|
});
|
|
37
39
|
}
|
|
38
|
-
return {
|
|
40
|
+
return {
|
|
41
|
+
status: "clean",
|
|
42
|
+
bytes: originalBytes,
|
|
43
|
+
redaction: scannedCleanRedactionMetadata(originalBytes),
|
|
44
|
+
};
|
|
39
45
|
}
|
|
40
|
-
catch {
|
|
46
|
+
catch (error) {
|
|
47
|
+
// The redactor itself crashed. The stub below is the safe answer — a
|
|
48
|
+
// placeholder goes up instead of unredacted bytes — but it is also
|
|
49
|
+
// indistinguishable downstream from a file that was legitimately masked,
|
|
50
|
+
// so a systematic redactor bug would look like a machine that simply
|
|
51
|
+
// writes a lot of secrets. Never the text, never the field values: name,
|
|
52
|
+
// code and size only (BLI-3238).
|
|
53
|
+
console.error("[raw-evidence-sanitize] redaction crashed; uploading a stub instead of the content", JSON.stringify({
|
|
54
|
+
reason: "redaction_crashed",
|
|
55
|
+
byte_size: originalBytes.byteLength,
|
|
56
|
+
redacted_field_count: options.redactedFields.length,
|
|
57
|
+
...describeError(error),
|
|
58
|
+
}));
|
|
41
59
|
return stubForCrashedRedaction({
|
|
42
60
|
text: options.text,
|
|
43
61
|
originalBytes,
|
|
@@ -147,6 +165,32 @@ function lineEndingOf(segment) {
|
|
|
147
165
|
return "\n";
|
|
148
166
|
return "";
|
|
149
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* The receipt for a file the scan cleared (BLI-3277).
|
|
170
|
+
*
|
|
171
|
+
* Same schema, same `mode` — the deterministic ruleset is what ran — with the
|
|
172
|
+
* verdict `scanned_clean` and zero of everything else. The content fields are
|
|
173
|
+
* the point: an evidence ref is only readable downstream when its redaction
|
|
174
|
+
* record hashes the bytes that are actually in the bucket, and for a clean file
|
|
175
|
+
* those are the original bytes, so both halves carry the same digest and size.
|
|
176
|
+
*/
|
|
177
|
+
function scannedCleanRedactionMetadata(originalBytes) {
|
|
178
|
+
const digest = sha256(originalBytes);
|
|
179
|
+
return {
|
|
180
|
+
schema_version: "raw-evidence-redaction.v1",
|
|
181
|
+
status: "scanned_clean",
|
|
182
|
+
mode: "deterministic_text_replacement",
|
|
183
|
+
applied_by: ["local_collector"],
|
|
184
|
+
rule_counts: [],
|
|
185
|
+
secret_like_match_count: 0,
|
|
186
|
+
redacted_fields: [],
|
|
187
|
+
redacted_ranges: [],
|
|
188
|
+
original_content_hash_sha256: digest,
|
|
189
|
+
sanitized_content_hash_sha256: digest,
|
|
190
|
+
original_byte_size: originalBytes.byteLength,
|
|
191
|
+
sanitized_byte_size: originalBytes.byteLength,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
150
194
|
function fallbackRedactionMetadata(options) {
|
|
151
195
|
const fullContentRedacted = options.fullContentRedacted !== false;
|
|
152
196
|
return {
|
|
@@ -7,6 +7,7 @@ import { makeSourceAdapterIdentity, } from "./common.js";
|
|
|
7
7
|
import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
|
|
8
8
|
import { defaultCodexSessionDirs, } from "./codex-attribution.js";
|
|
9
9
|
import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-policy.js";
|
|
10
|
+
import { describeError } from "../health-detail.js";
|
|
10
11
|
import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject, } from "../raw-evidence-staging.js";
|
|
11
12
|
import { isSecretLikePath, safeKeySegment, sha256, shortHash, } from "./raw-evidence-keys.js";
|
|
12
13
|
import { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
|
|
@@ -868,7 +869,10 @@ async function collectOneEvidenceFile(collection, options) {
|
|
|
868
869
|
}));
|
|
869
870
|
}
|
|
870
871
|
const evidenceBytes = sanitized.bytes;
|
|
872
|
+
// Both branches carry a record now (BLI-3277), so "was anything replaced?" is
|
|
873
|
+
// the status, never the presence of `redaction`.
|
|
871
874
|
const redaction = sanitized.redaction;
|
|
875
|
+
const wasRedacted = sanitized.status === "redacted";
|
|
872
876
|
const contentHash = sha256(evidenceBytes);
|
|
873
877
|
if (collection.skipContentHashes.has(contentHash)) {
|
|
874
878
|
collection.reused.push({
|
|
@@ -900,7 +904,7 @@ async function collectOneEvidenceFile(collection, options) {
|
|
|
900
904
|
localPath: staged.local_path,
|
|
901
905
|
relativePath,
|
|
902
906
|
mediaType: options.mediaType,
|
|
903
|
-
redactedSummary:
|
|
907
|
+
redactedSummary: wasRedacted
|
|
904
908
|
? `${options.redactedSummary} Secret-like values were deterministically redacted before upload.`
|
|
905
909
|
: options.redactedSummary,
|
|
906
910
|
redaction,
|
|
@@ -923,7 +927,16 @@ async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
|
|
|
923
927
|
try {
|
|
924
928
|
stat = await fs.stat(filePath);
|
|
925
929
|
}
|
|
926
|
-
catch {
|
|
930
|
+
catch (error) {
|
|
931
|
+
// The caller turns this into the `file_read_failed` skip label, which is
|
|
932
|
+
// the one label an operator can do nothing with. The file was discovered
|
|
933
|
+
// moments ago, so a failure here is a rotated session, a permission
|
|
934
|
+
// problem or a dead symlink — three different answers (BLI-3238).
|
|
935
|
+
console.error("[raw-evidence] evidence file could not be stat'd", JSON.stringify({
|
|
936
|
+
reason: "file_read_failed",
|
|
937
|
+
stage: "stat",
|
|
938
|
+
...describeError(error),
|
|
939
|
+
}));
|
|
927
940
|
return { status: "read_failed" };
|
|
928
941
|
}
|
|
929
942
|
if (stat.size > maxFileBytes)
|
|
@@ -933,7 +946,12 @@ async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
|
|
|
933
946
|
try {
|
|
934
947
|
bytes = await fs.readFile(filePath);
|
|
935
948
|
}
|
|
936
|
-
catch {
|
|
949
|
+
catch (error) {
|
|
950
|
+
console.error("[raw-evidence] evidence file could not be read", JSON.stringify({
|
|
951
|
+
reason: "file_read_failed",
|
|
952
|
+
stage: "read",
|
|
953
|
+
...describeError(error),
|
|
954
|
+
}));
|
|
937
955
|
return { status: "read_failed" };
|
|
938
956
|
}
|
|
939
957
|
if (maxFileBytes && bytes.byteLength > maxFileBytes) {
|
|
@@ -999,7 +1017,16 @@ async function collectGitDiffFiles(collection, repoRoot) {
|
|
|
999
1017
|
try {
|
|
1000
1018
|
diff = await runGitDiff(target.args, repoRoot);
|
|
1001
1019
|
}
|
|
1002
|
-
catch {
|
|
1020
|
+
catch (error) {
|
|
1021
|
+
// `git_diff_failed` is the skip label and stays. It covers git not being
|
|
1022
|
+
// installed, the folder not being a repo, a locked index and a diff that
|
|
1023
|
+
// exceeded the child-process buffer — and the diff is half the evidence
|
|
1024
|
+
// for what someone actually changed, so losing it quietly matters.
|
|
1025
|
+
console.error("[raw-evidence] git diff failed", JSON.stringify({
|
|
1026
|
+
reason: "git_diff_failed",
|
|
1027
|
+
diff_target: target.label,
|
|
1028
|
+
...describeError(error),
|
|
1029
|
+
}));
|
|
1003
1030
|
collection.skipped.push({
|
|
1004
1031
|
kind: "git_diff",
|
|
1005
1032
|
label: target.label,
|
|
@@ -1050,6 +1077,7 @@ async function stageOneGitDiff(collection, target) {
|
|
|
1050
1077
|
}
|
|
1051
1078
|
const raw = sanitized.bytes;
|
|
1052
1079
|
const redaction = sanitized.redaction;
|
|
1080
|
+
const wasRedacted = sanitized.status === "redacted";
|
|
1053
1081
|
const contentHash = sha256(raw);
|
|
1054
1082
|
if (collection.skipContentHashes.has(contentHash)) {
|
|
1055
1083
|
collection.reused.push({
|
|
@@ -1093,7 +1121,7 @@ async function stageOneGitDiff(collection, target) {
|
|
|
1093
1121
|
sourceKey: diffSourceKey,
|
|
1094
1122
|
mediaType: "text/x-diff",
|
|
1095
1123
|
redactedSummary: gitDiffSummary(target.label, {
|
|
1096
|
-
redacted:
|
|
1124
|
+
redacted: wasRedacted,
|
|
1097
1125
|
truncated: target.truncated,
|
|
1098
1126
|
}),
|
|
1099
1127
|
redaction,
|
|
@@ -1124,6 +1152,11 @@ async function walkJsonlFiles(dir, cutoffMs) {
|
|
|
1124
1152
|
const out = [];
|
|
1125
1153
|
const stack = Array.isArray(dir) ? [...dir] : [dir];
|
|
1126
1154
|
const seen = new Set();
|
|
1155
|
+
// Counted rather than logged per directory: a wide walk can hit many, and
|
|
1156
|
+
// the useful signal is "N directories in the session store were skipped and
|
|
1157
|
+
// here is the first reason", not N near-identical lines (BLI-3238).
|
|
1158
|
+
let unreadableDirCount = 0;
|
|
1159
|
+
let firstUnreadableDir = null;
|
|
1127
1160
|
while (stack.length > 0) {
|
|
1128
1161
|
const current = stack.pop();
|
|
1129
1162
|
if (!current || isSecretLikePath(current))
|
|
@@ -1132,7 +1165,11 @@ async function walkJsonlFiles(dir, cutoffMs) {
|
|
|
1132
1165
|
try {
|
|
1133
1166
|
entries = await fs.readdir(current, { withFileTypes: true });
|
|
1134
1167
|
}
|
|
1135
|
-
catch {
|
|
1168
|
+
catch (error) {
|
|
1169
|
+
// A directory that cannot be listed hides every session under it, and
|
|
1170
|
+
// the walk's only visible effect is a smaller file count.
|
|
1171
|
+
unreadableDirCount += 1;
|
|
1172
|
+
firstUnreadableDir ??= describeError(error);
|
|
1136
1173
|
continue;
|
|
1137
1174
|
}
|
|
1138
1175
|
for (const entry of entries) {
|
|
@@ -1155,6 +1192,14 @@ async function walkJsonlFiles(dir, cutoffMs) {
|
|
|
1155
1192
|
out.push({ file: full, mtimeMs: stat.mtimeMs });
|
|
1156
1193
|
}
|
|
1157
1194
|
}
|
|
1195
|
+
if (unreadableDirCount > 0) {
|
|
1196
|
+
console.error("[raw-evidence] session-store directories skipped during the walk", JSON.stringify({
|
|
1197
|
+
reason: "session_dir_unreadable",
|
|
1198
|
+
unreadable_dir_count: unreadableDirCount,
|
|
1199
|
+
found_file_count: out.length,
|
|
1200
|
+
...firstUnreadableDir,
|
|
1201
|
+
}));
|
|
1202
|
+
}
|
|
1158
1203
|
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
1159
1204
|
return out.map((entry) => entry.file);
|
|
1160
1205
|
}
|