@bli-cockpit/cli 0.2.28 → 0.2.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,4 @@
1
- import { EvidenceCompletenessPayloadSchema, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, SECRET_FILE_SEGMENT_PATTERN, SourceScanResultSchema, containsSecretLikeContent, redactSecretLikeContent, } from "@bli-cockpit/telemetry-core";
2
- import { spawn } from "node:child_process";
1
+ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, SourceScanResultSchema, } from "@bli-cockpit/telemetry-core";
3
2
  import crypto from "node:crypto";
4
3
  import fs from "node:fs/promises";
5
4
  import os from "node:os";
@@ -8,13 +7,18 @@ import { makeSourceAdapterIdentity, } from "./common.js";
8
7
  import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
9
8
  import { defaultCodexSessionDirs, } from "./codex-attribution.js";
10
9
  import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-policy.js";
11
- import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject, writeRawEvidenceStagingState, } from "../raw-evidence-staging.js";
10
+ import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject, } from "../raw-evidence-staging.js";
11
+ import { isSecretLikePath, safeKeySegment, sha256, shortHash, } from "./raw-evidence-keys.js";
12
+ import { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
13
+ import { countEvidenceEntries, makeEvidenceCompleteness, markBudgetCapApplied, markCapApplied, recordScanned, recordSkipCount, recordTruncationCount, } from "./raw-evidence-completeness.js";
14
+ import { evidenceEntry, pointerFromEntry, RAW_EVIDENCE_BUCKET, } from "./raw-evidence-manifest.js";
15
+ import { chmodPrivate, ensurePrivateDir, persistStagingState, promoteStagedPack, stageManifest, } from "./raw-evidence-pack-store.js";
16
+ import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, runGitDiff, } from "./raw-evidence-git-diff.js";
17
+ // Re-exported so every consumer keeps importing from `adapters/raw-evidence`.
18
+ export { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./raw-evidence-manifest.js";
19
+ export { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
12
20
  const DEFAULT_SINCE_MINUTES = 24 * 60;
13
21
  const DEFAULT_SESSION_LIMIT = 50;
14
- const MAX_GIT_DIFF_BYTES = 2 * 1024 * 1024;
15
- const GIT_DIFF_TIMEOUT_MS = 3_000;
16
- export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
17
- export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
18
22
  // Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
19
23
  // transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
20
24
  // so 2 GiB leaves room for several files without starving the sync; overflow
@@ -22,6 +26,9 @@ export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
22
26
  export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
23
27
  export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
24
28
  const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;
29
+ // ---------------------------------------------------------------------------
30
+ // The pass
31
+ // ---------------------------------------------------------------------------
25
32
  export async function collectRawEvidencePack(context, options) {
26
33
  const startedAt = context.now.toISOString();
27
34
  const rawEvidenceRoot = path.join(options.stateDir, "raw-evidence");
@@ -30,338 +37,383 @@ export async function collectRawEvidencePack(context, options) {
30
37
  // directory is then renamed to its content-keyed name — or dropped, when an
31
38
  // identical pack is already there.
32
39
  const stagingDir = path.join(rawEvidenceRoot, `.staging-${process.pid}-${crypto.randomUUID().slice(0, 8)}`);
33
- const filesDir = path.join(stagingDir, "files");
34
- const staging = await readRawEvidenceStagingState(options.stateDir);
35
- const entries = [];
36
- const skipped = [];
37
- const truncated = [];
38
- const failed = [];
39
- const redacted = [];
40
- const reused = [];
41
40
  const sinceMinutes = options.sinceMinutes ?? DEFAULT_SINCE_MINUTES;
42
- const sessionLimit = options.sessionLimit ?? DEFAULT_SESSION_LIMIT;
43
- const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
44
- const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
45
- const collection = {
46
- context,
47
- filesDir,
41
+ const collection = await openCollection(context, options, {
48
42
  rawEvidenceRoot,
49
- packId: "",
50
- staging,
51
- heldSources: heldSourceKeys(staging, context.now),
52
- stagedReusedCount: 0,
53
- stagedNewCount: 0,
54
- deliveryHeldCount: 0,
55
- entries,
56
- skipped,
57
- truncated,
58
- failed,
59
- redacted,
60
- reused,
61
- scanned: new Map(),
62
- caps: [
63
- {
64
- source: "raw_evidence",
65
- cap_type: "byte_budget",
66
- limit: byteBudget,
67
- observed: options.budget?.remainingBytes ?? byteBudget,
68
- applied: false,
69
- },
70
- {
71
- source: "raw_evidence",
72
- cap_type: "object_budget",
73
- limit: objectBudget,
74
- observed: options.budget?.remainingObjects ?? objectBudget,
75
- applied: false,
76
- },
77
- {
78
- source: "git_diff",
79
- cap_type: "max_bytes_per_diff",
80
- limit: MAX_GIT_DIFF_BYTES,
81
- applied: false,
82
- },
83
- {
84
- source: "git_diff",
85
- cap_type: "timeout_ms",
86
- limit: GIT_DIFF_TIMEOUT_MS,
87
- applied: false,
88
- },
89
- {
90
- source: "codex_jsonl",
91
- cap_type: "max_file_bytes",
92
- limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
93
- applied: false,
94
- },
95
- {
96
- source: "claude_jsonl",
97
- cap_type: "max_file_bytes",
98
- limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
99
- applied: false,
100
- },
101
- {
102
- source: "claude_jsonl_sidecar",
103
- cap_type: "max_file_bytes",
104
- limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
105
- applied: false,
106
- },
107
- ],
108
- skipContentHashes: options.skipContentHashes ?? new Set(),
109
- budget: options.budget ?? {
110
- remainingBytes: byteBudget,
111
- remainingObjects: objectBudget,
112
- },
113
- index: { value: 0 },
43
+ stagingDir,
44
+ });
45
+ const scanWindow = {
46
+ startedAt,
47
+ finishedAt: () => context.now.toISOString(),
48
+ sinceMinutes,
114
49
  };
115
50
  try {
116
51
  await ensurePrivateDir(stagingDir);
117
- await ensurePrivateDir(filesDir);
118
- recordAttributionCompleteness(collection, {
119
- codex: options.codexAttributionScan,
120
- claude: options.claudeAttributionScan,
121
- selectedCodexPaths: new Set(options.codexSessionFiles?.map((file) => file.local_path) ?? []),
122
- selectedClaudePaths: new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []),
123
- });
52
+ await ensurePrivateDir(collection.filesDir);
53
+ recordAttributionCompleteness(collection, options);
124
54
  if (options.includeCodexJsonl !== false) {
125
55
  await collectCodexJsonlFiles(collection, {
126
56
  codexSessionFiles: options.codexSessionFiles,
127
57
  sessionsDir: options.sessionsDir,
128
58
  sessionsDirs: options.sessionsDirs,
129
59
  sinceMinutes,
130
- limit: sessionLimit,
60
+ limit: options.sessionLimit ?? DEFAULT_SESSION_LIMIT,
131
61
  });
132
62
  }
133
63
  if (options.includeClaudeJsonl !== false && options.claudeSessionFiles) {
134
64
  await collectClaudeJsonlFiles(collection, options.claudeSessionFiles);
135
65
  }
136
66
  await collectGitDiffFiles(collection, options.repoRoot);
137
- const deferredByteBudgetCount = skipped.filter((entry) => entry.reason === "deferred_byte_budget").reduce((sum, entry) => sum + evidenceEntryCount(entry), 0);
138
- const deferredObjectBudgetCount = skipped.filter((entry) => entry.reason === "deferred_object_budget").reduce((sum, entry) => sum + evidenceEntryCount(entry), 0);
139
- if (entries.length === 0) {
140
- // Nothing collected: drop the staging directory instead of leaving an
141
- // empty pack behind. This used to leave one empty `work-*` dir per sync.
142
- await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
143
- const packId = contentKeyedRawEvidencePackId({
144
- workContextId: context.workContextId,
145
- contentHashes: [],
67
+ if (collection.entries.length === 0) {
68
+ return await finishWithEmptyPack(collection, options, {
69
+ stagingDir,
70
+ window: scanWindow,
146
71
  });
147
- collection.packId = packId;
148
- const evidenceDir = path.join(rawEvidenceRoot, packId);
149
- await persistStagingState(options.stateDir, collection);
150
- const evidenceCompleteness = makeEvidenceCompleteness(collection, {
151
- startedAt,
152
- finishedAt: context.now.toISOString(),
153
- sinceMinutes,
154
- });
155
- const facts = {
156
- pack_id: packId,
157
- manifest_path: path.join(evidenceDir, "manifest.json"),
158
- evidence_dir: evidenceDir,
159
- storage_bucket: RAW_EVIDENCE_BUCKET,
160
- file_count: 0,
161
- byte_size: 0,
162
- skipped_count: countEvidenceEntries(skipped),
163
- sanitized_count: redacted.length,
164
- reused_count: reused.length,
165
- staged_reused_count: collection.stagedReusedCount,
166
- staged_new_count: collection.stagedNewCount,
167
- delivery_held_count: collection.deliveryHeldCount,
168
- stage_state: "empty",
169
- deferred_byte_budget_count: deferredByteBudgetCount,
170
- deferred_object_budget_count: deferredObjectBudgetCount,
171
- content_kinds: [],
172
- evidence_completeness: evidenceCompleteness,
173
- pointers: [],
174
- upload_files: [],
175
- reused,
176
- };
177
- return {
178
- facts,
179
- scan: makeRawEvidenceScan({
180
- context,
181
- startedAt,
182
- status: "partial",
183
- facts,
184
- }),
185
- };
186
72
  }
187
- // The pack is named by what is in it, never by when it was made. Identical
188
- // content on the next sync resolves to the identical directory, which is
189
- // the whole fix for BLI-3066.
190
- const packId = contentKeyedRawEvidencePackId({
191
- workContextId: context.workContextId,
192
- contentHashes: entries.map((entry) => entry.content_hash_sha256),
73
+ return await finishWithPromotedPack(collection, options, {
74
+ stagingDir,
75
+ window: scanWindow,
193
76
  });
194
- collection.packId = packId;
195
- const promotion = await promoteStagedPack({
196
- rawEvidenceRoot,
77
+ }
78
+ catch (error) {
79
+ return await finishWithFailedPack(collection, {
197
80
  stagingDir,
198
- packId,
199
- workContextId: context.workContextId,
200
- entries,
81
+ window: scanWindow,
82
+ error,
201
83
  });
202
- const evidenceDir = promotion.evidenceDir;
203
- for (const entry of entries) {
204
- if (!entry.staged_in_pack)
205
- continue;
206
- entry.local_path = path.join(evidenceDir, "files", path.basename(entry.local_path));
207
- recordStagedObject(collection.staging, entry.content_hash_sha256, {
208
- pack_id: packId,
209
- relative_path: `files/${path.basename(entry.local_path)}`,
210
- byte_size: entry.byte_size,
211
- source_key: entry.source_key,
212
- staged_at: context.now.toISOString(),
213
- });
214
- }
215
- const manifestPath = path.join(evidenceDir, "manifest.json");
216
- const manifestBytes = await stageManifest({
217
- context,
84
+ }
85
+ }
86
+ async function openCollection(context, options, places) {
87
+ const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
88
+ const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
89
+ const staging = await readRawEvidenceStagingState(options.stateDir);
90
+ return {
91
+ context,
92
+ filesDir: path.join(places.stagingDir, "files"),
93
+ rawEvidenceRoot: places.rawEvidenceRoot,
94
+ packId: "",
95
+ staging,
96
+ heldSources: collectionHeldSources(staging, context.now, options),
97
+ stagedReusedCount: 0,
98
+ stagedNewCount: 0,
99
+ deliveryHeldCount: 0,
100
+ entries: [],
101
+ skipped: [],
102
+ truncated: [],
103
+ failed: [],
104
+ redacted: [],
105
+ reused: [],
106
+ scanned: new Map(),
107
+ caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
108
+ skipContentHashes: options.skipContentHashes ?? new Set(),
109
+ budget: options.budget ?? {
110
+ remainingBytes: byteBudget,
111
+ remainingObjects: objectBudget,
112
+ },
113
+ index: { value: 0 },
114
+ };
115
+ }
116
+ /**
117
+ * Which sources this pass refuses to even read because their delivery is inside
118
+ * a backoff window — none of them, when a person asked for this pass.
119
+ *
120
+ * The bypass is logged rather than assumed: an operator retry that quietly
121
+ * ignored a hold would be as unreadable as the hold that quietly blocked it.
122
+ */
123
+ function collectionHeldSources(staging, now, options) {
124
+ const held = heldSourceKeys(staging, now);
125
+ if (deliveryBackoffApplies(options.deliveryMode))
126
+ return held;
127
+ if (held.size > 0) {
128
+ console.error("[raw-evidence] delivery backoff bypassed for operator retry", JSON.stringify({
129
+ reason: DELIVERY_BACKOFF_BYPASS_REASON,
130
+ source_count: held.size,
131
+ }));
132
+ }
133
+ return new Set();
134
+ }
135
+ /** Every cap this pass could hit, declared up front and flipped when applied. */
136
+ function startingCaps(options) {
137
+ return [
138
+ {
139
+ source: "raw_evidence",
140
+ cap_type: "byte_budget",
141
+ limit: options.byteBudget,
142
+ observed: options.budget?.remainingBytes ?? options.byteBudget,
143
+ applied: false,
144
+ },
145
+ {
146
+ source: "raw_evidence",
147
+ cap_type: "object_budget",
148
+ limit: options.objectBudget,
149
+ observed: options.budget?.remainingObjects ?? options.objectBudget,
150
+ applied: false,
151
+ },
152
+ {
153
+ source: "git_diff",
154
+ cap_type: "max_bytes_per_diff",
155
+ limit: MAX_GIT_DIFF_BYTES,
156
+ applied: false,
157
+ },
158
+ {
159
+ source: "git_diff",
160
+ cap_type: "timeout_ms",
161
+ limit: GIT_DIFF_TIMEOUT_MS,
162
+ applied: false,
163
+ },
164
+ {
165
+ source: "codex_jsonl",
166
+ cap_type: "max_file_bytes",
167
+ limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
168
+ applied: false,
169
+ },
170
+ {
171
+ source: "claude_jsonl",
172
+ cap_type: "max_file_bytes",
173
+ limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
174
+ applied: false,
175
+ },
176
+ {
177
+ source: "claude_jsonl_sidecar",
178
+ cap_type: "max_file_bytes",
179
+ limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
180
+ applied: false,
181
+ },
182
+ ];
183
+ }
184
+ // ---------------------------------------------------------------------------
185
+ // The three ways a pass ends
186
+ // ---------------------------------------------------------------------------
187
+ /**
188
+ * Nothing collected: drop the staging directory instead of leaving an empty
189
+ * pack behind. This used to leave one empty `work-*` dir per sync.
190
+ */
191
+ async function finishWithEmptyPack(collection, options, run) {
192
+ await discardStagingDir(run.stagingDir);
193
+ collection.packId = contentKeyedRawEvidencePackId({
194
+ workContextId: collection.context.workContextId,
195
+ contentHashes: [],
196
+ });
197
+ await persistStagingState(options.stateDir, collection.staging, collection.context.now.toISOString());
198
+ const facts = makeEmptyPackFacts(collection, run.window);
199
+ return {
200
+ facts,
201
+ scan: makeRawEvidenceScan({
202
+ context: collection.context,
203
+ startedAt: run.window.startedAt,
204
+ status: "partial",
205
+ facts,
206
+ }),
207
+ };
208
+ }
209
+ async function finishWithPromotedPack(collection, options, run) {
210
+ const context = collection.context;
211
+ const entries = collection.entries;
212
+ // The pack is named by what is in it, never by when it was made. Identical
213
+ // content on the next sync resolves to the identical directory, which is
214
+ // the whole fix for BLI-3066.
215
+ const packId = contentKeyedRawEvidencePackId({
216
+ workContextId: context.workContextId,
217
+ contentHashes: entries.map((entry) => entry.content_hash_sha256),
218
+ });
219
+ collection.packId = packId;
220
+ const promotion = await promoteStagedPack({
221
+ rawEvidenceRoot: collection.rawEvidenceRoot,
222
+ stagingDir: run.stagingDir,
223
+ packId,
224
+ workContextId: context.workContextId,
225
+ entries,
226
+ });
227
+ const evidenceDir = promotion.evidenceDir;
228
+ rebaseStagedEntriesOntoPack(collection, evidenceDir);
229
+ const manifestPath = path.join(evidenceDir, "manifest.json");
230
+ const manifestBytes = await stageManifest({
231
+ context,
232
+ packId,
233
+ manifestPath,
234
+ entries,
235
+ skipped: collection.skipped,
236
+ redacted: collection.redacted,
237
+ reused: collection.reused,
238
+ reusePack: promotion.state === "reused",
239
+ });
240
+ entries.push(evidenceEntry({
241
+ context,
242
+ kind: "manifest",
243
+ packId,
244
+ localPath: manifestPath,
245
+ relativePath: "manifest.json",
246
+ mediaType: "application/json",
247
+ redactedSummary: "Local raw evidence pack manifest.",
248
+ bytes: manifestBytes,
249
+ }));
250
+ await persistStagingState(options.stateDir, collection.staging, context.now.toISOString());
251
+ console.error("[raw-evidence] pack staged", JSON.stringify({
252
+ pack_id: packId,
253
+ stage_state: promotion.state,
254
+ reason: stageReasonLabel(promotion.state, promotion.priorPackCount),
255
+ prior_pack_count: promotion.priorPackCount,
256
+ refilled_file_count: promotion.refilledFileCount,
257
+ file_count: entries.length,
258
+ byte_size: totalByteSize(entries),
259
+ staged_new: collection.stagedNewCount,
260
+ staged_reused: collection.stagedReusedCount,
261
+ delivery_held: collection.deliveryHeldCount,
262
+ }));
263
+ const facts = {
264
+ ...packFactsShell(collection, {
218
265
  packId,
219
266
  manifestPath,
220
- entries,
221
- skipped,
222
- redacted,
223
- reused,
224
- reusePack: promotion.state === "reused",
225
- });
226
- const manifestEntry = evidenceEntry({
267
+ evidenceDir,
268
+ window: run.window,
269
+ }),
270
+ stage_state: promotion.state,
271
+ file_count: entries.length,
272
+ byte_size: totalByteSize(entries),
273
+ content_kinds: [...new Set(entries.map((entry) => entry.kind))],
274
+ pointers: entries.map(pointerFromEntry),
275
+ upload_files: entries.map(uploadFileFromEntry),
276
+ };
277
+ return {
278
+ facts,
279
+ scan: makeRawEvidenceScan({
227
280
  context,
228
- kind: "manifest",
229
- packId,
230
- localPath: manifestPath,
231
- relativePath: "manifest.json",
232
- mediaType: "application/json",
233
- redactedSummary: "Local raw evidence pack manifest.",
234
- bytes: manifestBytes,
235
- });
236
- entries.push(manifestEntry);
237
- await persistStagingState(options.stateDir, collection);
238
- console.error("[raw-evidence] pack staged", JSON.stringify({
239
- pack_id: packId,
240
- stage_state: promotion.state,
241
- reason: promotion.state === "reused"
242
- ? "staged_reused"
243
- : promotion.state === "restaged_incomplete"
244
- ? "restaged_incomplete"
245
- : promotion.priorPackCount > 0
246
- ? "restaged_content_changed"
247
- : "staged_new",
248
- prior_pack_count: promotion.priorPackCount,
249
- refilled_file_count: promotion.refilledFileCount,
250
- file_count: entries.length,
251
- byte_size: entries.reduce((sum, entry) => sum + entry.byte_size, 0),
252
- staged_new: collection.stagedNewCount,
253
- staged_reused: collection.stagedReusedCount,
254
- delivery_held: collection.deliveryHeldCount,
255
- }));
256
- const evidenceCompleteness = makeEvidenceCompleteness(collection, {
257
- startedAt,
258
- finishedAt: context.now.toISOString(),
259
- sinceMinutes,
260
- });
261
- const facts = {
262
- pack_id: packId,
263
- manifest_path: manifestPath,
264
- evidence_dir: evidenceDir,
265
- storage_bucket: RAW_EVIDENCE_BUCKET,
266
- staged_reused_count: collection.stagedReusedCount,
267
- staged_new_count: collection.stagedNewCount,
268
- delivery_held_count: collection.deliveryHeldCount,
269
- stage_state: promotion.state,
270
- file_count: entries.length,
271
- byte_size: entries.reduce((sum, entry) => sum + entry.byte_size, 0),
272
- skipped_count: countEvidenceEntries(skipped),
273
- sanitized_count: redacted.length,
274
- reused_count: reused.length,
275
- deferred_byte_budget_count: deferredByteBudgetCount,
276
- deferred_object_budget_count: deferredObjectBudgetCount,
277
- content_kinds: [...new Set(entries.map((entry) => entry.kind))],
278
- evidence_completeness: evidenceCompleteness,
279
- pointers: entries.map(pointerFromEntry),
280
- upload_files: entries.map((entry) => ({
281
- pointer: pointerFromEntry(entry),
282
- local_path: entry.local_path,
283
- kind: entry.kind,
284
- codex_session_id: entry.codex_session_id ?? null,
285
- ...(entry.artifact_metadata
286
- ? { artifact_metadata: entry.artifact_metadata }
287
- : {}),
288
- })),
289
- reused,
290
- };
291
- return {
281
+ startedAt: run.window.startedAt,
282
+ status: entries.length > 1 ? "ok" : "partial",
292
283
  facts,
293
- scan: makeRawEvidenceScan({
294
- context,
295
- startedAt,
296
- status: entries.length > 1 ? "ok" : "partial",
297
- facts,
298
- }),
299
- };
300
- }
301
- catch (error) {
302
- failed.push({
303
- kind: "raw_evidence",
304
- reason: "collection_failed",
305
- });
306
- // Staging is per-attempt scratch: a crashed pass must not leave a partial
307
- // directory behind to be counted, re-hashed or swept later.
308
- await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
309
- const packId = collection.packId ||
284
+ }),
285
+ };
286
+ }
287
+ /**
288
+ * A crashed pass. The staged-object index is deliberately NOT written: this
289
+ * attempt's bookkeeping describes a directory that is about to be deleted.
290
+ */
291
+ async function finishWithFailedPack(collection, run) {
292
+ collection.failed.push({
293
+ kind: "raw_evidence",
294
+ reason: "collection_failed",
295
+ });
296
+ // Staging is per-attempt scratch: a crashed pass must not leave a partial
297
+ // directory behind to be counted, re-hashed or swept later.
298
+ await discardStagingDir(run.stagingDir);
299
+ collection.packId =
300
+ collection.packId ||
310
301
  contentKeyedRawEvidencePackId({
311
- workContextId: context.workContextId,
312
- contentHashes: entries.map((entry) => entry.content_hash_sha256),
302
+ workContextId: collection.context.workContextId,
303
+ contentHashes: collection.entries.map((entry) => entry.content_hash_sha256),
313
304
  });
314
- const evidenceDir = path.join(rawEvidenceRoot, packId);
315
- console.error("[raw-evidence] pack collection failed", JSON.stringify({
316
- pack_id: packId,
317
- reason: "collection_failed",
318
- detail: error instanceof Error ? error.name : typeof error,
319
- collected_file_count: entries.length,
320
- staged_new: collection.stagedNewCount,
321
- staged_reused: collection.stagedReusedCount,
322
- }));
323
- const evidenceCompleteness = makeEvidenceCompleteness(collection, {
324
- startedAt,
325
- finishedAt: context.now.toISOString(),
326
- sinceMinutes,
327
- });
328
- const facts = {
329
- pack_id: packId,
330
- manifest_path: path.join(evidenceDir, "manifest.json"),
331
- evidence_dir: evidenceDir,
332
- storage_bucket: RAW_EVIDENCE_BUCKET,
333
- file_count: 0,
334
- byte_size: 0,
335
- skipped_count: countEvidenceEntries(skipped),
336
- sanitized_count: redacted.length,
337
- reused_count: reused.length,
338
- staged_reused_count: collection.stagedReusedCount,
339
- staged_new_count: collection.stagedNewCount,
340
- delivery_held_count: collection.deliveryHeldCount,
341
- stage_state: "empty",
342
- deferred_byte_budget_count: skipped
343
- .filter((entry) => entry.reason === "deferred_byte_budget")
344
- .reduce((sum, entry) => sum + evidenceEntryCount(entry), 0),
345
- deferred_object_budget_count: skipped
346
- .filter((entry) => entry.reason === "deferred_object_budget")
347
- .reduce((sum, entry) => sum + evidenceEntryCount(entry), 0),
348
- content_kinds: [],
349
- evidence_completeness: evidenceCompleteness,
350
- pointers: [],
351
- upload_files: [],
352
- reused,
353
- };
354
- return {
305
+ console.error("[raw-evidence] pack collection failed", JSON.stringify({
306
+ pack_id: collection.packId,
307
+ reason: "collection_failed",
308
+ detail: run.error instanceof Error ? run.error.name : typeof run.error,
309
+ collected_file_count: collection.entries.length,
310
+ staged_new: collection.stagedNewCount,
311
+ staged_reused: collection.stagedReusedCount,
312
+ }));
313
+ const facts = makeEmptyPackFacts(collection, run.window);
314
+ return {
315
+ facts,
316
+ scan: makeRawEvidenceScan({
317
+ context: collection.context,
318
+ startedAt: run.window.startedAt,
319
+ status: "failed",
355
320
  facts,
356
- scan: makeRawEvidenceScan({
357
- context,
358
- startedAt,
359
- status: "failed",
360
- facts,
361
- }),
362
- };
321
+ }),
322
+ };
323
+ }
324
+ /**
325
+ * A pack with no files of its own: the empty pass and the crashed pass report
326
+ * the same shape, differing only in the scan status and the reason already
327
+ * logged. `evidence_completeness` still carries every gap this pass recorded.
328
+ */
329
+ function makeEmptyPackFacts(collection, scanWindow) {
330
+ const evidenceDir = path.join(collection.rawEvidenceRoot, collection.packId);
331
+ return {
332
+ ...packFactsShell(collection, {
333
+ packId: collection.packId,
334
+ manifestPath: path.join(evidenceDir, "manifest.json"),
335
+ evidenceDir,
336
+ window: scanWindow,
337
+ }),
338
+ stage_state: "empty",
339
+ file_count: 0,
340
+ byte_size: 0,
341
+ content_kinds: [],
342
+ pointers: [],
343
+ upload_files: [],
344
+ };
345
+ }
346
+ /** The counts every ending reports identically, however the pass ended. */
347
+ function packFactsShell(collection, place) {
348
+ return {
349
+ pack_id: place.packId,
350
+ manifest_path: place.manifestPath,
351
+ evidence_dir: place.evidenceDir,
352
+ storage_bucket: RAW_EVIDENCE_BUCKET,
353
+ skipped_count: countEvidenceEntries(collection.skipped),
354
+ sanitized_count: collection.redacted.length,
355
+ reused_count: collection.reused.length,
356
+ staged_reused_count: collection.stagedReusedCount,
357
+ staged_new_count: collection.stagedNewCount,
358
+ delivery_held_count: collection.deliveryHeldCount,
359
+ deferred_byte_budget_count: countDeferred(collection, "deferred_byte_budget"),
360
+ deferred_object_budget_count: countDeferred(collection, "deferred_object_budget"),
361
+ evidence_completeness: makeEvidenceCompleteness(collection, {
362
+ startedAt: place.window.startedAt,
363
+ finishedAt: place.window.finishedAt(),
364
+ sinceMinutes: place.window.sinceMinutes,
365
+ }),
366
+ reused: collection.reused,
367
+ };
368
+ }
369
+ function countDeferred(collection, reason) {
370
+ return countEvidenceEntries(collection.skipped, (entry) => entry.reason === reason);
371
+ }
372
+ function totalByteSize(entries) {
373
+ return entries.reduce((sum, entry) => sum + entry.byte_size, 0);
374
+ }
375
+ function uploadFileFromEntry(entry) {
376
+ return {
377
+ pointer: pointerFromEntry(entry),
378
+ local_path: entry.local_path,
379
+ kind: entry.kind,
380
+ codex_session_id: entry.codex_session_id ?? null,
381
+ ...(entry.artifact_metadata
382
+ ? { artifact_metadata: entry.artifact_metadata }
383
+ : {}),
384
+ };
385
+ }
386
+ function stageReasonLabel(state, priorPackCount) {
387
+ if (state === "reused")
388
+ return "staged_reused";
389
+ if (state === "restaged_incomplete")
390
+ return "restaged_incomplete";
391
+ return priorPackCount > 0 ? "restaged_content_changed" : "staged_new";
392
+ }
393
+ /**
394
+ * Point every entry this pass staged at its home in the promoted pack, and
395
+ * remember the content hash so the next sync can adopt the copy instead of
396
+ * writing it again. Entries adopted from another pack keep their path.
397
+ */
398
+ function rebaseStagedEntriesOntoPack(collection, evidenceDir) {
399
+ for (const entry of collection.entries) {
400
+ if (!entry.staged_in_pack)
401
+ continue;
402
+ entry.local_path = path.join(evidenceDir, "files", path.basename(entry.local_path));
403
+ recordStagedObject(collection.staging, entry.content_hash_sha256, {
404
+ pack_id: collection.packId,
405
+ relative_path: `files/${path.basename(entry.local_path)}`,
406
+ byte_size: entry.byte_size,
407
+ source_key: entry.source_key,
408
+ staged_at: collection.context.now.toISOString(),
409
+ });
363
410
  }
364
411
  }
412
+ async function discardStagingDir(stagingDir) {
413
+ await fs
414
+ .rm(stagingDir, { recursive: true, force: true })
415
+ .catch(() => undefined);
416
+ }
365
417
  function makeRawEvidenceScan(options) {
366
418
  return SourceScanResultSchema.parse({
367
419
  adapter: makeSourceAdapterIdentity("collector_runtime", "raw-evidence-pack"),
@@ -402,12 +454,20 @@ function makeRawEvidenceScan(options) {
402
454
  ],
403
455
  });
404
456
  }
405
- function recordAttributionCompleteness(collection, scans) {
406
- if (scans.codex) {
407
- recordCodexAttributionCompleteness(collection, scans.codex, scans.selectedCodexPaths);
457
+ // ---------------------------------------------------------------------------
458
+ // What attribution already knew
459
+ // ---------------------------------------------------------------------------
460
+ /**
461
+ * Attribution ran before collection and already knows which sessions it could
462
+ * not name. Those are gaps in this pass even though no byte was read, so they
463
+ * are recorded first — a session Cockpit never looked at must not be invisible.
464
+ */
465
+ function recordAttributionCompleteness(collection, options) {
466
+ if (options.codexAttributionScan) {
467
+ recordCodexAttributionCompleteness(collection, options.codexAttributionScan, new Set(options.codexSessionFiles?.map((file) => file.local_path) ?? []));
408
468
  }
409
- if (scans.claude) {
410
- recordClaudeAttributionCompleteness(collection, scans.claude, scans.selectedClaudePaths);
469
+ if (options.claudeAttributionScan) {
470
+ recordClaudeAttributionCompleteness(collection, options.claudeAttributionScan, new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []));
411
471
  }
412
472
  }
413
473
  function recordCodexAttributionCompleteness(collection, scan, selectedPaths) {
@@ -485,39 +545,46 @@ function recordClaudeAttributionCompleteness(collection, scan, selectedPaths) {
485
545
  }
486
546
  function recordAttributionResultSkips(collection, source, results, selectedPaths) {
487
547
  for (const result of results) {
488
- // A selected session is being collected by this pass. A live-sync-safe
489
- // synthetic target is also not a skip when another workspace pack owns it.
490
- if (result.state === "attributed" ||
491
- selectedPaths.has(result.file_path) ||
492
- (result.worktree !== null &&
493
- isLiveRawEvidenceSyncAttribution(result.state, result.reason, true))) {
548
+ if (isAttributionAccountedFor(result, selectedPaths))
494
549
  continue;
495
- }
496
550
  const reason = result.state === "skipped"
497
551
  ? result.reason
498
552
  : `attribution_${result.state}:${result.reason}`;
499
553
  recordSkipCount(collection, source, reason, 1);
500
554
  }
501
555
  }
556
+ /**
557
+ * Not every unattributed result is a gap. A selected session is being collected
558
+ * by this pass, and a live-sync-safe synthetic target is owned by another
559
+ * workspace's pack — neither is missing evidence.
560
+ */
561
+ function isAttributionAccountedFor(result, selectedPaths) {
562
+ if (result.state === "attributed")
563
+ return true;
564
+ if (selectedPaths.has(result.file_path))
565
+ return true;
566
+ return (result.worktree !== null &&
567
+ isLiveRawEvidenceSyncAttribution(result.state, result.reason, true));
568
+ }
569
+ // ---------------------------------------------------------------------------
570
+ // Codex transcripts
571
+ // ---------------------------------------------------------------------------
502
572
  async function collectCodexJsonlFiles(collection, options) {
573
+ const attributed = Boolean(options.codexSessionFiles);
503
574
  const resolvedCandidates = options.codexSessionFiles
504
575
  ? options.codexSessionFiles.map((file) => ({
505
576
  filePath: file.local_path,
506
577
  codexSessionId: file.codex_session_id,
507
578
  }))
508
- : (await walkJsonlFiles(options.sessionsDirs ??
509
- (options.sessionsDir
510
- ? [options.sessionsDir]
511
- : defaultCodexSessionDirs(os.homedir())), collection.context.now.getTime() - options.sinceMinutes * 60 * 1000))
512
- .map((filePath) => ({ filePath, codexSessionId: null }));
579
+ : (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
513
580
  collection.caps.push({
514
581
  source: "codex_jsonl",
515
582
  cap_type: "session_limit",
516
583
  limit: options.limit,
517
584
  observed: resolvedCandidates.length,
518
- applied: !options.codexSessionFiles && resolvedCandidates.length > options.limit,
585
+ applied: !attributed && resolvedCandidates.length > options.limit,
519
586
  });
520
- const candidates = options.codexSessionFiles
587
+ const candidates = attributed
521
588
  ? resolvedCandidates
522
589
  : resolvedCandidates.slice(0, options.limit);
523
590
  for (const candidate of candidates) {
@@ -534,95 +601,118 @@ async function collectCodexJsonlFiles(collection, options) {
534
601
  redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
535
602
  contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
536
603
  });
537
- if (transcriptAccepted) {
538
- await collectAgentImagesFromTranscript(collection, {
539
- filePath: candidate.filePath,
540
- source: "codex",
541
- sessionId: codexSessionId,
542
- kind: "codex_image_attachment",
543
- contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
544
- });
545
- }
604
+ if (!transcriptAccepted)
605
+ continue;
606
+ await collectAgentImagesFromTranscript(collection, {
607
+ filePath: candidate.filePath,
608
+ source: "codex",
609
+ sessionId: codexSessionId,
610
+ kind: "codex_image_attachment",
611
+ contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
612
+ });
546
613
  }
547
614
  }
615
+ /** The unattributed fallback: every recent Codex transcript on this machine. */
616
+ function walkRecentCodexJsonlFiles(collection, options) {
617
+ const dirs = options.sessionsDirs ??
618
+ (options.sessionsDir
619
+ ? [options.sessionsDir]
620
+ : defaultCodexSessionDirs(os.homedir()));
621
+ return walkJsonlFiles(dirs, collection.context.now.getTime() - options.sinceMinutes * 60 * 1000);
622
+ }
623
+ // ---------------------------------------------------------------------------
624
+ // Claude transcripts
625
+ // ---------------------------------------------------------------------------
548
626
  async function collectClaudeJsonlFiles(collection, sessions) {
549
627
  recordScanned(collection, "claude_jsonl", sessions.reduce((count, session) => count + 1 + session.sidecar_files.length, 0));
550
628
  for (const session of sessions) {
551
- const sessionId = session.claude_session_id;
552
- if (session.main_file_oversized) {
553
- // D7: the oversized main was attributed via a streamed read but its bytes
554
- // are never uploaded (server commit assembles in memory). Its sidecars
555
- // still collect below.
556
- collection.skipped.push({
557
- kind: "claude_jsonl",
558
- label: path.basename(session.local_path),
559
- reason: "file_too_large",
560
- });
561
- }
562
- else if (session.skip_main) {
563
- // D9 damped: prior durable copy is still good enough; collect sidecars
564
- // only. The session reports reused_existing from cursor state, so no
565
- // skipped entry is recorded here.
566
- }
567
- else {
568
- const mainAccepted = await collectOneEvidenceFile(collection, {
569
- filePath: session.local_path,
570
- kind: "claude_jsonl",
571
- sessionId,
572
- mediaType: "application/jsonl",
573
- // Re-check size at collection: a main that grew past the cap between
574
- // attribution and collection is an honest file_too_large skip, not an
575
- // upload_failed at the chunk client.
576
- maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
577
- redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
578
- contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/${hash16}.jsonl`,
579
- });
580
- if (mainAccepted) {
581
- await collectAgentImagesFromTranscript(collection, {
582
- filePath: session.local_path,
583
- source: "claude_code",
584
- sessionId,
585
- kind: "claude_image_attachment",
586
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
587
- });
588
- }
589
- }
590
- if (session.skip_main && !session.main_file_oversized) {
591
- await collectAgentImagesFromTranscript(collection, {
592
- filePath: session.local_path,
593
- source: "claude_code",
594
- sessionId,
595
- kind: "claude_image_attachment",
596
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
597
- });
598
- }
599
- for (const sidecar of session.sidecar_files) {
600
- const stem = path.basename(sidecar.local_path).replace(/\.jsonl$/i, "");
601
- const safeSidecarId = isSecretLikePath(stem)
602
- ? "redacted-file-name"
603
- : safeKeySegment(stem);
604
- const sidecarAccepted = await collectOneEvidenceFile(collection, {
605
- filePath: sidecar.local_path,
606
- kind: "claude_jsonl_sidecar",
607
- sessionId,
608
- mediaType: "application/jsonl",
609
- maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
610
- redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
611
- contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
612
- });
613
- if (sidecarAccepted) {
614
- await collectAgentImagesFromTranscript(collection, {
615
- filePath: sidecar.local_path,
616
- source: "claude_code",
617
- sessionId,
618
- sidecarId: safeSidecarId,
619
- kind: "claude_image_attachment",
620
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
621
- });
622
- }
623
- }
629
+ await collectOneClaudeSession(collection, session);
624
630
  }
625
631
  }
632
+ async function collectOneClaudeSession(collection, session) {
633
+ const sessionId = session.claude_session_id;
634
+ const mainOutcome = await collectClaudeMainFile(collection, session);
635
+ if (mainOutcome === "collected" || mainOutcome === "damped_reuse") {
636
+ await collectAgentImagesFromTranscript(collection, {
637
+ filePath: session.local_path,
638
+ source: "claude_code",
639
+ sessionId,
640
+ kind: "claude_image_attachment",
641
+ contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
642
+ });
643
+ }
644
+ for (const sidecar of session.sidecar_files) {
645
+ await collectOneClaudeSidecar(collection, session, sidecar.local_path);
646
+ }
647
+ }
648
+ /**
649
+ * What happens to a Claude main file, as a decision table:
650
+ *
651
+ * | condition | main bytes | outcome |
652
+ * | --------------------- | ------------------------- | ------------------- |
653
+ * | `main_file_oversized` | skipped `file_too_large` | `skipped_too_large` |
654
+ * | `skip_main` (D9) | not re-collected | `damped_reuse` |
655
+ * | otherwise | collected, or named skip | `collected` / `not_collected` |
656
+ *
657
+ * Its images are collected for every outcome except `skipped_too_large` and
658
+ * `not_collected` — the caller decides that, this function only reports.
659
+ *
660
+ * D7: an oversized main was attributed via a streamed read but its bytes are
661
+ * never uploaded (server commit assembles in memory). Its sidecars still
662
+ * collect. D9 damped: the prior durable copy is still good enough, and the
663
+ * session reports `reused_existing` from cursor state, so no skip is recorded.
664
+ */
665
+ async function collectClaudeMainFile(collection, session) {
666
+ if (session.main_file_oversized) {
667
+ collection.skipped.push({
668
+ kind: "claude_jsonl",
669
+ label: path.basename(session.local_path),
670
+ reason: "file_too_large",
671
+ });
672
+ return "skipped_too_large";
673
+ }
674
+ if (session.skip_main)
675
+ return "damped_reuse";
676
+ const accepted = await collectOneEvidenceFile(collection, {
677
+ filePath: session.local_path,
678
+ kind: "claude_jsonl",
679
+ sessionId: session.claude_session_id,
680
+ mediaType: "application/jsonl",
681
+ // Re-check size at collection: a main that grew past the cap between
682
+ // attribution and collection is an honest file_too_large skip, not an
683
+ // upload_failed at the chunk client.
684
+ maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
685
+ redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
686
+ contentAddress: (hash16) => `claude/${safeKeySegment(session.claude_session_id)}/${hash16}.jsonl`,
687
+ });
688
+ return accepted ? "collected" : "not_collected";
689
+ }
690
+ async function collectOneClaudeSidecar(collection, session, sidecarPath) {
691
+ const sessionId = session.claude_session_id;
692
+ const stem = path.basename(sidecarPath).replace(/\.jsonl$/i, "");
693
+ const safeSidecarId = isSecretLikePath(stem)
694
+ ? "redacted-file-name"
695
+ : safeKeySegment(stem);
696
+ const sidecarAccepted = await collectOneEvidenceFile(collection, {
697
+ filePath: sidecarPath,
698
+ kind: "claude_jsonl_sidecar",
699
+ sessionId,
700
+ mediaType: "application/jsonl",
701
+ maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
702
+ redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
703
+ contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
704
+ });
705
+ if (!sidecarAccepted)
706
+ return;
707
+ await collectAgentImagesFromTranscript(collection, {
708
+ filePath: sidecarPath,
709
+ source: "claude_code",
710
+ sessionId,
711
+ sidecarId: safeSidecarId,
712
+ kind: "claude_image_attachment",
713
+ contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
714
+ });
715
+ }
626
716
  async function collectAgentImagesFromTranscript(collection, options) {
627
717
  const result = await collectAgentImageEvidenceFromJsonlFile({
628
718
  filePath: options.filePath,
@@ -705,35 +795,43 @@ async function collectOneAgentImageFile(collection, options) {
705
795
  artifactMetadata: metadata,
706
796
  }));
707
797
  }
798
+ // ---------------------------------------------------------------------------
799
+ // One transcript, end to end
800
+ // ---------------------------------------------------------------------------
708
801
  /**
709
802
  * Reads, secret-guards, content-addresses, budget-checks, and copies one
710
- * attributed transcript into the pack. Attribution only reads metadata records;
711
- * this collection layer and the server commit layer are the two content guards
712
- * that decide whether transcript bytes can become durable evidence.
803
+ * attributed transcript into the pack.
804
+ *
805
+ * Returns whether this file's *content* is accounted for, which is not the same
806
+ * as "was staged": a transcript whose bytes are already durable remotely
807
+ * returns true without staging anything, and its images are still worth
808
+ * collecting. Every false return has pushed a named skip first.
713
809
  */
714
810
  async function collectOneEvidenceFile(collection, options) {
715
811
  const fileName = path.basename(options.filePath);
716
812
  const secretLikeFileName = isSecretLikePath(fileName);
717
- const evidenceLabel = secretLikeFileName
718
- ? "[REDACTED_FILE_NAME]"
719
- : fileName;
813
+ const evidenceLabel = secretLikeFileName ? "[REDACTED_FILE_NAME]" : fileName;
720
814
  const packedFileName = secretLikeFileName ? "redacted-file.jsonl" : fileName;
721
815
  const sourceKey = evidenceSourceKey({
722
816
  kind: options.kind,
723
817
  sessionId: options.sessionId,
724
818
  sourcePath: options.filePath,
725
819
  });
820
+ const skip = (reason) => {
821
+ collection.skipped.push({
822
+ kind: options.kind,
823
+ label: evidenceLabel,
824
+ reason,
825
+ });
826
+ return false;
827
+ };
726
828
  // Delivery backoff is checked BEFORE the file is read. An object whose commit
727
829
  // has failed repeatedly costs nothing at all this cycle — no read, no hash,
728
830
  // no copy, no request — and the hold is a named, retryable gap so a held
729
831
  // session cannot make the sync look clean (BLI-3066).
730
832
  if (collection.heldSources.has(sourceKey)) {
731
833
  collection.deliveryHeldCount += 1;
732
- collection.skipped.push({
733
- kind: options.kind,
734
- label: evidenceLabel,
735
- reason: DELIVERY_BACKOFF_HOLDING_REASON,
736
- });
834
+ skip(DELIVERY_BACKOFF_HOLDING_REASON);
737
835
  console.error("[raw-evidence] delivery backoff holding source", JSON.stringify({
738
836
  reason: DELIVERY_BACKOFF_HOLDING_REASON,
739
837
  kind: options.kind,
@@ -741,50 +839,14 @@ async function collectOneEvidenceFile(collection, options) {
741
839
  }));
742
840
  return false;
743
841
  }
744
- if (options.maxFileBytes) {
745
- let stat;
746
- try {
747
- stat = await fs.stat(options.filePath);
748
- }
749
- catch {
750
- collection.skipped.push({
751
- kind: options.kind,
752
- label: evidenceLabel,
753
- reason: "file_read_failed",
754
- });
755
- return false;
756
- }
757
- if (stat.size > options.maxFileBytes) {
758
- markCapApplied(collection, options.kind, "max_file_bytes");
759
- collection.skipped.push({
760
- kind: options.kind,
761
- label: evidenceLabel,
762
- reason: "file_too_large",
763
- });
764
- return false;
765
- }
766
- }
767
- let raw;
768
- try {
769
- raw = await fs.readFile(options.filePath);
770
- }
771
- catch {
772
- collection.skipped.push({
773
- kind: options.kind,
774
- label: evidenceLabel,
775
- reason: "file_read_failed",
776
- });
777
- return false;
778
- }
779
- if (options.maxFileBytes && raw.byteLength > options.maxFileBytes) {
842
+ const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
843
+ if (read.status === "read_failed")
844
+ return skip("file_read_failed");
845
+ if (read.status === "too_large") {
780
846
  markCapApplied(collection, options.kind, "max_file_bytes");
781
- collection.skipped.push({
782
- kind: options.kind,
783
- label: evidenceLabel,
784
- reason: "file_too_large",
785
- });
786
- return false;
847
+ return skip("file_too_large");
787
848
  }
849
+ const raw = read.bytes;
788
850
  const sanitized = sanitizeTextEvidenceForUpload({
789
851
  text: raw.toString("utf8"),
790
852
  originalBytes: raw,
@@ -820,12 +882,7 @@ async function collectOneEvidenceFile(collection, options) {
820
882
  const deferReason = admitToBudget(collection.budget, evidenceBytes.byteLength);
821
883
  if (deferReason) {
822
884
  markBudgetCapApplied(collection, deferReason);
823
- collection.skipped.push({
824
- kind: options.kind,
825
- label: evidenceLabel,
826
- reason: deferReason,
827
- });
828
- return false;
885
+ return skip(deferReason);
829
886
  }
830
887
  collection.index.value += 1;
831
888
  const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(options.filePath)}-${packedFileName}`);
@@ -855,6 +912,38 @@ async function collectOneEvidenceFile(collection, options) {
855
912
  }));
856
913
  return true;
857
914
  }
915
+ /**
916
+ * Size is checked twice on purpose: once by `stat` so an oversized transcript
917
+ * is never buffered at all, and once on the bytes actually read, because a live
918
+ * session can grow between the two.
919
+ */
920
+ async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
921
+ if (maxFileBytes) {
922
+ let stat;
923
+ try {
924
+ stat = await fs.stat(filePath);
925
+ }
926
+ catch {
927
+ return { status: "read_failed" };
928
+ }
929
+ if (stat.size > maxFileBytes)
930
+ return { status: "too_large" };
931
+ }
932
+ let bytes;
933
+ try {
934
+ bytes = await fs.readFile(filePath);
935
+ }
936
+ catch {
937
+ return { status: "read_failed" };
938
+ }
939
+ if (maxFileBytes && bytes.byteLength > maxFileBytes) {
940
+ return { status: "too_large" };
941
+ }
942
+ return { status: "ok", bytes };
943
+ }
944
+ // ---------------------------------------------------------------------------
945
+ // Staging and budgets
946
+ // ---------------------------------------------------------------------------
858
947
  /**
859
948
  * Put these bytes on disk once.
860
949
  *
@@ -896,145 +985,15 @@ function admitToBudget(budget, byteLength) {
896
985
  budget.remainingBytes -= byteLength;
897
986
  return null;
898
987
  }
899
- export function sanitizeTextEvidenceForUpload(options) {
900
- const originalBytes = options.originalBytes ?? Buffer.from(options.text, "utf8");
901
- const secretLikeContent = containsSecretLikeContent(options.text);
902
- try {
903
- const redactionResult = (options.redact ?? redactSecretLikeContent)(options.text, {
904
- appliedBy: "local_collector",
905
- redactedFields: options.redactedFields,
906
- });
907
- if (options.secretLikeFileName) {
908
- const sanitizedText = redactionResult.redacted
909
- ? redactionResult.text
910
- : options.text;
911
- const safeText = containsSecretLikeContent(sanitizedText)
912
- ? "[REDACTED_LINE:secret_redaction_failed]\n"
913
- : sanitizedText;
914
- const sanitizedBytes = Buffer.from(safeText, "utf8");
915
- return {
916
- status: "redacted",
917
- bytes: sanitizedBytes,
918
- redaction: withRedactionContentMetadata(redactionResult.metadata ??
919
- fallbackRedactionMetadata({
920
- ruleId: "secret_like_file_name",
921
- originalText: options.text,
922
- redactedFields: options.redactedFields,
923
- fullContentRedacted: false,
924
- }), { originalBytes, sanitizedBytes }),
925
- completenessLabel: "secret_like_name_masked",
926
- };
927
- }
928
- if (secretLikeContent) {
929
- const redaction = redactionResult.metadata ??
930
- fallbackRedactionMetadata({
931
- ruleId: "secret_like_content_guard",
932
- originalText: options.text,
933
- redactedFields: options.redactedFields,
934
- });
935
- let maskedText = maskSecretBearingLines(options.text, redaction);
936
- if (containsSecretLikeContent(maskedText)) {
937
- maskedText = "[REDACTED_LINE:secret_redaction_failed]\n";
938
- }
939
- const sanitizedBytes = Buffer.from(maskedText, "utf8");
940
- return {
941
- status: "redacted",
942
- bytes: sanitizedBytes,
943
- redaction: withRedactionContentMetadata(redaction, {
944
- originalBytes,
945
- sanitizedBytes,
946
- }),
947
- completenessLabel: "secret_content_masked",
948
- };
949
- }
950
- if (redactionResult.redacted) {
951
- const sanitizedBytes = Buffer.from(redactionResult.text, "utf8");
952
- return {
953
- status: "redacted",
954
- bytes: sanitizedBytes,
955
- redaction: withRedactionContentMetadata(redactionResult.metadata ??
956
- fallbackRedactionMetadata({
957
- ruleId: "secret_redaction_failed",
958
- originalText: options.text,
959
- redactedFields: options.redactedFields,
960
- }), { originalBytes, sanitizedBytes }),
961
- completenessLabel: "secret_content_masked",
962
- };
963
- }
964
- return { status: "clean", bytes: originalBytes };
965
- }
966
- catch {
967
- const sanitizedBytes = Buffer.from("[REDACTED_LINE:redaction_crashed_stubbed]\n", "utf8");
968
- return {
969
- status: "redacted",
970
- bytes: sanitizedBytes,
971
- redaction: withRedactionContentMetadata(fallbackRedactionMetadata({
972
- ruleId: "redaction_crashed_stubbed",
973
- originalText: options.text,
974
- redactedFields: options.redactedFields,
975
- }), { originalBytes, sanitizedBytes }),
976
- completenessLabel: "redaction_crashed_stubbed",
977
- };
978
- }
979
- }
980
- function maskSecretBearingLines(text, redaction) {
981
- if (redaction.redacted_ranges.length === 0) {
982
- return "[REDACTED_LINE:secret_like_content_guard]\n";
983
- }
984
- const segments = text.match(/[^\n]*(?:\n|$)/gu)?.filter(Boolean) ?? [];
985
- let offset = 0;
986
- return segments
987
- .map((segment) => {
988
- const start = offset;
989
- const end = offset + segment.length;
990
- offset = end;
991
- const matched = redaction.redacted_ranges.find((range) => range.start < end && start < range.end);
992
- if (!matched)
993
- return segment;
994
- const lineEnding = segment.endsWith("\r\n")
995
- ? "\r\n"
996
- : segment.endsWith("\n")
997
- ? "\n"
998
- : "";
999
- return `[REDACTED_LINE:${matched.rule_id}]${lineEnding}`;
1000
- })
1001
- .join("");
1002
- }
1003
- function fallbackRedactionMetadata(options) {
1004
- const fullContentRedacted = options.fullContentRedacted !== false;
1005
- return {
1006
- schema_version: "raw-evidence-redaction.v1",
1007
- status: "sanitized",
1008
- mode: "deterministic_text_replacement",
1009
- applied_by: ["local_collector"],
1010
- rule_counts: [
1011
- {
1012
- rule_id: options.ruleId,
1013
- match_count: 1,
1014
- redacted_char_count: fullContentRedacted
1015
- ? options.originalText.length
1016
- : 0,
1017
- },
1018
- ],
1019
- secret_like_match_count: 1,
1020
- redacted_fields: options.redactedFields,
1021
- redacted_ranges: fullContentRedacted && options.originalText.length > 0
1022
- ? [
1023
- {
1024
- start: 0,
1025
- end: options.originalText.length,
1026
- rule_id: options.ruleId,
1027
- },
1028
- ]
1029
- : [],
1030
- };
1031
- }
988
+ // ---------------------------------------------------------------------------
989
+ // Git diffs
990
+ // ---------------------------------------------------------------------------
991
+ const GIT_DIFF_TARGETS = [
992
+ { label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
993
+ { label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
994
+ ];
1032
995
  async function collectGitDiffFiles(collection, repoRoot) {
1033
- const diffTargets = [
1034
- { label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
1035
- { label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
1036
- ];
1037
- for (const target of diffTargets) {
996
+ for (const target of GIT_DIFF_TARGETS) {
1038
997
  recordScanned(collection, "git_diff");
1039
998
  let diff;
1040
999
  try {
@@ -1060,345 +1019,107 @@ async function collectGitDiffFiles(collection, repoRoot) {
1060
1019
  included_bytes: Buffer.byteLength(diff.stdout, "utf8"),
1061
1020
  });
1062
1021
  }
1022
+ // An empty diff is not a gap: there was simply nothing to record. The
1023
+ // truncation marker above still stands even when zero bytes survived.
1063
1024
  if (!diff.stdout.trim())
1064
1025
  continue;
1065
- const sanitized = sanitizeTextEvidenceForUpload({
1066
- text: diff.stdout,
1067
- redactedFields: [`git_diff.${target.label}`],
1068
- });
1069
- if (sanitized.status === "redacted") {
1070
- collection.redacted.push({
1071
- kind: "git_diff",
1072
- label: target.label,
1073
- redaction: sanitized.redaction,
1074
- completenessLabel: sanitized.completenessLabel,
1075
- });
1076
- console.error("[raw-evidence] git diff sanitized", JSON.stringify({
1077
- mode: sanitized.completenessLabel,
1078
- original_bytes: Buffer.byteLength(diff.stdout, "utf8"),
1079
- uploaded_bytes: sanitized.bytes.byteLength,
1080
- }));
1081
- }
1082
- const raw = sanitized.bytes;
1083
- const redaction = sanitized.redaction;
1084
- const contentHash = sha256(raw);
1085
- if (collection.skipContentHashes.has(contentHash)) {
1086
- collection.reused.push({
1087
- kind: "git_diff",
1088
- label: target.label,
1089
- content_hash_sha256: contentHash,
1090
- codex_session_id: null,
1091
- });
1092
- continue;
1093
- }
1094
- const deferReason = admitToBudget(collection.budget, raw.byteLength);
1095
- if (deferReason) {
1096
- markBudgetCapApplied(collection, deferReason);
1097
- collection.skipped.push({
1098
- kind: "git_diff",
1099
- label: target.label,
1100
- reason: deferReason,
1101
- });
1102
- continue;
1103
- }
1104
- const relativePath = path.join("files", `git-${target.label}.diff`);
1105
- const diffSourceKey = evidenceSourceKey({
1106
- kind: "git_diff",
1107
- sessionId: collection.context.workContextId,
1026
+ await stageOneGitDiff(collection, {
1108
1027
  label: target.label,
1028
+ diffText: diff.stdout,
1029
+ truncated: diff.truncated,
1109
1030
  });
1110
- const staged = await stageEvidenceBytes(collection, {
1111
- contentHash,
1112
- bytes: raw,
1113
- fileName: path.basename(relativePath),
1031
+ }
1032
+ }
1033
+ async function stageOneGitDiff(collection, target) {
1034
+ const sanitized = sanitizeTextEvidenceForUpload({
1035
+ text: target.diffText,
1036
+ redactedFields: [`git_diff.${target.label}`],
1037
+ });
1038
+ if (sanitized.status === "redacted") {
1039
+ collection.redacted.push({
1114
1040
  kind: "git_diff",
1115
- sourceKey: diffSourceKey,
1041
+ label: target.label,
1042
+ redaction: sanitized.redaction,
1043
+ completenessLabel: sanitized.completenessLabel,
1116
1044
  });
1117
- collection.entries.push(evidenceEntry({
1118
- context: collection.context,
1119
- kind: "git_diff",
1120
- packId: collection.packId,
1121
- localPath: staged.local_path,
1122
- relativePath,
1123
- stagedInPack: staged.staged_in_pack,
1124
- sourceKey: diffSourceKey,
1125
- mediaType: "text/x-diff",
1126
- redactedSummary: redaction
1127
- ? `Raw git ${target.label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`
1128
- : diff.truncated
1129
- ? `Raw git ${target.label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`
1130
- : `Raw git ${target.label} diff preserved locally with env/secret paths excluded.`,
1131
- redaction,
1132
- bytes: raw,
1133
- contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
1045
+ console.error("[raw-evidence] git diff sanitized", JSON.stringify({
1046
+ mode: sanitized.completenessLabel,
1047
+ original_bytes: Buffer.byteLength(target.diffText, "utf8"),
1048
+ uploaded_bytes: sanitized.bytes.byteLength,
1134
1049
  }));
1135
1050
  }
1136
- }
1137
- async function runGitDiff(args, repoRoot) {
1138
- const pathspec = [
1139
- ".",
1140
- ":(exclude).env",
1141
- ":(exclude).env.*",
1142
- ":(exclude)**/.env",
1143
- ":(exclude)**/.env.*",
1144
- ":(exclude)**/*secret*",
1145
- ":(exclude)**/*credential*",
1146
- ":(exclude)**/*private-key*",
1147
- ":(exclude)**/*.pem",
1148
- ":(exclude)**/*.key",
1149
- ];
1150
- return new Promise((resolve, reject) => {
1151
- const child = spawn("git", [...args, ...pathspec], {
1152
- cwd: repoRoot,
1153
- stdio: ["ignore", "pipe", "pipe"],
1154
- });
1155
- const stdoutChunks = [];
1156
- const stderrChunks = [];
1157
- let observedBytes = 0;
1158
- let includedBytes = 0;
1159
- let truncated = false;
1160
- let timedOut = false;
1161
- const timeout = setTimeout(() => {
1162
- timedOut = true;
1163
- truncated = true;
1164
- child.kill("SIGTERM");
1165
- }, GIT_DIFF_TIMEOUT_MS);
1166
- child.stdout.on("data", (chunk) => {
1167
- observedBytes += chunk.byteLength;
1168
- if (includedBytes < MAX_GIT_DIFF_BYTES) {
1169
- const remaining = MAX_GIT_DIFF_BYTES - includedBytes;
1170
- const next = chunk.subarray(0, remaining);
1171
- stdoutChunks.push(next);
1172
- includedBytes += next.byteLength;
1173
- }
1174
- if (observedBytes > MAX_GIT_DIFF_BYTES) {
1175
- truncated = true;
1176
- child.kill("SIGTERM");
1177
- }
1178
- });
1179
- child.stderr.on("data", (chunk) => {
1180
- if (stderrChunks.reduce((sum, item) => sum + item.byteLength, 0) < 4096) {
1181
- stderrChunks.push(chunk.subarray(0, 4096));
1182
- }
1183
- });
1184
- child.on("error", (error) => {
1185
- clearTimeout(timeout);
1186
- reject(error);
1051
+ const raw = sanitized.bytes;
1052
+ const redaction = sanitized.redaction;
1053
+ const contentHash = sha256(raw);
1054
+ if (collection.skipContentHashes.has(contentHash)) {
1055
+ collection.reused.push({
1056
+ kind: "git_diff",
1057
+ label: target.label,
1058
+ content_hash_sha256: contentHash,
1059
+ codex_session_id: null,
1187
1060
  });
1188
- child.on("close", (code, signal) => {
1189
- clearTimeout(timeout);
1190
- if (code === 0 || truncated || signal === "SIGTERM") {
1191
- resolve({
1192
- stdout: Buffer.concat(stdoutChunks).toString("utf8"),
1193
- truncated,
1194
- observedBytes,
1195
- truncationReason: timedOut
1196
- ? "git_diff_timeout"
1197
- : "max_git_diff_bytes",
1198
- truncationCapType: timedOut ? "timeout_ms" : "max_bytes_per_diff",
1199
- });
1200
- return;
1201
- }
1202
- const stderr = Buffer.concat(stderrChunks).toString("utf8").trim();
1203
- reject(new Error(stderr || `git diff failed with code ${code ?? signal}`));
1061
+ return;
1062
+ }
1063
+ const deferReason = admitToBudget(collection.budget, raw.byteLength);
1064
+ if (deferReason) {
1065
+ markBudgetCapApplied(collection, deferReason);
1066
+ collection.skipped.push({
1067
+ kind: "git_diff",
1068
+ label: target.label,
1069
+ reason: deferReason,
1204
1070
  });
1205
- });
1206
- }
1207
- function recordScanned(collection, source, count = 1) {
1208
- collection.scanned.set(source, (collection.scanned.get(source) ?? 0) + count);
1209
- }
1210
- function markBudgetCapApplied(collection, reason) {
1211
- markCapApplied(collection, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
1212
- }
1213
- function markCapApplied(collection, source, capType) {
1214
- const cap = collection.caps.find((candidate) => candidate.source === source && candidate.cap_type === capType);
1215
- if (cap)
1216
- cap.applied = true;
1217
- }
1218
- function recordSkipCount(collection, source, reason, count) {
1219
- if (count <= 0)
1220
1071
  return;
1221
- collection.skipped.push({
1222
- kind: source,
1223
- label: reason,
1224
- reason,
1225
- count,
1072
+ }
1073
+ const relativePath = path.join("files", `git-${target.label}.diff`);
1074
+ const diffSourceKey = evidenceSourceKey({
1075
+ kind: "git_diff",
1076
+ sessionId: collection.context.workContextId,
1077
+ label: target.label,
1226
1078
  });
1227
- }
1228
- function recordTruncationCount(collection, source, reason, count, details = {}) {
1229
- if (count <= 0)
1230
- return;
1231
- collection.truncated.push({
1232
- kind: source,
1233
- reason,
1234
- count,
1235
- ...details,
1079
+ const staged = await stageEvidenceBytes(collection, {
1080
+ contentHash,
1081
+ bytes: raw,
1082
+ fileName: path.basename(relativePath),
1083
+ kind: "git_diff",
1084
+ sourceKey: diffSourceKey,
1236
1085
  });
1086
+ collection.entries.push(evidenceEntry({
1087
+ context: collection.context,
1088
+ kind: "git_diff",
1089
+ packId: collection.packId,
1090
+ localPath: staged.local_path,
1091
+ relativePath,
1092
+ stagedInPack: staged.staged_in_pack,
1093
+ sourceKey: diffSourceKey,
1094
+ mediaType: "text/x-diff",
1095
+ redactedSummary: gitDiffSummary(target.label, {
1096
+ redacted: Boolean(redaction),
1097
+ truncated: target.truncated,
1098
+ }),
1099
+ redaction,
1100
+ bytes: raw,
1101
+ contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
1102
+ }));
1237
1103
  }
1238
- function evidenceEntryCount(entry) {
1239
- return entry.count ?? 1;
1240
- }
1241
- function countEvidenceEntries(entries, predicate = () => true) {
1242
- return entries.reduce((sum, entry) => sum + (predicate(entry) ? evidenceEntryCount(entry) : 0), 0);
1243
- }
1244
- function makeEvidenceCompleteness(collection, options) {
1245
- const sources = new Set(collection.scanned.keys());
1246
- for (const entry of collection.entries)
1247
- sources.add(entry.kind);
1248
- for (const entry of collection.skipped)
1249
- sources.add(entry.kind);
1250
- for (const entry of collection.reused)
1251
- sources.add(entry.kind);
1252
- for (const entry of collection.truncated)
1253
- sources.add(entry.kind);
1254
- for (const entry of collection.failed)
1255
- sources.add(entry.kind);
1256
- for (const entry of collection.redacted)
1257
- sources.add(entry.kind);
1258
- const sourceCounts = [...sources].sort().map((source) => {
1259
- const skipped = collection.skipped.filter((entry) => entry.kind === source);
1260
- return {
1261
- source,
1262
- scanned_count: collection.scanned.get(source) ?? 0,
1263
- included_count: collection.entries.filter((entry) => entry.kind === source).length,
1264
- skipped_count: countEvidenceEntries(skipped),
1265
- truncated_count: countEvidenceEntries(collection.truncated, (entry) => entry.kind === source),
1266
- deferred_count: countEvidenceEntries(skipped, (entry) => entry.reason.startsWith("deferred_")),
1267
- reused_count: collection.reused.filter((entry) => entry.kind === source).length,
1268
- failed_count: countEvidenceEntries(collection.failed, (entry) => entry.kind === source),
1269
- };
1270
- });
1271
- const totals = sourceCounts.reduce((sum, count) => ({
1272
- scanned_count: sum.scanned_count + count.scanned_count,
1273
- included_count: sum.included_count + count.included_count,
1274
- skipped_count: sum.skipped_count + count.skipped_count,
1275
- truncated_count: sum.truncated_count + count.truncated_count,
1276
- deferred_count: sum.deferred_count + count.deferred_count,
1277
- reused_count: sum.reused_count + count.reused_count,
1278
- failed_count: sum.failed_count + count.failed_count,
1279
- }), {
1280
- scanned_count: 0,
1281
- included_count: 0,
1282
- skipped_count: 0,
1283
- truncated_count: 0,
1284
- deferred_count: 0,
1285
- reused_count: 0,
1286
- failed_count: 0,
1287
- });
1288
- const skipReasonCounts = new Map();
1289
- for (const skipped of collection.skipped) {
1290
- const key = `${skipped.kind}:${skipped.reason}`;
1291
- const existing = skipReasonCounts.get(key);
1292
- if (existing) {
1293
- existing.count += evidenceEntryCount(skipped);
1294
- }
1295
- else {
1296
- skipReasonCounts.set(key, {
1297
- source: skipped.kind,
1298
- reason: skipped.reason,
1299
- count: evidenceEntryCount(skipped),
1300
- });
1301
- }
1104
+ function gitDiffSummary(label, state) {
1105
+ if (state.redacted) {
1106
+ return `Raw git ${label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`;
1302
1107
  }
1303
- const failureReasonCounts = new Map();
1304
- for (const failed of collection.failed) {
1305
- const key = `${failed.kind}:${failed.reason}`;
1306
- const existing = failureReasonCounts.get(key);
1307
- if (existing) {
1308
- existing.count += evidenceEntryCount(failed);
1309
- }
1310
- else {
1311
- failureReasonCounts.set(key, {
1312
- source: failed.kind,
1313
- reason: failed.reason,
1314
- count: evidenceEntryCount(failed),
1315
- });
1316
- }
1108
+ if (state.truncated) {
1109
+ return `Raw git ${label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`;
1317
1110
  }
1318
- const truncationCounts = new Map();
1319
- for (const truncated of collection.truncated) {
1320
- const key = `${truncated.kind}:${truncated.reason}`;
1321
- const existing = truncationCounts.get(key);
1322
- if (existing) {
1323
- existing.count += evidenceEntryCount(truncated);
1324
- existing.observed_bytes = Math.max(existing.observed_bytes ?? 0, truncated.observed_bytes ?? 0);
1325
- existing.included_bytes = Math.max(existing.included_bytes ?? 0, truncated.included_bytes ?? 0);
1326
- }
1327
- else {
1328
- truncationCounts.set(key, {
1329
- source: truncated.kind,
1330
- reason: truncated.reason,
1331
- count: evidenceEntryCount(truncated),
1332
- ...(truncated.max_bytes !== undefined
1333
- ? { max_bytes: truncated.max_bytes }
1334
- : {}),
1335
- ...(truncated.observed_bytes !== undefined
1336
- ? { observed_bytes: truncated.observed_bytes }
1337
- : {}),
1338
- ...(truncated.included_bytes !== undefined
1339
- ? { included_bytes: truncated.included_bytes }
1340
- : {}),
1341
- });
1342
- }
1343
- }
1344
- const redactionCounts = new Map();
1345
- for (const redacted of collection.redacted) {
1346
- const ruleIds = redacted.redaction.rule_counts.map((rule) => rule.rule_id);
1347
- const key = `${redacted.kind}:${redacted.completenessLabel}:${ruleIds.sort().join(",")}`;
1348
- const existing = redactionCounts.get(key);
1349
- if (existing) {
1350
- existing.count += 1;
1351
- existing.rule_ids = [...new Set([...existing.rule_ids, ...ruleIds])].sort();
1352
- }
1353
- else {
1354
- redactionCounts.set(key, {
1355
- source: redacted.kind,
1356
- status: "sanitized",
1357
- mode: redacted.completenessLabel,
1358
- count: 1,
1359
- rule_ids: [...new Set(ruleIds)].sort(),
1360
- });
1361
- }
1362
- }
1363
- const hasGaps = totals.skipped_count > 0 ||
1364
- totals.truncated_count > 0 ||
1365
- totals.deferred_count > 0 ||
1366
- totals.failed_count > 0 ||
1367
- collection.redacted.length > 0 ||
1368
- collection.caps.some((cap) => cap.applied);
1369
- const status = totals.failed_count > 0 &&
1370
- totals.included_count + totals.reused_count === 0
1371
- ? "failed"
1372
- : totals.included_count + totals.reused_count === 0 && !hasGaps
1373
- ? "empty"
1374
- : hasGaps
1375
- ? "partial"
1376
- : "complete";
1377
- return EvidenceCompletenessPayloadSchema.parse({
1378
- schema_version: "evidence-completeness.v1",
1379
- status,
1380
- generated_at: options.finishedAt,
1381
- scan_window: {
1382
- started_at: options.startedAt,
1383
- finished_at: options.finishedAt,
1384
- since_minutes: options.sinceMinutes,
1385
- },
1386
- source_counts: sourceCounts,
1387
- totals,
1388
- caps: collection.caps,
1389
- skip_reasons: [...skipReasonCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
1390
- failure_reasons: [...failureReasonCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
1391
- truncation_markers: [...truncationCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
1392
- redaction_markers: [...redactionCounts.values()].sort((a, b) => `${a.source}:${a.mode}`.localeCompare(`${b.source}:${b.mode}`)),
1393
- notes: totals.failed_count > 0
1394
- ? ["Evidence collection failed; downstream analysis should not infer confidence."]
1395
- : collection.redacted.length > 0
1396
- ? ["Evidence was sanitized before upload; downstream analysis should not treat it as raw-complete."]
1397
- : hasGaps
1398
- ? ["Evidence is incomplete; downstream analysis should lower confidence."]
1399
- : [],
1400
- });
1111
+ return `Raw git ${label} diff preserved locally with env/secret paths excluded.`;
1401
1112
  }
1113
+ // ---------------------------------------------------------------------------
1114
+ // Walking the session store
1115
+ // ---------------------------------------------------------------------------
1116
+ /**
1117
+ * Newest-first `.jsonl` files under these directories, modified since `cutoffMs`.
1118
+ *
1119
+ * Secret-like directory and file names are never descended into or opened;
1120
+ * symlinked duplicates are collapsed by real path so one transcript reachable
1121
+ * two ways is collected once.
1122
+ */
1402
1123
  async function walkJsonlFiles(dir, cutoffMs) {
1403
1124
  const out = [];
1404
1125
  const stack = Array.isArray(dir) ? [...dir] : [dir];
@@ -1425,337 +1146,15 @@ async function walkJsonlFiles(dir, cutoffMs) {
1425
1146
  if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
1426
1147
  continue;
1427
1148
  const stat = await fs.stat(full);
1428
- if (stat.mtimeMs >= cutoffMs) {
1429
- const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
1430
- if (!seen.has(dedupeKey)) {
1431
- seen.add(dedupeKey);
1432
- out.push({ file: full, mtimeMs: stat.mtimeMs });
1433
- }
1434
- }
1149
+ if (stat.mtimeMs < cutoffMs)
1150
+ continue;
1151
+ const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
1152
+ if (seen.has(dedupeKey))
1153
+ continue;
1154
+ seen.add(dedupeKey);
1155
+ out.push({ file: full, mtimeMs: stat.mtimeMs });
1435
1156
  }
1436
1157
  }
1437
1158
  out.sort((a, b) => b.mtimeMs - a.mtimeMs);
1438
1159
  return out.map((entry) => entry.file);
1439
- }
1440
- function makeManifest(options) {
1441
- return {
1442
- schema: "bli.local_raw_evidence_pack.v1",
1443
- pack_id: options.packId,
1444
- created_at: options.context.now.toISOString(),
1445
- work_context_id: options.context.workContextId,
1446
- session_id: options.context.sessionId,
1447
- operator_id: options.context.operatorId,
1448
- operator_label: options.context.operatorLabel,
1449
- repo_label: options.context.repoLabel ?? path.basename(options.context.repoRoot),
1450
- worktree_label: options.context.worktreeLabel,
1451
- active_ticket_id: options.context.activeTicketId ?? null,
1452
- repo_basename: path.basename(options.context.repoRoot),
1453
- branch: options.context.branch,
1454
- storage_bucket: RAW_EVIDENCE_BUCKET,
1455
- raw_policy: {
1456
- raw_prompts: "preserved_private_durable_remote",
1457
- raw_responses: "preserved_private_durable_remote",
1458
- transcripts: "preserved_private_durable_remote",
1459
- claude_transcripts: "preserved_private_durable_remote",
1460
- tool_payloads: "preserved_private_durable_remote",
1461
- git_diffs: "preserved_private_durable_remote_env_secret_paths_excluded",
1462
- agent_image_attachments: "preserved_private_durable_remote_explicit_agent_session_attachment_only",
1463
- env_files: "never_read",
1464
- stdout: "manifest_only_no_raw_content",
1465
- },
1466
- files: options.entries.map(redactManifestEntry),
1467
- skipped: options.skipped,
1468
- redacted: options.redacted,
1469
- reused: options.reused,
1470
- };
1471
- }
1472
- function evidenceEntry(options) {
1473
- const digest = sha256(options.bytes);
1474
- const objectKey = remoteObjectKey({
1475
- context: options.context,
1476
- packId: options.packId,
1477
- relativePath: options.relativePath,
1478
- contentAddress: options.contentAddress,
1479
- });
1480
- return {
1481
- kind: options.kind,
1482
- local_path: options.localPath,
1483
- relative_path: options.relativePath,
1484
- object_key: objectKey,
1485
- content_hash_sha256: digest,
1486
- byte_size: options.bytes.byteLength,
1487
- media_type: options.mediaType,
1488
- redacted_summary: options.redactedSummary,
1489
- ...(options.redaction ? { redaction: options.redaction } : {}),
1490
- codex_session_id: options.codexSessionId ?? null,
1491
- staged_in_pack: options.stagedInPack !== false,
1492
- source_key: options.sourceKey ?? null,
1493
- ...(options.artifactMetadata
1494
- ? {
1495
- artifact_metadata: {
1496
- ...options.artifactMetadata,
1497
- raw_evidence_pointer_id: objectKey,
1498
- storage_bucket: RAW_EVIDENCE_BUCKET,
1499
- object_key: objectKey,
1500
- content_hash_sha256: digest,
1501
- byte_size: options.bytes.byteLength,
1502
- },
1503
- }
1504
- : {}),
1505
- };
1506
- }
1507
- // `local_path` is redacted out, and so are the staging bookkeeping fields:
1508
- // where a copy happens to live on this disk is not part of the pack's identity,
1509
- // and putting it in the manifest would make byte-identical content produce
1510
- // different manifests.
1511
- function redactManifestEntry(entry) {
1512
- return {
1513
- kind: entry.kind,
1514
- relative_path: entry.relative_path,
1515
- object_key: entry.object_key,
1516
- content_hash_sha256: entry.content_hash_sha256,
1517
- byte_size: entry.byte_size,
1518
- media_type: entry.media_type,
1519
- redacted_summary: entry.redacted_summary,
1520
- ...(entry.redaction ? { redaction: entry.redaction } : {}),
1521
- ...(entry.artifact_metadata
1522
- ? { artifact_metadata: entry.artifact_metadata }
1523
- : {}),
1524
- };
1525
- }
1526
- function pointerFromEntry(entry) {
1527
- return {
1528
- raw_evidence_pointer_id: entry.object_key,
1529
- privacy_classification: "remote_durable_raw_evidence",
1530
- retention_policy: {
1531
- mode: RAW_EVIDENCE_RETENTION_MODE,
1532
- privacy_classification: "remote_durable_raw_evidence",
1533
- },
1534
- storage_scope: "remote_object",
1535
- storage_bucket: RAW_EVIDENCE_BUCKET,
1536
- object_key: entry.object_key,
1537
- content_hash_sha256: entry.content_hash_sha256,
1538
- byte_size: entry.byte_size,
1539
- media_type: entry.media_type,
1540
- redacted_summary: entry.redacted_summary,
1541
- ...(entry.redaction ? { redaction: entry.redaction } : {}),
1542
- };
1543
- }
1544
- /**
1545
- * Raw evidence keys start with human-readable context, then end in immutable
1546
- * content addresses or pack-relative manifest paths. The local cursor reuses
1547
- * prior content hashes across syncs; the readable date/session folders are for
1548
- * operator debugging and incident response.
1549
- */
1550
- function remoteObjectKey(options) {
1551
- const namespace = readableEvidenceNamespace(options.context);
1552
- if (options.contentAddress) {
1553
- return posixPath([
1554
- ...namespace,
1555
- options.contentAddress,
1556
- ]);
1557
- }
1558
- return posixPath([
1559
- ...namespace,
1560
- options.packId,
1561
- options.relativePath,
1562
- ]);
1563
- }
1564
- function readableEvidenceNamespace(context) {
1565
- return [
1566
- "operators",
1567
- operatorSlug(context),
1568
- "repos",
1569
- readableKeySegment(context.repoLabel ?? path.basename(context.repoRoot), "repo"),
1570
- "worktrees",
1571
- readableKeySegment(context.worktreeLabel ?? path.basename(context.repoRoot), "worktree"),
1572
- "tickets",
1573
- readableKeySegment(context.activeTicketId ?? "unbound", "unbound", {
1574
- lowercase: false,
1575
- }),
1576
- "dates",
1577
- context.now.toISOString().slice(0, 10),
1578
- "sessions",
1579
- readableKeySegment(context.sessionId, "session"),
1580
- "ids",
1581
- safeKeySegment(context.operatorId),
1582
- safeKeySegment(context.workContextId),
1583
- ];
1584
- }
1585
- function operatorSlug(context) {
1586
- const labelBeforeDomain = (context.operatorLabel ?? context.operatorId)
1587
- .split("@", 1)[0]
1588
- .trim();
1589
- const readable = readableKeySegment(labelBeforeDomain, "operator");
1590
- return `${readable}-${shortHash(context.operatorId).slice(0, 6)}`;
1591
- }
1592
- function readableKeySegment(value, fallback, options = {}) {
1593
- const base = options.lowercase === false ? value : value.toLowerCase();
1594
- const slug = base
1595
- .trim()
1596
- .replace(/[^A-Za-z0-9._-]+/g, "-")
1597
- .replace(/^-+|-+$/g, "")
1598
- .replace(/-{2,}/g, "-")
1599
- .slice(0, 80);
1600
- return slug || fallback;
1601
- }
1602
- function posixPath(parts) {
1603
- return parts.join("/").replace(/\\/g, "/").replace(/\/+/g, "/");
1604
- }
1605
- /**
1606
- * Turn the staging directory into the content-keyed pack directory.
1607
- *
1608
- * Three outcomes, all of them named:
1609
- * - the pack does not exist yet: rename staging into place (`new`);
1610
- * - it exists and holds every file this pass staged: adopt it and delete the
1611
- * staging copy (`reused`) — nothing is copied twice;
1612
- * - it exists but is missing files: fill the gaps from staging
1613
- * (`restaged_incomplete`), because a half-written pack must not be trusted.
1614
- *
1615
- * Never deletes an existing pack directory wholesale: another pack's entries
1616
- * can point into it through the staged-object index.
1617
- */
1618
- async function promoteStagedPack(options) {
1619
- const evidenceDir = path.join(options.rawEvidenceRoot, options.packId);
1620
- const priorPackCount = await countPriorPacks(options.rawEvidenceRoot, options.workContextId, options.packId);
1621
- const existing = await fs.stat(evidenceDir).catch(() => null);
1622
- if (!existing?.isDirectory()) {
1623
- try {
1624
- await fs.rename(options.stagingDir, evidenceDir);
1625
- return {
1626
- state: "new",
1627
- evidenceDir,
1628
- priorPackCount,
1629
- refilledFileCount: 0,
1630
- };
1631
- }
1632
- catch {
1633
- // A concurrent sync can win the race to the same content-keyed name.
1634
- // Losing it is fine: the winner staged the identical bytes.
1635
- }
1636
- }
1637
- let refilledFileCount = 0;
1638
- await ensurePrivateDir(path.join(evidenceDir, "files"));
1639
- for (const entry of options.entries) {
1640
- if (!entry.staged_in_pack)
1641
- continue;
1642
- const fileName = path.basename(entry.local_path);
1643
- const target = path.join(evidenceDir, "files", fileName);
1644
- const info = await fs.stat(target).catch(() => null);
1645
- if (info?.isFile() && info.size === entry.byte_size)
1646
- continue;
1647
- await fs.copyFile(entry.local_path, target).catch(() => undefined);
1648
- await chmodPrivate(target, 0o600);
1649
- refilledFileCount += 1;
1650
- }
1651
- await fs.rm(options.stagingDir, { recursive: true, force: true }).catch(() => undefined);
1652
- return {
1653
- state: refilledFileCount > 0 ? "restaged_incomplete" : "reused",
1654
- evidenceDir,
1655
- priorPackCount,
1656
- refilledFileCount,
1657
- };
1658
- }
1659
- async function countPriorPacks(rawEvidenceRoot, workContextId, packId) {
1660
- const entries = await fs
1661
- .readdir(rawEvidenceRoot, { withFileTypes: true })
1662
- .catch(() => []);
1663
- return entries.filter((entry) => entry.isDirectory() &&
1664
- entry.name !== packId &&
1665
- entry.name.startsWith(`${workContextId}-`)).length;
1666
- }
1667
- /**
1668
- * Write the manifest, or keep the one already in a reused pack.
1669
- *
1670
- * Byte-stability matters here: the manifest's object key is the only key that
1671
- * embeds the pack id, so rewriting it with a fresh `created_at` every sync
1672
- * would push different bytes at the same content-addressed key forever. A
1673
- * reused pack whose manifest already describes exactly these files keeps it.
1674
- */
1675
- async function stageManifest(options) {
1676
- if (options.reusePack) {
1677
- const existing = await fs.readFile(options.manifestPath).catch(() => null);
1678
- if (existing && manifestDescribesEntries(existing, options.entries)) {
1679
- return existing;
1680
- }
1681
- }
1682
- const manifestBytes = Buffer.from(`${JSON.stringify(makeManifest({
1683
- context: options.context,
1684
- packId: options.packId,
1685
- entries: options.entries,
1686
- skipped: options.skipped,
1687
- redacted: options.redacted,
1688
- reused: options.reused,
1689
- }), null, 2)}\n`, "utf8");
1690
- await fs.writeFile(options.manifestPath, manifestBytes, { mode: 0o600 });
1691
- await chmodPrivate(options.manifestPath, 0o600);
1692
- return manifestBytes;
1693
- }
1694
- function manifestDescribesEntries(manifestBytes, entries) {
1695
- try {
1696
- const parsed = JSON.parse(manifestBytes.toString("utf8"));
1697
- const recorded = (parsed.files ?? [])
1698
- .map((file) => typeof file.content_hash_sha256 === "string"
1699
- ? file.content_hash_sha256
1700
- : "")
1701
- .filter(Boolean)
1702
- .sort();
1703
- const expected = entries
1704
- .map((entry) => entry.content_hash_sha256)
1705
- .sort();
1706
- return (recorded.length === expected.length &&
1707
- recorded.every((hash, index) => hash === expected[index]));
1708
- }
1709
- catch {
1710
- return false;
1711
- }
1712
- }
1713
- async function persistStagingState(stateDir, collection) {
1714
- collection.staging.updated_at = collection.context.now.toISOString();
1715
- await writeRawEvidenceStagingState(stateDir, collection.staging).catch((error) => {
1716
- // Losing this file costs reuse and attempt counts, not evidence, so it
1717
- // must never fail a collection — but it must never be silent either.
1718
- console.error("[raw-evidence] staging state write failed", JSON.stringify({
1719
- reason: "staging_state_write_failed",
1720
- detail: error instanceof Error ? error.name : typeof error,
1721
- staged_count: Object.keys(collection.staging.staged).length,
1722
- }));
1723
- });
1724
- }
1725
- function shortHash(value) {
1726
- return crypto.createHash("sha256").update(value, "utf8").digest("hex").slice(0, 12);
1727
- }
1728
- /**
1729
- * Object keys must satisfy the server's key pattern; ids derived from file
1730
- * content fall back to a hash rather than failing the whole upload batch.
1731
- */
1732
- function safeKeySegment(value) {
1733
- return /^[A-Za-z0-9._-]{1,80}$/.test(value) ? value : shortHash(value);
1734
- }
1735
- function sha256(value) {
1736
- return crypto.createHash("sha256").update(value).digest("hex");
1737
- }
1738
- function withRedactionContentMetadata(metadata, options) {
1739
- if (!metadata) {
1740
- throw new Error("redacted evidence is missing redaction metadata");
1741
- }
1742
- return {
1743
- ...metadata,
1744
- original_content_hash_sha256: sha256(options.originalBytes),
1745
- sanitized_content_hash_sha256: sha256(options.sanitizedBytes),
1746
- original_byte_size: options.originalBytes.byteLength,
1747
- sanitized_byte_size: options.sanitizedBytes.byteLength,
1748
- };
1749
- }
1750
- async function ensurePrivateDir(dir) {
1751
- await fs.mkdir(dir, { recursive: true, mode: 0o700 });
1752
- await chmodPrivate(dir, 0o700);
1753
- }
1754
- async function chmodPrivate(target, mode) {
1755
- if (process.platform === "win32")
1756
- return;
1757
- await fs.chmod(target, mode).catch(() => undefined);
1758
- }
1759
- function isSecretLikePath(value) {
1760
- return SECRET_FILE_SEGMENT_PATTERN.test(value);
1761
1160
  }