@bli-cockpit/cli 0.2.27 → 0.2.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,4 @@
1
- import { EvidenceCompletenessPayloadSchema, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, SECRET_FILE_SEGMENT_PATTERN, SourceScanResultSchema, containsSecretLikeContent, redactSecretLikeContent, } from "@bli-cockpit/telemetry-core";
2
- import { spawn } from "node:child_process";
1
+ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, SourceScanResultSchema, } from "@bli-cockpit/telemetry-core";
3
2
  import crypto from "node:crypto";
4
3
  import fs from "node:fs/promises";
5
4
  import os from "node:os";
@@ -8,12 +7,18 @@ import { makeSourceAdapterIdentity, } from "./common.js";
8
7
  import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
9
8
  import { defaultCodexSessionDirs, } from "./codex-attribution.js";
10
9
  import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-policy.js";
10
+ import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject, } from "../raw-evidence-staging.js";
11
+ import { isSecretLikePath, safeKeySegment, sha256, shortHash, } from "./raw-evidence-keys.js";
12
+ import { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
13
+ import { countEvidenceEntries, makeEvidenceCompleteness, markBudgetCapApplied, markCapApplied, recordScanned, recordSkipCount, recordTruncationCount, } from "./raw-evidence-completeness.js";
14
+ import { evidenceEntry, pointerFromEntry, RAW_EVIDENCE_BUCKET, } from "./raw-evidence-manifest.js";
15
+ import { chmodPrivate, ensurePrivateDir, persistStagingState, promoteStagedPack, stageManifest, } from "./raw-evidence-pack-store.js";
16
+ import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, runGitDiff, } from "./raw-evidence-git-diff.js";
17
+ // Re-exported so every consumer keeps importing from `adapters/raw-evidence`.
18
+ export { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./raw-evidence-manifest.js";
19
+ export { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
11
20
  const DEFAULT_SINCE_MINUTES = 24 * 60;
12
21
  const DEFAULT_SESSION_LIMIT = 50;
13
- const MAX_GIT_DIFF_BYTES = 2 * 1024 * 1024;
14
- const GIT_DIFF_TIMEOUT_MS = 3_000;
15
- export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
16
- export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
17
22
  // Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
18
23
  // transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
19
24
  // so 2 GiB leaves room for several files without starving the sync; overflow
@@ -21,250 +26,394 @@ export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
21
26
  export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
22
27
  export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
23
28
  const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;
29
+ // ---------------------------------------------------------------------------
30
+ // The pass
31
+ // ---------------------------------------------------------------------------
24
32
  export async function collectRawEvidencePack(context, options) {
25
33
  const startedAt = context.now.toISOString();
26
- const packId = rawEvidencePackId(context);
27
- const evidenceDir = path.join(options.stateDir, "raw-evidence", packId);
28
- const filesDir = path.join(evidenceDir, "files");
29
- const entries = [];
30
- const skipped = [];
31
- const truncated = [];
32
- const failed = [];
33
- const redacted = [];
34
- const reused = [];
34
+ const rawEvidenceRoot = path.join(options.stateDir, "raw-evidence");
35
+ // Staging first, promotion second (BLI-3066). The pack id cannot be known
36
+ // until the content is, so bytes land in a private staging directory and the
37
+ // directory is then renamed to its content-keyed name — or dropped, when an
38
+ // identical pack is already there.
39
+ const stagingDir = path.join(rawEvidenceRoot, `.staging-${process.pid}-${crypto.randomUUID().slice(0, 8)}`);
35
40
  const sinceMinutes = options.sinceMinutes ?? DEFAULT_SINCE_MINUTES;
36
- const sessionLimit = options.sessionLimit ?? DEFAULT_SESSION_LIMIT;
37
- const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
38
- const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
39
- const collection = {
40
- context,
41
- filesDir,
42
- packId,
43
- entries,
44
- skipped,
45
- truncated,
46
- failed,
47
- redacted,
48
- reused,
49
- scanned: new Map(),
50
- caps: [
51
- {
52
- source: "raw_evidence",
53
- cap_type: "byte_budget",
54
- limit: byteBudget,
55
- observed: options.budget?.remainingBytes ?? byteBudget,
56
- applied: false,
57
- },
58
- {
59
- source: "raw_evidence",
60
- cap_type: "object_budget",
61
- limit: objectBudget,
62
- observed: options.budget?.remainingObjects ?? objectBudget,
63
- applied: false,
64
- },
65
- {
66
- source: "git_diff",
67
- cap_type: "max_bytes_per_diff",
68
- limit: MAX_GIT_DIFF_BYTES,
69
- applied: false,
70
- },
71
- {
72
- source: "git_diff",
73
- cap_type: "timeout_ms",
74
- limit: GIT_DIFF_TIMEOUT_MS,
75
- applied: false,
76
- },
77
- {
78
- source: "codex_jsonl",
79
- cap_type: "max_file_bytes",
80
- limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
81
- applied: false,
82
- },
83
- {
84
- source: "claude_jsonl",
85
- cap_type: "max_file_bytes",
86
- limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
87
- applied: false,
88
- },
89
- {
90
- source: "claude_jsonl_sidecar",
91
- cap_type: "max_file_bytes",
92
- limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
93
- applied: false,
94
- },
95
- ],
96
- skipContentHashes: options.skipContentHashes ?? new Set(),
97
- budget: options.budget ?? {
98
- remainingBytes: byteBudget,
99
- remainingObjects: objectBudget,
100
- },
101
- index: { value: 0 },
41
+ const collection = await openCollection(context, options, {
42
+ rawEvidenceRoot,
43
+ stagingDir,
44
+ });
45
+ const scanWindow = {
46
+ startedAt,
47
+ finishedAt: () => context.now.toISOString(),
48
+ sinceMinutes,
102
49
  };
103
50
  try {
104
- await ensurePrivateDir(evidenceDir);
105
- await ensurePrivateDir(filesDir);
106
- recordAttributionCompleteness(collection, {
107
- codex: options.codexAttributionScan,
108
- claude: options.claudeAttributionScan,
109
- selectedCodexPaths: new Set(options.codexSessionFiles?.map((file) => file.local_path) ?? []),
110
- selectedClaudePaths: new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []),
111
- });
51
+ await ensurePrivateDir(stagingDir);
52
+ await ensurePrivateDir(collection.filesDir);
53
+ recordAttributionCompleteness(collection, options);
112
54
  if (options.includeCodexJsonl !== false) {
113
55
  await collectCodexJsonlFiles(collection, {
114
56
  codexSessionFiles: options.codexSessionFiles,
115
57
  sessionsDir: options.sessionsDir,
116
58
  sessionsDirs: options.sessionsDirs,
117
59
  sinceMinutes,
118
- limit: sessionLimit,
60
+ limit: options.sessionLimit ?? DEFAULT_SESSION_LIMIT,
119
61
  });
120
62
  }
121
63
  if (options.includeClaudeJsonl !== false && options.claudeSessionFiles) {
122
64
  await collectClaudeJsonlFiles(collection, options.claudeSessionFiles);
123
65
  }
124
66
  await collectGitDiffFiles(collection, options.repoRoot);
125
- const deferredByteBudgetCount = skipped.filter((entry) => entry.reason === "deferred_byte_budget").reduce((sum, entry) => sum + evidenceEntryCount(entry), 0);
126
- const deferredObjectBudgetCount = skipped.filter((entry) => entry.reason === "deferred_object_budget").reduce((sum, entry) => sum + evidenceEntryCount(entry), 0);
127
- if (entries.length === 0) {
128
- const evidenceCompleteness = makeEvidenceCompleteness(collection, {
129
- startedAt,
130
- finishedAt: context.now.toISOString(),
131
- sinceMinutes,
67
+ if (collection.entries.length === 0) {
68
+ return await finishWithEmptyPack(collection, options, {
69
+ stagingDir,
70
+ window: scanWindow,
132
71
  });
133
- const facts = {
134
- pack_id: packId,
135
- manifest_path: path.join(evidenceDir, "manifest.json"),
136
- evidence_dir: evidenceDir,
137
- storage_bucket: RAW_EVIDENCE_BUCKET,
138
- file_count: 0,
139
- byte_size: 0,
140
- skipped_count: countEvidenceEntries(skipped),
141
- sanitized_count: redacted.length,
142
- reused_count: reused.length,
143
- deferred_byte_budget_count: deferredByteBudgetCount,
144
- deferred_object_budget_count: deferredObjectBudgetCount,
145
- content_kinds: [],
146
- evidence_completeness: evidenceCompleteness,
147
- pointers: [],
148
- upload_files: [],
149
- reused,
150
- };
151
- return {
152
- facts,
153
- scan: makeRawEvidenceScan({
154
- context,
155
- startedAt,
156
- status: "partial",
157
- facts,
158
- }),
159
- };
160
72
  }
161
- const manifestWithoutSelf = makeManifest({
162
- context,
163
- packId,
164
- entries,
165
- skipped,
166
- redacted,
167
- reused,
73
+ return await finishWithPromotedPack(collection, options, {
74
+ stagingDir,
75
+ window: scanWindow,
168
76
  });
169
- const manifestPath = path.join(evidenceDir, "manifest.json");
170
- const manifestBytes = Buffer.from(`${JSON.stringify(manifestWithoutSelf, null, 2)}\n`, "utf8");
171
- await fs.writeFile(manifestPath, manifestBytes, { mode: 0o600 });
172
- await chmodPrivate(manifestPath, 0o600);
173
- const manifestEntry = evidenceEntry({
174
- context,
175
- kind: "manifest",
176
- packId,
177
- localPath: manifestPath,
178
- relativePath: "manifest.json",
179
- mediaType: "application/json",
180
- redactedSummary: "Local raw evidence pack manifest.",
181
- bytes: manifestBytes,
182
- });
183
- entries.push(manifestEntry);
184
- const evidenceCompleteness = makeEvidenceCompleteness(collection, {
185
- startedAt,
186
- finishedAt: context.now.toISOString(),
187
- sinceMinutes,
188
- });
189
- const facts = {
190
- pack_id: packId,
191
- manifest_path: manifestPath,
192
- evidence_dir: evidenceDir,
193
- storage_bucket: RAW_EVIDENCE_BUCKET,
194
- file_count: entries.length,
195
- byte_size: entries.reduce((sum, entry) => sum + entry.byte_size, 0),
196
- skipped_count: countEvidenceEntries(skipped),
197
- sanitized_count: redacted.length,
198
- reused_count: reused.length,
199
- deferred_byte_budget_count: deferredByteBudgetCount,
200
- deferred_object_budget_count: deferredObjectBudgetCount,
201
- content_kinds: [...new Set(entries.map((entry) => entry.kind))],
202
- evidence_completeness: evidenceCompleteness,
203
- pointers: entries.map(pointerFromEntry),
204
- upload_files: entries.map((entry) => ({
205
- pointer: pointerFromEntry(entry),
206
- local_path: entry.local_path,
207
- kind: entry.kind,
208
- codex_session_id: entry.codex_session_id ?? null,
209
- ...(entry.artifact_metadata
210
- ? { artifact_metadata: entry.artifact_metadata }
211
- : {}),
212
- })),
213
- reused,
214
- };
215
- return {
216
- facts,
217
- scan: makeRawEvidenceScan({
218
- context,
219
- startedAt,
220
- status: entries.length > 1 ? "ok" : "partial",
221
- facts,
222
- }),
223
- };
224
77
  }
225
- catch {
226
- failed.push({
227
- kind: "raw_evidence",
228
- reason: "collection_failed",
78
+ catch (error) {
79
+ return await finishWithFailedPack(collection, {
80
+ stagingDir,
81
+ window: scanWindow,
82
+ error,
229
83
  });
230
- const evidenceCompleteness = makeEvidenceCompleteness(collection, {
231
- startedAt,
232
- finishedAt: context.now.toISOString(),
233
- sinceMinutes,
234
- });
235
- const facts = {
236
- pack_id: packId,
237
- manifest_path: path.join(evidenceDir, "manifest.json"),
238
- evidence_dir: evidenceDir,
239
- storage_bucket: RAW_EVIDENCE_BUCKET,
240
- file_count: 0,
241
- byte_size: 0,
242
- skipped_count: countEvidenceEntries(skipped),
243
- sanitized_count: redacted.length,
244
- reused_count: reused.length,
245
- deferred_byte_budget_count: skipped
246
- .filter((entry) => entry.reason === "deferred_byte_budget")
247
- .reduce((sum, entry) => sum + evidenceEntryCount(entry), 0),
248
- deferred_object_budget_count: skipped
249
- .filter((entry) => entry.reason === "deferred_object_budget")
250
- .reduce((sum, entry) => sum + evidenceEntryCount(entry), 0),
251
- content_kinds: [],
252
- evidence_completeness: evidenceCompleteness,
253
- pointers: [],
254
- upload_files: [],
255
- reused,
256
- };
257
- return {
84
+ }
85
+ }
86
+ async function openCollection(context, options, places) {
87
+ const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
88
+ const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
89
+ const staging = await readRawEvidenceStagingState(options.stateDir);
90
+ return {
91
+ context,
92
+ filesDir: path.join(places.stagingDir, "files"),
93
+ rawEvidenceRoot: places.rawEvidenceRoot,
94
+ packId: "",
95
+ staging,
96
+ heldSources: collectionHeldSources(staging, context.now, options),
97
+ stagedReusedCount: 0,
98
+ stagedNewCount: 0,
99
+ deliveryHeldCount: 0,
100
+ entries: [],
101
+ skipped: [],
102
+ truncated: [],
103
+ failed: [],
104
+ redacted: [],
105
+ reused: [],
106
+ scanned: new Map(),
107
+ caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
108
+ skipContentHashes: options.skipContentHashes ?? new Set(),
109
+ budget: options.budget ?? {
110
+ remainingBytes: byteBudget,
111
+ remainingObjects: objectBudget,
112
+ },
113
+ index: { value: 0 },
114
+ };
115
+ }
116
+ /**
117
+ * Which sources this pass refuses to even read because their delivery is inside
118
+ * a backoff window — none of them, when a person asked for this pass.
119
+ *
120
+ * The bypass is logged rather than assumed: an operator retry that quietly
121
+ * ignored a hold would be as unreadable as the hold that quietly blocked it.
122
+ */
123
+ function collectionHeldSources(staging, now, options) {
124
+ const held = heldSourceKeys(staging, now);
125
+ if (deliveryBackoffApplies(options.deliveryMode))
126
+ return held;
127
+ if (held.size > 0) {
128
+ console.error("[raw-evidence] delivery backoff bypassed for operator retry", JSON.stringify({
129
+ reason: DELIVERY_BACKOFF_BYPASS_REASON,
130
+ source_count: held.size,
131
+ }));
132
+ }
133
+ return new Set();
134
+ }
135
+ /** Every cap this pass could hit, declared up front and flipped when applied. */
136
+ function startingCaps(options) {
137
+ return [
138
+ {
139
+ source: "raw_evidence",
140
+ cap_type: "byte_budget",
141
+ limit: options.byteBudget,
142
+ observed: options.budget?.remainingBytes ?? options.byteBudget,
143
+ applied: false,
144
+ },
145
+ {
146
+ source: "raw_evidence",
147
+ cap_type: "object_budget",
148
+ limit: options.objectBudget,
149
+ observed: options.budget?.remainingObjects ?? options.objectBudget,
150
+ applied: false,
151
+ },
152
+ {
153
+ source: "git_diff",
154
+ cap_type: "max_bytes_per_diff",
155
+ limit: MAX_GIT_DIFF_BYTES,
156
+ applied: false,
157
+ },
158
+ {
159
+ source: "git_diff",
160
+ cap_type: "timeout_ms",
161
+ limit: GIT_DIFF_TIMEOUT_MS,
162
+ applied: false,
163
+ },
164
+ {
165
+ source: "codex_jsonl",
166
+ cap_type: "max_file_bytes",
167
+ limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
168
+ applied: false,
169
+ },
170
+ {
171
+ source: "claude_jsonl",
172
+ cap_type: "max_file_bytes",
173
+ limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
174
+ applied: false,
175
+ },
176
+ {
177
+ source: "claude_jsonl_sidecar",
178
+ cap_type: "max_file_bytes",
179
+ limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
180
+ applied: false,
181
+ },
182
+ ];
183
+ }
184
+ // ---------------------------------------------------------------------------
185
+ // The three ways a pass ends
186
+ // ---------------------------------------------------------------------------
187
+ /**
188
+ * Nothing collected: drop the staging directory instead of leaving an empty
189
+ * pack behind. This used to leave one empty `work-*` dir per sync.
190
+ */
191
+ async function finishWithEmptyPack(collection, options, run) {
192
+ await discardStagingDir(run.stagingDir);
193
+ collection.packId = contentKeyedRawEvidencePackId({
194
+ workContextId: collection.context.workContextId,
195
+ contentHashes: [],
196
+ });
197
+ await persistStagingState(options.stateDir, collection.staging, collection.context.now.toISOString());
198
+ const facts = makeEmptyPackFacts(collection, run.window);
199
+ return {
200
+ facts,
201
+ scan: makeRawEvidenceScan({
202
+ context: collection.context,
203
+ startedAt: run.window.startedAt,
204
+ status: "partial",
205
+ facts,
206
+ }),
207
+ };
208
+ }
209
+ async function finishWithPromotedPack(collection, options, run) {
210
+ const context = collection.context;
211
+ const entries = collection.entries;
212
+ // The pack is named by what is in it, never by when it was made. Identical
213
+ // content on the next sync resolves to the identical directory, which is
214
+ // the whole fix for BLI-3066.
215
+ const packId = contentKeyedRawEvidencePackId({
216
+ workContextId: context.workContextId,
217
+ contentHashes: entries.map((entry) => entry.content_hash_sha256),
218
+ });
219
+ collection.packId = packId;
220
+ const promotion = await promoteStagedPack({
221
+ rawEvidenceRoot: collection.rawEvidenceRoot,
222
+ stagingDir: run.stagingDir,
223
+ packId,
224
+ workContextId: context.workContextId,
225
+ entries,
226
+ });
227
+ const evidenceDir = promotion.evidenceDir;
228
+ rebaseStagedEntriesOntoPack(collection, evidenceDir);
229
+ const manifestPath = path.join(evidenceDir, "manifest.json");
230
+ const manifestBytes = await stageManifest({
231
+ context,
232
+ packId,
233
+ manifestPath,
234
+ entries,
235
+ skipped: collection.skipped,
236
+ redacted: collection.redacted,
237
+ reused: collection.reused,
238
+ reusePack: promotion.state === "reused",
239
+ });
240
+ entries.push(evidenceEntry({
241
+ context,
242
+ kind: "manifest",
243
+ packId,
244
+ localPath: manifestPath,
245
+ relativePath: "manifest.json",
246
+ mediaType: "application/json",
247
+ redactedSummary: "Local raw evidence pack manifest.",
248
+ bytes: manifestBytes,
249
+ }));
250
+ await persistStagingState(options.stateDir, collection.staging, context.now.toISOString());
251
+ console.error("[raw-evidence] pack staged", JSON.stringify({
252
+ pack_id: packId,
253
+ stage_state: promotion.state,
254
+ reason: stageReasonLabel(promotion.state, promotion.priorPackCount),
255
+ prior_pack_count: promotion.priorPackCount,
256
+ refilled_file_count: promotion.refilledFileCount,
257
+ file_count: entries.length,
258
+ byte_size: totalByteSize(entries),
259
+ staged_new: collection.stagedNewCount,
260
+ staged_reused: collection.stagedReusedCount,
261
+ delivery_held: collection.deliveryHeldCount,
262
+ }));
263
+ const facts = {
264
+ ...packFactsShell(collection, {
265
+ packId,
266
+ manifestPath,
267
+ evidenceDir,
268
+ window: run.window,
269
+ }),
270
+ stage_state: promotion.state,
271
+ file_count: entries.length,
272
+ byte_size: totalByteSize(entries),
273
+ content_kinds: [...new Set(entries.map((entry) => entry.kind))],
274
+ pointers: entries.map(pointerFromEntry),
275
+ upload_files: entries.map(uploadFileFromEntry),
276
+ };
277
+ return {
278
+ facts,
279
+ scan: makeRawEvidenceScan({
280
+ context,
281
+ startedAt: run.window.startedAt,
282
+ status: entries.length > 1 ? "ok" : "partial",
283
+ facts,
284
+ }),
285
+ };
286
+ }
287
+ /**
288
+ * A crashed pass. The staged-object index is deliberately NOT written: this
289
+ * attempt's bookkeeping describes a directory that is about to be deleted.
290
+ */
291
+ async function finishWithFailedPack(collection, run) {
292
+ collection.failed.push({
293
+ kind: "raw_evidence",
294
+ reason: "collection_failed",
295
+ });
296
+ // Staging is per-attempt scratch: a crashed pass must not leave a partial
297
+ // directory behind to be counted, re-hashed or swept later.
298
+ await discardStagingDir(run.stagingDir);
299
+ collection.packId =
300
+ collection.packId ||
301
+ contentKeyedRawEvidencePackId({
302
+ workContextId: collection.context.workContextId,
303
+ contentHashes: collection.entries.map((entry) => entry.content_hash_sha256),
304
+ });
305
+ console.error("[raw-evidence] pack collection failed", JSON.stringify({
306
+ pack_id: collection.packId,
307
+ reason: "collection_failed",
308
+ detail: run.error instanceof Error ? run.error.name : typeof run.error,
309
+ collected_file_count: collection.entries.length,
310
+ staged_new: collection.stagedNewCount,
311
+ staged_reused: collection.stagedReusedCount,
312
+ }));
313
+ const facts = makeEmptyPackFacts(collection, run.window);
314
+ return {
315
+ facts,
316
+ scan: makeRawEvidenceScan({
317
+ context: collection.context,
318
+ startedAt: run.window.startedAt,
319
+ status: "failed",
258
320
  facts,
259
- scan: makeRawEvidenceScan({
260
- context,
261
- startedAt,
262
- status: "failed",
263
- facts,
264
- }),
265
- };
321
+ }),
322
+ };
323
+ }
324
+ /**
325
+ * A pack with no files of its own: the empty pass and the crashed pass report
326
+ * the same shape, differing only in the scan status and the reason already
327
+ * logged. `evidence_completeness` still carries every gap this pass recorded.
328
+ */
329
+ function makeEmptyPackFacts(collection, scanWindow) {
330
+ const evidenceDir = path.join(collection.rawEvidenceRoot, collection.packId);
331
+ return {
332
+ ...packFactsShell(collection, {
333
+ packId: collection.packId,
334
+ manifestPath: path.join(evidenceDir, "manifest.json"),
335
+ evidenceDir,
336
+ window: scanWindow,
337
+ }),
338
+ stage_state: "empty",
339
+ file_count: 0,
340
+ byte_size: 0,
341
+ content_kinds: [],
342
+ pointers: [],
343
+ upload_files: [],
344
+ };
345
+ }
346
+ /** The counts every ending reports identically, however the pass ended. */
347
+ function packFactsShell(collection, place) {
348
+ return {
349
+ pack_id: place.packId,
350
+ manifest_path: place.manifestPath,
351
+ evidence_dir: place.evidenceDir,
352
+ storage_bucket: RAW_EVIDENCE_BUCKET,
353
+ skipped_count: countEvidenceEntries(collection.skipped),
354
+ sanitized_count: collection.redacted.length,
355
+ reused_count: collection.reused.length,
356
+ staged_reused_count: collection.stagedReusedCount,
357
+ staged_new_count: collection.stagedNewCount,
358
+ delivery_held_count: collection.deliveryHeldCount,
359
+ deferred_byte_budget_count: countDeferred(collection, "deferred_byte_budget"),
360
+ deferred_object_budget_count: countDeferred(collection, "deferred_object_budget"),
361
+ evidence_completeness: makeEvidenceCompleteness(collection, {
362
+ startedAt: place.window.startedAt,
363
+ finishedAt: place.window.finishedAt(),
364
+ sinceMinutes: place.window.sinceMinutes,
365
+ }),
366
+ reused: collection.reused,
367
+ };
368
+ }
369
+ function countDeferred(collection, reason) {
370
+ return countEvidenceEntries(collection.skipped, (entry) => entry.reason === reason);
371
+ }
372
+ function totalByteSize(entries) {
373
+ return entries.reduce((sum, entry) => sum + entry.byte_size, 0);
374
+ }
375
+ function uploadFileFromEntry(entry) {
376
+ return {
377
+ pointer: pointerFromEntry(entry),
378
+ local_path: entry.local_path,
379
+ kind: entry.kind,
380
+ codex_session_id: entry.codex_session_id ?? null,
381
+ ...(entry.artifact_metadata
382
+ ? { artifact_metadata: entry.artifact_metadata }
383
+ : {}),
384
+ };
385
+ }
386
+ function stageReasonLabel(state, priorPackCount) {
387
+ if (state === "reused")
388
+ return "staged_reused";
389
+ if (state === "restaged_incomplete")
390
+ return "restaged_incomplete";
391
+ return priorPackCount > 0 ? "restaged_content_changed" : "staged_new";
392
+ }
393
+ /**
394
+ * Point every entry this pass staged at its home in the promoted pack, and
395
+ * remember the content hash so the next sync can adopt the copy instead of
396
+ * writing it again. Entries adopted from another pack keep their path.
397
+ */
398
+ function rebaseStagedEntriesOntoPack(collection, evidenceDir) {
399
+ for (const entry of collection.entries) {
400
+ if (!entry.staged_in_pack)
401
+ continue;
402
+ entry.local_path = path.join(evidenceDir, "files", path.basename(entry.local_path));
403
+ recordStagedObject(collection.staging, entry.content_hash_sha256, {
404
+ pack_id: collection.packId,
405
+ relative_path: `files/${path.basename(entry.local_path)}`,
406
+ byte_size: entry.byte_size,
407
+ source_key: entry.source_key,
408
+ staged_at: collection.context.now.toISOString(),
409
+ });
266
410
  }
267
411
  }
412
+ async function discardStagingDir(stagingDir) {
413
+ await fs
414
+ .rm(stagingDir, { recursive: true, force: true })
415
+ .catch(() => undefined);
416
+ }
268
417
  function makeRawEvidenceScan(options) {
269
418
  return SourceScanResultSchema.parse({
270
419
  adapter: makeSourceAdapterIdentity("collector_runtime", "raw-evidence-pack"),
@@ -293,6 +442,10 @@ function makeRawEvidenceScan(options) {
293
442
  `bytes:${options.facts.byte_size}`,
294
443
  `skipped:${options.facts.skipped_count}`,
295
444
  `reused:${options.facts.reused_count}`,
445
+ `stage_state:${options.facts.stage_state}`,
446
+ `staged_new:${options.facts.staged_new_count}`,
447
+ `staged_reused:${options.facts.staged_reused_count}`,
448
+ `delivery_held:${options.facts.delivery_held_count}`,
296
449
  `completeness:${options.facts.evidence_completeness.status}`,
297
450
  `truncated:${options.facts.evidence_completeness.totals.truncated_count}`,
298
451
  `deferred:${options.facts.evidence_completeness.totals.deferred_count}`,
@@ -301,12 +454,20 @@ function makeRawEvidenceScan(options) {
301
454
  ],
302
455
  });
303
456
  }
304
- function recordAttributionCompleteness(collection, scans) {
305
- if (scans.codex) {
306
- recordCodexAttributionCompleteness(collection, scans.codex, scans.selectedCodexPaths);
457
+ // ---------------------------------------------------------------------------
458
+ // What attribution already knew
459
+ // ---------------------------------------------------------------------------
460
+ /**
461
+ * Attribution ran before collection and already knows which sessions it could
462
+ * not name. Those are gaps in this pass even though no byte was read, so they
463
+ * are recorded first — a session Cockpit never looked at must not be invisible.
464
+ */
465
+ function recordAttributionCompleteness(collection, options) {
466
+ if (options.codexAttributionScan) {
467
+ recordCodexAttributionCompleteness(collection, options.codexAttributionScan, new Set(options.codexSessionFiles?.map((file) => file.local_path) ?? []));
307
468
  }
308
- if (scans.claude) {
309
- recordClaudeAttributionCompleteness(collection, scans.claude, scans.selectedClaudePaths);
469
+ if (options.claudeAttributionScan) {
470
+ recordClaudeAttributionCompleteness(collection, options.claudeAttributionScan, new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []));
310
471
  }
311
472
  }
312
473
  function recordCodexAttributionCompleteness(collection, scan, selectedPaths) {
@@ -384,39 +545,46 @@ function recordClaudeAttributionCompleteness(collection, scan, selectedPaths) {
384
545
  }
385
546
  function recordAttributionResultSkips(collection, source, results, selectedPaths) {
386
547
  for (const result of results) {
387
- // A selected session is being collected by this pass. A live-sync-safe
388
- // synthetic target is also not a skip when another workspace pack owns it.
389
- if (result.state === "attributed" ||
390
- selectedPaths.has(result.file_path) ||
391
- (result.worktree !== null &&
392
- isLiveRawEvidenceSyncAttribution(result.state, result.reason, true))) {
548
+ if (isAttributionAccountedFor(result, selectedPaths))
393
549
  continue;
394
- }
395
550
  const reason = result.state === "skipped"
396
551
  ? result.reason
397
552
  : `attribution_${result.state}:${result.reason}`;
398
553
  recordSkipCount(collection, source, reason, 1);
399
554
  }
400
555
  }
556
+ /**
557
+ * Not every unattributed result is a gap. A selected session is being collected
558
+ * by this pass, and a live-sync-safe synthetic target is owned by another
559
+ * workspace's pack — neither is missing evidence.
560
+ */
561
+ function isAttributionAccountedFor(result, selectedPaths) {
562
+ if (result.state === "attributed")
563
+ return true;
564
+ if (selectedPaths.has(result.file_path))
565
+ return true;
566
+ return (result.worktree !== null &&
567
+ isLiveRawEvidenceSyncAttribution(result.state, result.reason, true));
568
+ }
569
+ // ---------------------------------------------------------------------------
570
+ // Codex transcripts
571
+ // ---------------------------------------------------------------------------
401
572
  async function collectCodexJsonlFiles(collection, options) {
573
+ const attributed = Boolean(options.codexSessionFiles);
402
574
  const resolvedCandidates = options.codexSessionFiles
403
575
  ? options.codexSessionFiles.map((file) => ({
404
576
  filePath: file.local_path,
405
577
  codexSessionId: file.codex_session_id,
406
578
  }))
407
- : (await walkJsonlFiles(options.sessionsDirs ??
408
- (options.sessionsDir
409
- ? [options.sessionsDir]
410
- : defaultCodexSessionDirs(os.homedir())), collection.context.now.getTime() - options.sinceMinutes * 60 * 1000))
411
- .map((filePath) => ({ filePath, codexSessionId: null }));
579
+ : (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
412
580
  collection.caps.push({
413
581
  source: "codex_jsonl",
414
582
  cap_type: "session_limit",
415
583
  limit: options.limit,
416
584
  observed: resolvedCandidates.length,
417
- applied: !options.codexSessionFiles && resolvedCandidates.length > options.limit,
585
+ applied: !attributed && resolvedCandidates.length > options.limit,
418
586
  });
419
- const candidates = options.codexSessionFiles
587
+ const candidates = attributed
420
588
  ? resolvedCandidates
421
589
  : resolvedCandidates.slice(0, options.limit);
422
590
  for (const candidate of candidates) {
@@ -433,95 +601,118 @@ async function collectCodexJsonlFiles(collection, options) {
433
601
  redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
434
602
  contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
435
603
  });
436
- if (transcriptAccepted) {
437
- await collectAgentImagesFromTranscript(collection, {
438
- filePath: candidate.filePath,
439
- source: "codex",
440
- sessionId: codexSessionId,
441
- kind: "codex_image_attachment",
442
- contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
443
- });
444
- }
604
+ if (!transcriptAccepted)
605
+ continue;
606
+ await collectAgentImagesFromTranscript(collection, {
607
+ filePath: candidate.filePath,
608
+ source: "codex",
609
+ sessionId: codexSessionId,
610
+ kind: "codex_image_attachment",
611
+ contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
612
+ });
445
613
  }
446
614
  }
615
+ /** The unattributed fallback: every recent Codex transcript on this machine. */
616
+ function walkRecentCodexJsonlFiles(collection, options) {
617
+ const dirs = options.sessionsDirs ??
618
+ (options.sessionsDir
619
+ ? [options.sessionsDir]
620
+ : defaultCodexSessionDirs(os.homedir()));
621
+ return walkJsonlFiles(dirs, collection.context.now.getTime() - options.sinceMinutes * 60 * 1000);
622
+ }
623
+ // ---------------------------------------------------------------------------
624
+ // Claude transcripts
625
+ // ---------------------------------------------------------------------------
447
626
  async function collectClaudeJsonlFiles(collection, sessions) {
448
627
  recordScanned(collection, "claude_jsonl", sessions.reduce((count, session) => count + 1 + session.sidecar_files.length, 0));
449
628
  for (const session of sessions) {
450
- const sessionId = session.claude_session_id;
451
- if (session.main_file_oversized) {
452
- // D7: the oversized main was attributed via a streamed read but its bytes
453
- // are never uploaded (server commit assembles in memory). Its sidecars
454
- // still collect below.
455
- collection.skipped.push({
456
- kind: "claude_jsonl",
457
- label: path.basename(session.local_path),
458
- reason: "file_too_large",
459
- });
460
- }
461
- else if (session.skip_main) {
462
- // D9 damped: prior durable copy is still good enough; collect sidecars
463
- // only. The session reports reused_existing from cursor state, so no
464
- // skipped entry is recorded here.
465
- }
466
- else {
467
- const mainAccepted = await collectOneEvidenceFile(collection, {
468
- filePath: session.local_path,
469
- kind: "claude_jsonl",
470
- sessionId,
471
- mediaType: "application/jsonl",
472
- // Re-check size at collection: a main that grew past the cap between
473
- // attribution and collection is an honest file_too_large skip, not an
474
- // upload_failed at the chunk client.
475
- maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
476
- redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
477
- contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/${hash16}.jsonl`,
478
- });
479
- if (mainAccepted) {
480
- await collectAgentImagesFromTranscript(collection, {
481
- filePath: session.local_path,
482
- source: "claude_code",
483
- sessionId,
484
- kind: "claude_image_attachment",
485
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
486
- });
487
- }
488
- }
489
- if (session.skip_main && !session.main_file_oversized) {
490
- await collectAgentImagesFromTranscript(collection, {
491
- filePath: session.local_path,
492
- source: "claude_code",
493
- sessionId,
494
- kind: "claude_image_attachment",
495
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
496
- });
497
- }
498
- for (const sidecar of session.sidecar_files) {
499
- const stem = path.basename(sidecar.local_path).replace(/\.jsonl$/i, "");
500
- const safeSidecarId = isSecretLikePath(stem)
501
- ? "redacted-file-name"
502
- : safeKeySegment(stem);
503
- const sidecarAccepted = await collectOneEvidenceFile(collection, {
504
- filePath: sidecar.local_path,
505
- kind: "claude_jsonl_sidecar",
506
- sessionId,
507
- mediaType: "application/jsonl",
508
- maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
509
- redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
510
- contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
511
- });
512
- if (sidecarAccepted) {
513
- await collectAgentImagesFromTranscript(collection, {
514
- filePath: sidecar.local_path,
515
- source: "claude_code",
516
- sessionId,
517
- sidecarId: safeSidecarId,
518
- kind: "claude_image_attachment",
519
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
520
- });
521
- }
522
- }
629
+ await collectOneClaudeSession(collection, session);
630
+ }
631
+ }
632
+ async function collectOneClaudeSession(collection, session) {
633
+ const sessionId = session.claude_session_id;
634
+ const mainOutcome = await collectClaudeMainFile(collection, session);
635
+ if (mainOutcome === "collected" || mainOutcome === "damped_reuse") {
636
+ await collectAgentImagesFromTranscript(collection, {
637
+ filePath: session.local_path,
638
+ source: "claude_code",
639
+ sessionId,
640
+ kind: "claude_image_attachment",
641
+ contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
642
+ });
643
+ }
644
+ for (const sidecar of session.sidecar_files) {
645
+ await collectOneClaudeSidecar(collection, session, sidecar.local_path);
523
646
  }
524
647
  }
648
+ /**
649
+ * What happens to a Claude main file, as a decision table:
650
+ *
651
+ * | condition | main bytes | outcome |
652
+ * | --------------------- | ------------------------- | ------------------- |
653
+ * | `main_file_oversized` | skipped `file_too_large` | `skipped_too_large` |
654
+ * | `skip_main` (D9) | not re-collected | `damped_reuse` |
655
+ * | otherwise | collected, or named skip | `collected` / `not_collected` |
656
+ *
657
+ * Its images are collected for every outcome except `skipped_too_large` and
658
+ * `not_collected` — the caller decides that, this function only reports.
659
+ *
660
+ * D7: an oversized main was attributed via a streamed read but its bytes are
661
+ * never uploaded (server commit assembles in memory). Its sidecars still
662
+ * collect. D9 damped: the prior durable copy is still good enough, and the
663
+ * session reports `reused_existing` from cursor state, so no skip is recorded.
664
+ */
665
+ async function collectClaudeMainFile(collection, session) {
666
+ if (session.main_file_oversized) {
667
+ collection.skipped.push({
668
+ kind: "claude_jsonl",
669
+ label: path.basename(session.local_path),
670
+ reason: "file_too_large",
671
+ });
672
+ return "skipped_too_large";
673
+ }
674
+ if (session.skip_main)
675
+ return "damped_reuse";
676
+ const accepted = await collectOneEvidenceFile(collection, {
677
+ filePath: session.local_path,
678
+ kind: "claude_jsonl",
679
+ sessionId: session.claude_session_id,
680
+ mediaType: "application/jsonl",
681
+ // Re-check size at collection: a main that grew past the cap between
682
+ // attribution and collection is an honest file_too_large skip, not an
683
+ // upload_failed at the chunk client.
684
+ maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
685
+ redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
686
+ contentAddress: (hash16) => `claude/${safeKeySegment(session.claude_session_id)}/${hash16}.jsonl`,
687
+ });
688
+ return accepted ? "collected" : "not_collected";
689
+ }
690
+ async function collectOneClaudeSidecar(collection, session, sidecarPath) {
691
+ const sessionId = session.claude_session_id;
692
+ const stem = path.basename(sidecarPath).replace(/\.jsonl$/i, "");
693
+ const safeSidecarId = isSecretLikePath(stem)
694
+ ? "redacted-file-name"
695
+ : safeKeySegment(stem);
696
+ const sidecarAccepted = await collectOneEvidenceFile(collection, {
697
+ filePath: sidecarPath,
698
+ kind: "claude_jsonl_sidecar",
699
+ sessionId,
700
+ mediaType: "application/jsonl",
701
+ maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
702
+ redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
703
+ contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
704
+ });
705
+ if (!sidecarAccepted)
706
+ return;
707
+ await collectAgentImagesFromTranscript(collection, {
708
+ filePath: sidecarPath,
709
+ source: "claude_code",
710
+ sessionId,
711
+ sidecarId: safeSidecarId,
712
+ kind: "claude_image_attachment",
713
+ contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
714
+ });
715
+ }
525
716
  async function collectAgentImagesFromTranscript(collection, options) {
526
717
  const result = await collectAgentImageEvidenceFromJsonlFile({
527
718
  filePath: options.filePath,
@@ -576,15 +767,26 @@ async function collectOneAgentImageFile(collection, options) {
576
767
  }
577
768
  collection.index.value += 1;
578
769
  const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-agent-image-${contentHash.slice(0, 16)}.${options.image.extension}`);
579
- const destination = path.join(collection.filesDir, path.basename(relativePath));
580
- await fs.writeFile(destination, raw, { mode: 0o600 });
581
- await chmodPrivate(destination, 0o600);
770
+ const imageSourceKey = evidenceSourceKey({
771
+ kind: options.kind,
772
+ sessionId: options.sessionId,
773
+ label: options.image.label,
774
+ });
775
+ const staged = await stageEvidenceBytes(collection, {
776
+ contentHash,
777
+ bytes: raw,
778
+ fileName: path.basename(relativePath),
779
+ kind: options.kind,
780
+ sourceKey: imageSourceKey,
781
+ });
582
782
  collection.entries.push(evidenceEntry({
583
783
  context: collection.context,
584
784
  kind: options.kind,
585
785
  packId: collection.packId,
586
- localPath: destination,
786
+ localPath: staged.local_path,
587
787
  relativePath,
788
+ stagedInPack: staged.staged_in_pack,
789
+ sourceKey: imageSourceKey,
588
790
  mediaType: metadata.media_type,
589
791
  redactedSummary: "Raw image explicitly attached to an agent session, preserved in private durable storage.",
590
792
  bytes: raw,
@@ -593,63 +795,58 @@ async function collectOneAgentImageFile(collection, options) {
593
795
  artifactMetadata: metadata,
594
796
  }));
595
797
  }
798
+ // ---------------------------------------------------------------------------
799
+ // One transcript, end to end
800
+ // ---------------------------------------------------------------------------
596
801
  /**
597
802
  * Reads, secret-guards, content-addresses, budget-checks, and copies one
598
- * attributed transcript into the pack. Attribution only reads metadata records;
599
- * this collection layer and the server commit layer are the two content guards
600
- * that decide whether transcript bytes can become durable evidence.
803
+ * attributed transcript into the pack.
804
+ *
805
+ * Returns whether this file's *content* is accounted for, which is not the same
806
+ * as "was staged": a transcript whose bytes are already durable remotely
807
+ * returns true without staging anything, and its images are still worth
808
+ * collecting. Every false return has pushed a named skip first.
601
809
  */
602
810
  async function collectOneEvidenceFile(collection, options) {
603
811
  const fileName = path.basename(options.filePath);
604
812
  const secretLikeFileName = isSecretLikePath(fileName);
605
- const evidenceLabel = secretLikeFileName
606
- ? "[REDACTED_FILE_NAME]"
607
- : fileName;
813
+ const evidenceLabel = secretLikeFileName ? "[REDACTED_FILE_NAME]" : fileName;
608
814
  const packedFileName = secretLikeFileName ? "redacted-file.jsonl" : fileName;
609
- if (options.maxFileBytes) {
610
- let stat;
611
- try {
612
- stat = await fs.stat(options.filePath);
613
- }
614
- catch {
615
- collection.skipped.push({
616
- kind: options.kind,
617
- label: evidenceLabel,
618
- reason: "file_read_failed",
619
- });
620
- return false;
621
- }
622
- if (stat.size > options.maxFileBytes) {
623
- markCapApplied(collection, options.kind, "max_file_bytes");
624
- collection.skipped.push({
625
- kind: options.kind,
626
- label: evidenceLabel,
627
- reason: "file_too_large",
628
- });
629
- return false;
630
- }
631
- }
632
- let raw;
633
- try {
634
- raw = await fs.readFile(options.filePath);
635
- }
636
- catch {
815
+ const sourceKey = evidenceSourceKey({
816
+ kind: options.kind,
817
+ sessionId: options.sessionId,
818
+ sourcePath: options.filePath,
819
+ });
820
+ const skip = (reason) => {
637
821
  collection.skipped.push({
638
822
  kind: options.kind,
639
823
  label: evidenceLabel,
640
- reason: "file_read_failed",
824
+ reason,
641
825
  });
642
826
  return false;
643
- }
644
- if (options.maxFileBytes && raw.byteLength > options.maxFileBytes) {
645
- markCapApplied(collection, options.kind, "max_file_bytes");
646
- collection.skipped.push({
827
+ };
828
+ // Delivery backoff is checked BEFORE the file is read. An object whose commit
829
+ // has failed repeatedly costs nothing at all this cycle — no read, no hash,
830
+ // no copy, no request — and the hold is a named, retryable gap so a held
831
+ // session cannot make the sync look clean (BLI-3066).
832
+ if (collection.heldSources.has(sourceKey)) {
833
+ collection.deliveryHeldCount += 1;
834
+ skip(DELIVERY_BACKOFF_HOLDING_REASON);
835
+ console.error("[raw-evidence] delivery backoff holding source", JSON.stringify({
836
+ reason: DELIVERY_BACKOFF_HOLDING_REASON,
647
837
  kind: options.kind,
648
- label: evidenceLabel,
649
- reason: "file_too_large",
650
- });
838
+ source_key: sourceKey,
839
+ }));
651
840
  return false;
652
841
  }
842
+ const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
843
+ if (read.status === "read_failed")
844
+ return skip("file_read_failed");
845
+ if (read.status === "too_large") {
846
+ markCapApplied(collection, options.kind, "max_file_bytes");
847
+ return skip("file_too_large");
848
+ }
849
+ const raw = read.bytes;
653
850
  const sanitized = sanitizeTextEvidenceForUpload({
654
851
  text: raw.toString("utf8"),
655
852
  originalBytes: raw,
@@ -685,23 +882,22 @@ async function collectOneEvidenceFile(collection, options) {
685
882
  const deferReason = admitToBudget(collection.budget, evidenceBytes.byteLength);
686
883
  if (deferReason) {
687
884
  markBudgetCapApplied(collection, deferReason);
688
- collection.skipped.push({
689
- kind: options.kind,
690
- label: evidenceLabel,
691
- reason: deferReason,
692
- });
693
- return false;
885
+ return skip(deferReason);
694
886
  }
695
887
  collection.index.value += 1;
696
888
  const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(options.filePath)}-${packedFileName}`);
697
- const destination = path.join(collection.filesDir, path.basename(relativePath));
698
- await fs.writeFile(destination, evidenceBytes, { mode: 0o600 });
699
- await chmodPrivate(destination, 0o600);
889
+ const staged = await stageEvidenceBytes(collection, {
890
+ contentHash,
891
+ bytes: evidenceBytes,
892
+ fileName: path.basename(relativePath),
893
+ kind: options.kind,
894
+ sourceKey,
895
+ });
700
896
  collection.entries.push(evidenceEntry({
701
897
  context: collection.context,
702
898
  kind: options.kind,
703
899
  packId: collection.packId,
704
- localPath: destination,
900
+ localPath: staged.local_path,
705
901
  relativePath,
706
902
  mediaType: options.mediaType,
707
903
  redactedSummary: redaction
@@ -711,9 +907,70 @@ async function collectOneEvidenceFile(collection, options) {
711
907
  bytes: evidenceBytes,
712
908
  codexSessionId: options.sessionId,
713
909
  contentAddress: options.contentAddress(contentHash.slice(0, 16)),
910
+ stagedInPack: staged.staged_in_pack,
911
+ sourceKey,
714
912
  }));
715
913
  return true;
716
914
  }
915
+ /**
916
+ * Size is checked twice on purpose: once by `stat` so an oversized transcript
917
+ * is never buffered at all, and once on the bytes actually read, because a live
918
+ * session can grow between the two.
919
+ */
920
+ async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
921
+ if (maxFileBytes) {
922
+ let stat;
923
+ try {
924
+ stat = await fs.stat(filePath);
925
+ }
926
+ catch {
927
+ return { status: "read_failed" };
928
+ }
929
+ if (stat.size > maxFileBytes)
930
+ return { status: "too_large" };
931
+ }
932
+ let bytes;
933
+ try {
934
+ bytes = await fs.readFile(filePath);
935
+ }
936
+ catch {
937
+ return { status: "read_failed" };
938
+ }
939
+ if (maxFileBytes && bytes.byteLength > maxFileBytes) {
940
+ return { status: "too_large" };
941
+ }
942
+ return { status: "ok", bytes };
943
+ }
944
+ // ---------------------------------------------------------------------------
945
+ // Staging and budgets
946
+ // ---------------------------------------------------------------------------
947
+ /**
948
+ * Put these bytes on disk once.
949
+ *
950
+ * If an earlier sync already staged this exact content and the copy is still
951
+ * there, that copy is used — the uploader only needs a readable path, and it
952
+ * does not care which pack directory holds it. This is the branch that stops
953
+ * one 334 MB rollout from becoming 559 copies while its commit keeps failing.
954
+ */
955
+ async function stageEvidenceBytes(collection, options) {
956
+ const existing = await resolveStagedObject(collection.rawEvidenceRoot, collection.staging, options.contentHash);
957
+ if (existing) {
958
+ collection.stagedReusedCount += 1;
959
+ console.error("[raw-evidence] staged copy reused", JSON.stringify({
960
+ reason: "staged_reused",
961
+ kind: options.kind,
962
+ content_hash_prefix: options.contentHash.slice(0, 16),
963
+ byte_size: existing.entry.byte_size,
964
+ pack_id: existing.entry.pack_id,
965
+ }));
966
+ return { local_path: existing.local_path, staged_in_pack: false };
967
+ }
968
+ const destination = path.join(collection.filesDir, options.fileName);
969
+ await fs.writeFile(destination, options.bytes, { mode: 0o600 });
970
+ await chmodPrivate(destination, 0o600);
971
+ collection.stagedNewCount += 1;
972
+ return { local_path: destination, staged_in_pack: true };
973
+ }
717
974
  /**
718
975
  * Decrements the per-sync budget when a file fits, or returns a deferred-skip
719
976
  * reason when it does not. The object budget bounds request count; the byte
@@ -728,145 +985,15 @@ function admitToBudget(budget, byteLength) {
728
985
  budget.remainingBytes -= byteLength;
729
986
  return null;
730
987
  }
731
- export function sanitizeTextEvidenceForUpload(options) {
732
- const originalBytes = options.originalBytes ?? Buffer.from(options.text, "utf8");
733
- const secretLikeContent = containsSecretLikeContent(options.text);
734
- try {
735
- const redactionResult = (options.redact ?? redactSecretLikeContent)(options.text, {
736
- appliedBy: "local_collector",
737
- redactedFields: options.redactedFields,
738
- });
739
- if (options.secretLikeFileName) {
740
- const sanitizedText = redactionResult.redacted
741
- ? redactionResult.text
742
- : options.text;
743
- const safeText = containsSecretLikeContent(sanitizedText)
744
- ? "[REDACTED_LINE:secret_redaction_failed]\n"
745
- : sanitizedText;
746
- const sanitizedBytes = Buffer.from(safeText, "utf8");
747
- return {
748
- status: "redacted",
749
- bytes: sanitizedBytes,
750
- redaction: withRedactionContentMetadata(redactionResult.metadata ??
751
- fallbackRedactionMetadata({
752
- ruleId: "secret_like_file_name",
753
- originalText: options.text,
754
- redactedFields: options.redactedFields,
755
- fullContentRedacted: false,
756
- }), { originalBytes, sanitizedBytes }),
757
- completenessLabel: "secret_like_name_masked",
758
- };
759
- }
760
- if (secretLikeContent) {
761
- const redaction = redactionResult.metadata ??
762
- fallbackRedactionMetadata({
763
- ruleId: "secret_like_content_guard",
764
- originalText: options.text,
765
- redactedFields: options.redactedFields,
766
- });
767
- let maskedText = maskSecretBearingLines(options.text, redaction);
768
- if (containsSecretLikeContent(maskedText)) {
769
- maskedText = "[REDACTED_LINE:secret_redaction_failed]\n";
770
- }
771
- const sanitizedBytes = Buffer.from(maskedText, "utf8");
772
- return {
773
- status: "redacted",
774
- bytes: sanitizedBytes,
775
- redaction: withRedactionContentMetadata(redaction, {
776
- originalBytes,
777
- sanitizedBytes,
778
- }),
779
- completenessLabel: "secret_content_masked",
780
- };
781
- }
782
- if (redactionResult.redacted) {
783
- const sanitizedBytes = Buffer.from(redactionResult.text, "utf8");
784
- return {
785
- status: "redacted",
786
- bytes: sanitizedBytes,
787
- redaction: withRedactionContentMetadata(redactionResult.metadata ??
788
- fallbackRedactionMetadata({
789
- ruleId: "secret_redaction_failed",
790
- originalText: options.text,
791
- redactedFields: options.redactedFields,
792
- }), { originalBytes, sanitizedBytes }),
793
- completenessLabel: "secret_content_masked",
794
- };
795
- }
796
- return { status: "clean", bytes: originalBytes };
797
- }
798
- catch {
799
- const sanitizedBytes = Buffer.from("[REDACTED_LINE:redaction_crashed_stubbed]\n", "utf8");
800
- return {
801
- status: "redacted",
802
- bytes: sanitizedBytes,
803
- redaction: withRedactionContentMetadata(fallbackRedactionMetadata({
804
- ruleId: "redaction_crashed_stubbed",
805
- originalText: options.text,
806
- redactedFields: options.redactedFields,
807
- }), { originalBytes, sanitizedBytes }),
808
- completenessLabel: "redaction_crashed_stubbed",
809
- };
810
- }
811
- }
812
- function maskSecretBearingLines(text, redaction) {
813
- if (redaction.redacted_ranges.length === 0) {
814
- return "[REDACTED_LINE:secret_like_content_guard]\n";
815
- }
816
- const segments = text.match(/[^\n]*(?:\n|$)/gu)?.filter(Boolean) ?? [];
817
- let offset = 0;
818
- return segments
819
- .map((segment) => {
820
- const start = offset;
821
- const end = offset + segment.length;
822
- offset = end;
823
- const matched = redaction.redacted_ranges.find((range) => range.start < end && start < range.end);
824
- if (!matched)
825
- return segment;
826
- const lineEnding = segment.endsWith("\r\n")
827
- ? "\r\n"
828
- : segment.endsWith("\n")
829
- ? "\n"
830
- : "";
831
- return `[REDACTED_LINE:${matched.rule_id}]${lineEnding}`;
832
- })
833
- .join("");
834
- }
835
- function fallbackRedactionMetadata(options) {
836
- const fullContentRedacted = options.fullContentRedacted !== false;
837
- return {
838
- schema_version: "raw-evidence-redaction.v1",
839
- status: "sanitized",
840
- mode: "deterministic_text_replacement",
841
- applied_by: ["local_collector"],
842
- rule_counts: [
843
- {
844
- rule_id: options.ruleId,
845
- match_count: 1,
846
- redacted_char_count: fullContentRedacted
847
- ? options.originalText.length
848
- : 0,
849
- },
850
- ],
851
- secret_like_match_count: 1,
852
- redacted_fields: options.redactedFields,
853
- redacted_ranges: fullContentRedacted && options.originalText.length > 0
854
- ? [
855
- {
856
- start: 0,
857
- end: options.originalText.length,
858
- rule_id: options.ruleId,
859
- },
860
- ]
861
- : [],
862
- };
863
- }
988
+ // ---------------------------------------------------------------------------
989
+ // Git diffs
990
+ // ---------------------------------------------------------------------------
991
+ const GIT_DIFF_TARGETS = [
992
+ { label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
993
+ { label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
994
+ ];
864
995
  async function collectGitDiffFiles(collection, repoRoot) {
865
- const diffTargets = [
866
- { label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
867
- { label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
868
- ];
869
- for (const target of diffTargets) {
996
+ for (const target of GIT_DIFF_TARGETS) {
870
997
  recordScanned(collection, "git_diff");
871
998
  let diff;
872
999
  try {
@@ -892,334 +1019,107 @@ async function collectGitDiffFiles(collection, repoRoot) {
892
1019
  included_bytes: Buffer.byteLength(diff.stdout, "utf8"),
893
1020
  });
894
1021
  }
1022
+ // An empty diff is not a gap: there was simply nothing to record. The
1023
+ // truncation marker above still stands even when zero bytes survived.
895
1024
  if (!diff.stdout.trim())
896
1025
  continue;
897
- const sanitized = sanitizeTextEvidenceForUpload({
898
- text: diff.stdout,
899
- redactedFields: [`git_diff.${target.label}`],
1026
+ await stageOneGitDiff(collection, {
1027
+ label: target.label,
1028
+ diffText: diff.stdout,
1029
+ truncated: diff.truncated,
900
1030
  });
901
- if (sanitized.status === "redacted") {
902
- collection.redacted.push({
903
- kind: "git_diff",
904
- label: target.label,
905
- redaction: sanitized.redaction,
906
- completenessLabel: sanitized.completenessLabel,
907
- });
908
- console.error("[raw-evidence] git diff sanitized", JSON.stringify({
909
- mode: sanitized.completenessLabel,
910
- original_bytes: Buffer.byteLength(diff.stdout, "utf8"),
911
- uploaded_bytes: sanitized.bytes.byteLength,
912
- }));
913
- }
914
- const raw = sanitized.bytes;
915
- const redaction = sanitized.redaction;
916
- const contentHash = sha256(raw);
917
- if (collection.skipContentHashes.has(contentHash)) {
918
- collection.reused.push({
919
- kind: "git_diff",
920
- label: target.label,
921
- content_hash_sha256: contentHash,
922
- codex_session_id: null,
923
- });
924
- continue;
925
- }
926
- const deferReason = admitToBudget(collection.budget, raw.byteLength);
927
- if (deferReason) {
928
- markBudgetCapApplied(collection, deferReason);
929
- collection.skipped.push({
930
- kind: "git_diff",
931
- label: target.label,
932
- reason: deferReason,
933
- });
934
- continue;
935
- }
936
- const relativePath = path.join("files", `git-${target.label}.diff`);
937
- const destination = path.join(collection.filesDir, path.basename(relativePath));
938
- await fs.writeFile(destination, raw, { mode: 0o600 });
939
- await chmodPrivate(destination, 0o600);
940
- collection.entries.push(evidenceEntry({
941
- context: collection.context,
942
- kind: "git_diff",
943
- packId: collection.packId,
944
- localPath: destination,
945
- relativePath,
946
- mediaType: "text/x-diff",
947
- redactedSummary: redaction
948
- ? `Raw git ${target.label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`
949
- : diff.truncated
950
- ? `Raw git ${target.label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`
951
- : `Raw git ${target.label} diff preserved locally with env/secret paths excluded.`,
952
- redaction,
953
- bytes: raw,
954
- contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
955
- }));
956
1031
  }
957
1032
  }
958
- async function runGitDiff(args, repoRoot) {
959
- const pathspec = [
960
- ".",
961
- ":(exclude).env",
962
- ":(exclude).env.*",
963
- ":(exclude)**/.env",
964
- ":(exclude)**/.env.*",
965
- ":(exclude)**/*secret*",
966
- ":(exclude)**/*credential*",
967
- ":(exclude)**/*private-key*",
968
- ":(exclude)**/*.pem",
969
- ":(exclude)**/*.key",
970
- ];
971
- return new Promise((resolve, reject) => {
972
- const child = spawn("git", [...args, ...pathspec], {
973
- cwd: repoRoot,
974
- stdio: ["ignore", "pipe", "pipe"],
975
- });
976
- const stdoutChunks = [];
977
- const stderrChunks = [];
978
- let observedBytes = 0;
979
- let includedBytes = 0;
980
- let truncated = false;
981
- let timedOut = false;
982
- const timeout = setTimeout(() => {
983
- timedOut = true;
984
- truncated = true;
985
- child.kill("SIGTERM");
986
- }, GIT_DIFF_TIMEOUT_MS);
987
- child.stdout.on("data", (chunk) => {
988
- observedBytes += chunk.byteLength;
989
- if (includedBytes < MAX_GIT_DIFF_BYTES) {
990
- const remaining = MAX_GIT_DIFF_BYTES - includedBytes;
991
- const next = chunk.subarray(0, remaining);
992
- stdoutChunks.push(next);
993
- includedBytes += next.byteLength;
994
- }
995
- if (observedBytes > MAX_GIT_DIFF_BYTES) {
996
- truncated = true;
997
- child.kill("SIGTERM");
998
- }
999
- });
1000
- child.stderr.on("data", (chunk) => {
1001
- if (stderrChunks.reduce((sum, item) => sum + item.byteLength, 0) < 4096) {
1002
- stderrChunks.push(chunk.subarray(0, 4096));
1003
- }
1033
+ async function stageOneGitDiff(collection, target) {
1034
+ const sanitized = sanitizeTextEvidenceForUpload({
1035
+ text: target.diffText,
1036
+ redactedFields: [`git_diff.${target.label}`],
1037
+ });
1038
+ if (sanitized.status === "redacted") {
1039
+ collection.redacted.push({
1040
+ kind: "git_diff",
1041
+ label: target.label,
1042
+ redaction: sanitized.redaction,
1043
+ completenessLabel: sanitized.completenessLabel,
1004
1044
  });
1005
- child.on("error", (error) => {
1006
- clearTimeout(timeout);
1007
- reject(error);
1045
+ console.error("[raw-evidence] git diff sanitized", JSON.stringify({
1046
+ mode: sanitized.completenessLabel,
1047
+ original_bytes: Buffer.byteLength(target.diffText, "utf8"),
1048
+ uploaded_bytes: sanitized.bytes.byteLength,
1049
+ }));
1050
+ }
1051
+ const raw = sanitized.bytes;
1052
+ const redaction = sanitized.redaction;
1053
+ const contentHash = sha256(raw);
1054
+ if (collection.skipContentHashes.has(contentHash)) {
1055
+ collection.reused.push({
1056
+ kind: "git_diff",
1057
+ label: target.label,
1058
+ content_hash_sha256: contentHash,
1059
+ codex_session_id: null,
1008
1060
  });
1009
- child.on("close", (code, signal) => {
1010
- clearTimeout(timeout);
1011
- if (code === 0 || truncated || signal === "SIGTERM") {
1012
- resolve({
1013
- stdout: Buffer.concat(stdoutChunks).toString("utf8"),
1014
- truncated,
1015
- observedBytes,
1016
- truncationReason: timedOut
1017
- ? "git_diff_timeout"
1018
- : "max_git_diff_bytes",
1019
- truncationCapType: timedOut ? "timeout_ms" : "max_bytes_per_diff",
1020
- });
1021
- return;
1022
- }
1023
- const stderr = Buffer.concat(stderrChunks).toString("utf8").trim();
1024
- reject(new Error(stderr || `git diff failed with code ${code ?? signal}`));
1061
+ return;
1062
+ }
1063
+ const deferReason = admitToBudget(collection.budget, raw.byteLength);
1064
+ if (deferReason) {
1065
+ markBudgetCapApplied(collection, deferReason);
1066
+ collection.skipped.push({
1067
+ kind: "git_diff",
1068
+ label: target.label,
1069
+ reason: deferReason,
1025
1070
  });
1026
- });
1027
- }
1028
- function recordScanned(collection, source, count = 1) {
1029
- collection.scanned.set(source, (collection.scanned.get(source) ?? 0) + count);
1030
- }
1031
- function markBudgetCapApplied(collection, reason) {
1032
- markCapApplied(collection, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
1033
- }
1034
- function markCapApplied(collection, source, capType) {
1035
- const cap = collection.caps.find((candidate) => candidate.source === source && candidate.cap_type === capType);
1036
- if (cap)
1037
- cap.applied = true;
1038
- }
1039
- function recordSkipCount(collection, source, reason, count) {
1040
- if (count <= 0)
1041
1071
  return;
1042
- collection.skipped.push({
1043
- kind: source,
1044
- label: reason,
1045
- reason,
1046
- count,
1072
+ }
1073
+ const relativePath = path.join("files", `git-${target.label}.diff`);
1074
+ const diffSourceKey = evidenceSourceKey({
1075
+ kind: "git_diff",
1076
+ sessionId: collection.context.workContextId,
1077
+ label: target.label,
1047
1078
  });
1048
- }
1049
- function recordTruncationCount(collection, source, reason, count, details = {}) {
1050
- if (count <= 0)
1051
- return;
1052
- collection.truncated.push({
1053
- kind: source,
1054
- reason,
1055
- count,
1056
- ...details,
1079
+ const staged = await stageEvidenceBytes(collection, {
1080
+ contentHash,
1081
+ bytes: raw,
1082
+ fileName: path.basename(relativePath),
1083
+ kind: "git_diff",
1084
+ sourceKey: diffSourceKey,
1057
1085
  });
1086
+ collection.entries.push(evidenceEntry({
1087
+ context: collection.context,
1088
+ kind: "git_diff",
1089
+ packId: collection.packId,
1090
+ localPath: staged.local_path,
1091
+ relativePath,
1092
+ stagedInPack: staged.staged_in_pack,
1093
+ sourceKey: diffSourceKey,
1094
+ mediaType: "text/x-diff",
1095
+ redactedSummary: gitDiffSummary(target.label, {
1096
+ redacted: Boolean(redaction),
1097
+ truncated: target.truncated,
1098
+ }),
1099
+ redaction,
1100
+ bytes: raw,
1101
+ contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
1102
+ }));
1058
1103
  }
1059
- function evidenceEntryCount(entry) {
1060
- return entry.count ?? 1;
1061
- }
1062
- function countEvidenceEntries(entries, predicate = () => true) {
1063
- return entries.reduce((sum, entry) => sum + (predicate(entry) ? evidenceEntryCount(entry) : 0), 0);
1064
- }
1065
- function makeEvidenceCompleteness(collection, options) {
1066
- const sources = new Set(collection.scanned.keys());
1067
- for (const entry of collection.entries)
1068
- sources.add(entry.kind);
1069
- for (const entry of collection.skipped)
1070
- sources.add(entry.kind);
1071
- for (const entry of collection.reused)
1072
- sources.add(entry.kind);
1073
- for (const entry of collection.truncated)
1074
- sources.add(entry.kind);
1075
- for (const entry of collection.failed)
1076
- sources.add(entry.kind);
1077
- for (const entry of collection.redacted)
1078
- sources.add(entry.kind);
1079
- const sourceCounts = [...sources].sort().map((source) => {
1080
- const skipped = collection.skipped.filter((entry) => entry.kind === source);
1081
- return {
1082
- source,
1083
- scanned_count: collection.scanned.get(source) ?? 0,
1084
- included_count: collection.entries.filter((entry) => entry.kind === source).length,
1085
- skipped_count: countEvidenceEntries(skipped),
1086
- truncated_count: countEvidenceEntries(collection.truncated, (entry) => entry.kind === source),
1087
- deferred_count: countEvidenceEntries(skipped, (entry) => entry.reason.startsWith("deferred_")),
1088
- reused_count: collection.reused.filter((entry) => entry.kind === source).length,
1089
- failed_count: countEvidenceEntries(collection.failed, (entry) => entry.kind === source),
1090
- };
1091
- });
1092
- const totals = sourceCounts.reduce((sum, count) => ({
1093
- scanned_count: sum.scanned_count + count.scanned_count,
1094
- included_count: sum.included_count + count.included_count,
1095
- skipped_count: sum.skipped_count + count.skipped_count,
1096
- truncated_count: sum.truncated_count + count.truncated_count,
1097
- deferred_count: sum.deferred_count + count.deferred_count,
1098
- reused_count: sum.reused_count + count.reused_count,
1099
- failed_count: sum.failed_count + count.failed_count,
1100
- }), {
1101
- scanned_count: 0,
1102
- included_count: 0,
1103
- skipped_count: 0,
1104
- truncated_count: 0,
1105
- deferred_count: 0,
1106
- reused_count: 0,
1107
- failed_count: 0,
1108
- });
1109
- const skipReasonCounts = new Map();
1110
- for (const skipped of collection.skipped) {
1111
- const key = `${skipped.kind}:${skipped.reason}`;
1112
- const existing = skipReasonCounts.get(key);
1113
- if (existing) {
1114
- existing.count += evidenceEntryCount(skipped);
1115
- }
1116
- else {
1117
- skipReasonCounts.set(key, {
1118
- source: skipped.kind,
1119
- reason: skipped.reason,
1120
- count: evidenceEntryCount(skipped),
1121
- });
1122
- }
1123
- }
1124
- const failureReasonCounts = new Map();
1125
- for (const failed of collection.failed) {
1126
- const key = `${failed.kind}:${failed.reason}`;
1127
- const existing = failureReasonCounts.get(key);
1128
- if (existing) {
1129
- existing.count += evidenceEntryCount(failed);
1130
- }
1131
- else {
1132
- failureReasonCounts.set(key, {
1133
- source: failed.kind,
1134
- reason: failed.reason,
1135
- count: evidenceEntryCount(failed),
1136
- });
1137
- }
1138
- }
1139
- const truncationCounts = new Map();
1140
- for (const truncated of collection.truncated) {
1141
- const key = `${truncated.kind}:${truncated.reason}`;
1142
- const existing = truncationCounts.get(key);
1143
- if (existing) {
1144
- existing.count += evidenceEntryCount(truncated);
1145
- existing.observed_bytes = Math.max(existing.observed_bytes ?? 0, truncated.observed_bytes ?? 0);
1146
- existing.included_bytes = Math.max(existing.included_bytes ?? 0, truncated.included_bytes ?? 0);
1147
- }
1148
- else {
1149
- truncationCounts.set(key, {
1150
- source: truncated.kind,
1151
- reason: truncated.reason,
1152
- count: evidenceEntryCount(truncated),
1153
- ...(truncated.max_bytes !== undefined
1154
- ? { max_bytes: truncated.max_bytes }
1155
- : {}),
1156
- ...(truncated.observed_bytes !== undefined
1157
- ? { observed_bytes: truncated.observed_bytes }
1158
- : {}),
1159
- ...(truncated.included_bytes !== undefined
1160
- ? { included_bytes: truncated.included_bytes }
1161
- : {}),
1162
- });
1163
- }
1104
+ function gitDiffSummary(label, state) {
1105
+ if (state.redacted) {
1106
+ return `Raw git ${label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`;
1164
1107
  }
1165
- const redactionCounts = new Map();
1166
- for (const redacted of collection.redacted) {
1167
- const ruleIds = redacted.redaction.rule_counts.map((rule) => rule.rule_id);
1168
- const key = `${redacted.kind}:${redacted.completenessLabel}:${ruleIds.sort().join(",")}`;
1169
- const existing = redactionCounts.get(key);
1170
- if (existing) {
1171
- existing.count += 1;
1172
- existing.rule_ids = [...new Set([...existing.rule_ids, ...ruleIds])].sort();
1173
- }
1174
- else {
1175
- redactionCounts.set(key, {
1176
- source: redacted.kind,
1177
- status: "sanitized",
1178
- mode: redacted.completenessLabel,
1179
- count: 1,
1180
- rule_ids: [...new Set(ruleIds)].sort(),
1181
- });
1182
- }
1108
+ if (state.truncated) {
1109
+ return `Raw git ${label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`;
1183
1110
  }
1184
- const hasGaps = totals.skipped_count > 0 ||
1185
- totals.truncated_count > 0 ||
1186
- totals.deferred_count > 0 ||
1187
- totals.failed_count > 0 ||
1188
- collection.redacted.length > 0 ||
1189
- collection.caps.some((cap) => cap.applied);
1190
- const status = totals.failed_count > 0 &&
1191
- totals.included_count + totals.reused_count === 0
1192
- ? "failed"
1193
- : totals.included_count + totals.reused_count === 0 && !hasGaps
1194
- ? "empty"
1195
- : hasGaps
1196
- ? "partial"
1197
- : "complete";
1198
- return EvidenceCompletenessPayloadSchema.parse({
1199
- schema_version: "evidence-completeness.v1",
1200
- status,
1201
- generated_at: options.finishedAt,
1202
- scan_window: {
1203
- started_at: options.startedAt,
1204
- finished_at: options.finishedAt,
1205
- since_minutes: options.sinceMinutes,
1206
- },
1207
- source_counts: sourceCounts,
1208
- totals,
1209
- caps: collection.caps,
1210
- skip_reasons: [...skipReasonCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
1211
- failure_reasons: [...failureReasonCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
1212
- truncation_markers: [...truncationCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
1213
- redaction_markers: [...redactionCounts.values()].sort((a, b) => `${a.source}:${a.mode}`.localeCompare(`${b.source}:${b.mode}`)),
1214
- notes: totals.failed_count > 0
1215
- ? ["Evidence collection failed; downstream analysis should not infer confidence."]
1216
- : collection.redacted.length > 0
1217
- ? ["Evidence was sanitized before upload; downstream analysis should not treat it as raw-complete."]
1218
- : hasGaps
1219
- ? ["Evidence is incomplete; downstream analysis should lower confidence."]
1220
- : [],
1221
- });
1111
+ return `Raw git ${label} diff preserved locally with env/secret paths excluded.`;
1222
1112
  }
1113
+ // ---------------------------------------------------------------------------
1114
+ // Walking the session store
1115
+ // ---------------------------------------------------------------------------
1116
+ /**
1117
+ * Newest-first `.jsonl` files under these directories, modified since `cutoffMs`.
1118
+ *
1119
+ * Secret-like directory and file names are never descended into or opened;
1120
+ * symlinked duplicates are collapsed by real path so one transcript reachable
1121
+ * two ways is collected once.
1122
+ */
1223
1123
  async function walkJsonlFiles(dir, cutoffMs) {
1224
1124
  const out = [];
1225
1125
  const stack = Array.isArray(dir) ? [...dir] : [dir];
@@ -1246,219 +1146,15 @@ async function walkJsonlFiles(dir, cutoffMs) {
1246
1146
  if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
1247
1147
  continue;
1248
1148
  const stat = await fs.stat(full);
1249
- if (stat.mtimeMs >= cutoffMs) {
1250
- const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
1251
- if (!seen.has(dedupeKey)) {
1252
- seen.add(dedupeKey);
1253
- out.push({ file: full, mtimeMs: stat.mtimeMs });
1254
- }
1255
- }
1149
+ if (stat.mtimeMs < cutoffMs)
1150
+ continue;
1151
+ const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
1152
+ if (seen.has(dedupeKey))
1153
+ continue;
1154
+ seen.add(dedupeKey);
1155
+ out.push({ file: full, mtimeMs: stat.mtimeMs });
1256
1156
  }
1257
1157
  }
1258
1158
  out.sort((a, b) => b.mtimeMs - a.mtimeMs);
1259
1159
  return out.map((entry) => entry.file);
1260
- }
1261
- function makeManifest(options) {
1262
- return {
1263
- schema: "bli.local_raw_evidence_pack.v1",
1264
- pack_id: options.packId,
1265
- created_at: options.context.now.toISOString(),
1266
- work_context_id: options.context.workContextId,
1267
- session_id: options.context.sessionId,
1268
- operator_id: options.context.operatorId,
1269
- operator_label: options.context.operatorLabel,
1270
- repo_label: options.context.repoLabel ?? path.basename(options.context.repoRoot),
1271
- worktree_label: options.context.worktreeLabel,
1272
- active_ticket_id: options.context.activeTicketId ?? null,
1273
- repo_basename: path.basename(options.context.repoRoot),
1274
- branch: options.context.branch,
1275
- storage_bucket: RAW_EVIDENCE_BUCKET,
1276
- raw_policy: {
1277
- raw_prompts: "preserved_private_durable_remote",
1278
- raw_responses: "preserved_private_durable_remote",
1279
- transcripts: "preserved_private_durable_remote",
1280
- claude_transcripts: "preserved_private_durable_remote",
1281
- tool_payloads: "preserved_private_durable_remote",
1282
- git_diffs: "preserved_private_durable_remote_env_secret_paths_excluded",
1283
- agent_image_attachments: "preserved_private_durable_remote_explicit_agent_session_attachment_only",
1284
- env_files: "never_read",
1285
- stdout: "manifest_only_no_raw_content",
1286
- },
1287
- files: options.entries.map(redactManifestEntry),
1288
- skipped: options.skipped,
1289
- redacted: options.redacted,
1290
- reused: options.reused,
1291
- };
1292
- }
1293
- function evidenceEntry(options) {
1294
- const digest = sha256(options.bytes);
1295
- const objectKey = remoteObjectKey({
1296
- context: options.context,
1297
- packId: options.packId,
1298
- relativePath: options.relativePath,
1299
- contentAddress: options.contentAddress,
1300
- });
1301
- return {
1302
- kind: options.kind,
1303
- local_path: options.localPath,
1304
- relative_path: options.relativePath,
1305
- object_key: objectKey,
1306
- content_hash_sha256: digest,
1307
- byte_size: options.bytes.byteLength,
1308
- media_type: options.mediaType,
1309
- redacted_summary: options.redactedSummary,
1310
- ...(options.redaction ? { redaction: options.redaction } : {}),
1311
- codex_session_id: options.codexSessionId ?? null,
1312
- ...(options.artifactMetadata
1313
- ? {
1314
- artifact_metadata: {
1315
- ...options.artifactMetadata,
1316
- raw_evidence_pointer_id: objectKey,
1317
- storage_bucket: RAW_EVIDENCE_BUCKET,
1318
- object_key: objectKey,
1319
- content_hash_sha256: digest,
1320
- byte_size: options.bytes.byteLength,
1321
- },
1322
- }
1323
- : {}),
1324
- };
1325
- }
1326
- function redactManifestEntry(entry) {
1327
- return {
1328
- kind: entry.kind,
1329
- relative_path: entry.relative_path,
1330
- object_key: entry.object_key,
1331
- content_hash_sha256: entry.content_hash_sha256,
1332
- byte_size: entry.byte_size,
1333
- media_type: entry.media_type,
1334
- redacted_summary: entry.redacted_summary,
1335
- ...(entry.redaction ? { redaction: entry.redaction } : {}),
1336
- ...(entry.artifact_metadata
1337
- ? { artifact_metadata: entry.artifact_metadata }
1338
- : {}),
1339
- };
1340
- }
1341
- function pointerFromEntry(entry) {
1342
- return {
1343
- raw_evidence_pointer_id: entry.object_key,
1344
- privacy_classification: "remote_durable_raw_evidence",
1345
- retention_policy: {
1346
- mode: RAW_EVIDENCE_RETENTION_MODE,
1347
- privacy_classification: "remote_durable_raw_evidence",
1348
- },
1349
- storage_scope: "remote_object",
1350
- storage_bucket: RAW_EVIDENCE_BUCKET,
1351
- object_key: entry.object_key,
1352
- content_hash_sha256: entry.content_hash_sha256,
1353
- byte_size: entry.byte_size,
1354
- media_type: entry.media_type,
1355
- redacted_summary: entry.redacted_summary,
1356
- ...(entry.redaction ? { redaction: entry.redaction } : {}),
1357
- };
1358
- }
1359
- /**
1360
- * Raw evidence keys start with human-readable context, then end in immutable
1361
- * content addresses or pack-relative manifest paths. The local cursor reuses
1362
- * prior content hashes across syncs; the readable date/session folders are for
1363
- * operator debugging and incident response.
1364
- */
1365
- function remoteObjectKey(options) {
1366
- const namespace = readableEvidenceNamespace(options.context);
1367
- if (options.contentAddress) {
1368
- return posixPath([
1369
- ...namespace,
1370
- options.contentAddress,
1371
- ]);
1372
- }
1373
- return posixPath([
1374
- ...namespace,
1375
- options.packId,
1376
- options.relativePath,
1377
- ]);
1378
- }
1379
- function readableEvidenceNamespace(context) {
1380
- return [
1381
- "operators",
1382
- operatorSlug(context),
1383
- "repos",
1384
- readableKeySegment(context.repoLabel ?? path.basename(context.repoRoot), "repo"),
1385
- "worktrees",
1386
- readableKeySegment(context.worktreeLabel ?? path.basename(context.repoRoot), "worktree"),
1387
- "tickets",
1388
- readableKeySegment(context.activeTicketId ?? "unbound", "unbound", {
1389
- lowercase: false,
1390
- }),
1391
- "dates",
1392
- context.now.toISOString().slice(0, 10),
1393
- "sessions",
1394
- readableKeySegment(context.sessionId, "session"),
1395
- "ids",
1396
- safeKeySegment(context.operatorId),
1397
- safeKeySegment(context.workContextId),
1398
- ];
1399
- }
1400
- function operatorSlug(context) {
1401
- const labelBeforeDomain = (context.operatorLabel ?? context.operatorId)
1402
- .split("@", 1)[0]
1403
- .trim();
1404
- const readable = readableKeySegment(labelBeforeDomain, "operator");
1405
- return `${readable}-${shortHash(context.operatorId).slice(0, 6)}`;
1406
- }
1407
- function readableKeySegment(value, fallback, options = {}) {
1408
- const base = options.lowercase === false ? value : value.toLowerCase();
1409
- const slug = base
1410
- .trim()
1411
- .replace(/[^A-Za-z0-9._-]+/g, "-")
1412
- .replace(/^-+|-+$/g, "")
1413
- .replace(/-{2,}/g, "-")
1414
- .slice(0, 80);
1415
- return slug || fallback;
1416
- }
1417
- function posixPath(parts) {
1418
- return parts.join("/").replace(/\\/g, "/").replace(/\/+/g, "/");
1419
- }
1420
- function rawEvidencePackId(context) {
1421
- const material = [
1422
- context.workContextId,
1423
- context.sessionId,
1424
- context.now.toISOString(),
1425
- ].join(":");
1426
- return `${context.workContextId}-${shortHash(material)}`;
1427
- }
1428
- function shortHash(value) {
1429
- return crypto.createHash("sha256").update(value, "utf8").digest("hex").slice(0, 12);
1430
- }
1431
- /**
1432
- * Object keys must satisfy the server's key pattern; ids derived from file
1433
- * content fall back to a hash rather than failing the whole upload batch.
1434
- */
1435
- function safeKeySegment(value) {
1436
- return /^[A-Za-z0-9._-]{1,80}$/.test(value) ? value : shortHash(value);
1437
- }
1438
- function sha256(value) {
1439
- return crypto.createHash("sha256").update(value).digest("hex");
1440
- }
1441
- function withRedactionContentMetadata(metadata, options) {
1442
- if (!metadata) {
1443
- throw new Error("redacted evidence is missing redaction metadata");
1444
- }
1445
- return {
1446
- ...metadata,
1447
- original_content_hash_sha256: sha256(options.originalBytes),
1448
- sanitized_content_hash_sha256: sha256(options.sanitizedBytes),
1449
- original_byte_size: options.originalBytes.byteLength,
1450
- sanitized_byte_size: options.sanitizedBytes.byteLength,
1451
- };
1452
- }
1453
- async function ensurePrivateDir(dir) {
1454
- await fs.mkdir(dir, { recursive: true, mode: 0o700 });
1455
- await chmodPrivate(dir, 0o700);
1456
- }
1457
- async function chmodPrivate(target, mode) {
1458
- if (process.platform === "win32")
1459
- return;
1460
- await fs.chmod(target, mode).catch(() => undefined);
1461
- }
1462
- function isSecretLikePath(value) {
1463
- return SECRET_FILE_SEGMENT_PATTERN.test(value);
1464
1160
  }