@bli-cockpit/cli 0.2.49 → 0.2.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/adapters/raw-evidence-claude-reader.js +108 -0
  2. package/dist/adapters/raw-evidence-codex-reader.js +147 -0
  3. package/dist/adapters/raw-evidence-collection-state.js +199 -0
  4. package/dist/adapters/raw-evidence-facts.js +338 -0
  5. package/dist/adapters/raw-evidence-git-diff-reader.js +187 -0
  6. package/dist/adapters/raw-evidence-image-reader.js +107 -0
  7. package/dist/adapters/raw-evidence-sanitize.js +56 -0
  8. package/dist/adapters/raw-evidence-transcript-file.js +182 -0
  9. package/dist/adapters/raw-evidence.js +63 -1183
  10. package/dist/commands/backfill-batches.js +34 -0
  11. package/dist/commands/backfill-candidates.js +54 -0
  12. package/dist/commands/backfill-checkpoint.js +101 -0
  13. package/dist/commands/backfill-command-line.js +70 -0
  14. package/dist/commands/backfill-evidence-outcomes.js +104 -0
  15. package/dist/commands/backfill-issues.js +265 -0
  16. package/dist/commands/backfill-output.js +75 -0
  17. package/dist/commands/backfill-plan.js +71 -0
  18. package/dist/commands/backfill-reasons.js +107 -0
  19. package/dist/commands/backfill-report.js +298 -0
  20. package/dist/commands/backfill-result.js +150 -0
  21. package/dist/commands/backfill-scan.js +274 -0
  22. package/dist/commands/backfill-scope.js +114 -0
  23. package/dist/commands/backfill-session-report.js +145 -0
  24. package/dist/commands/backfill-types.js +1 -0
  25. package/dist/commands/backfill-upload.js +212 -0
  26. package/dist/commands/backfill.js +41 -1961
  27. package/dist/commands/doctor.js +57 -0
  28. package/dist/commands/jarvis-trace.js +184 -0
  29. package/dist/commands/jarvis.js +144 -4
  30. package/dist/commands/local-args-collector.js +26 -0
  31. package/dist/commands/local-args-tower.js +21 -0
  32. package/dist/commands/local-args.js +3 -1
  33. package/dist/commands/local-help.js +19 -2
  34. package/dist/commands/local.js +3 -0
  35. package/dist/commands/memory-install-claude.js +294 -0
  36. package/dist/commands/memory-install-codex.js +205 -0
  37. package/dist/commands/memory-install-contract.js +231 -0
  38. package/dist/commands/memory-install-files.js +63 -0
  39. package/dist/commands/memory-install-skills.js +121 -0
  40. package/dist/commands/memory-install-toml.js +265 -0
  41. package/dist/commands/memory-install.js +378 -0
  42. package/dist/commands/public-root.js +1 -1
  43. package/dist/commands/sync-followups.js +105 -0
  44. package/dist/commands/sync.js +7 -1
  45. package/dist/local-state-attributed-target.js +75 -0
  46. package/dist/local-state-config.js +147 -0
  47. package/dist/local-state-files.js +59 -0
  48. package/dist/local-state-identity.js +73 -0
  49. package/dist/local-state-pairing.js +263 -0
  50. package/dist/local-state-paths.js +61 -0
  51. package/dist/local-state-session.js +68 -0
  52. package/dist/local-state-status.js +163 -0
  53. package/dist/local-state-work-context.js +190 -0
  54. package/dist/local-state.js +34 -848
  55. package/dist/tower-client.js +3 -2
  56. package/dist/tower-stream.js +57 -3
  57. package/package.json +2 -1
@@ -1,35 +1,71 @@
1
- import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, SourceScanResultSchema, } from "@bli-cockpit/telemetry-core";
2
- import crypto from "node:crypto";
3
- import fs from "node:fs/promises";
4
- import os from "node:os";
1
+ /**
2
+ * Collect one repo's raw evidence into a private, content-keyed pack.
3
+ *
4
+ * This is the content guard between "a transcript exists on this laptop" and
5
+ * "these bytes may become durable evidence". Attribution before it only reads
6
+ * metadata; the server commit after it is the second guard.
7
+ *
8
+ * The pass, in order:
9
+ *
10
+ * 1. **Open a staging directory.** The pack's id comes from its content, so it
11
+ * cannot be known yet (BLI-3066). Bytes land in `.staging-<pid>-<rand>`.
12
+ * 2. **Record what attribution already knew** — every session it scanned and
13
+ * every one it could not name, as completeness records, before a single byte
14
+ * is read.
15
+ * 3. **Collect** every source: Codex transcripts, Claude transcripts and their
16
+ * subagent sidecars, images explicitly attached to either, and the two git
17
+ * diffs.
18
+ * 4. **Finish** one of three ways — empty, promoted, or failed — each reporting
19
+ * the same facts, pointers and scan.
20
+ *
21
+ * Three doctrines hold everywhere in this family:
22
+ *
23
+ * - **Nothing gets in the way of collecting a session.** Redaction masks and
24
+ * continues; it never drops a file.
25
+ * - **A failure must name itself.** Every path that declines a file records a
26
+ * reason label in the gap ledger.
27
+ * - **Never succeed silently either.** Staging, reuse, holds and failures all
28
+ * log metadata — ids, sizes, counts, reason labels. Never content, never a
29
+ * local path.
30
+ *
31
+ * Who owns what (BLI-3586 moved the bodies out; every public name is still
32
+ * exported from this file, so no consumer's import changed):
33
+ *
34
+ * | module | owns |
35
+ * | ------------------------------- | --------------------------------------- |
36
+ * | `raw-evidence-collection-state` | what a pass carries, its budget, staging bytes |
37
+ * | `raw-evidence-codex-reader` | Codex transcripts, and the capped fallback walk |
38
+ * | `raw-evidence-claude-reader` | Claude mains, sidecars, the D7/D9 outcomes |
39
+ * | `raw-evidence-image-reader` | images attached to either transcript |
40
+ * | `raw-evidence-git-diff-reader` | the unstaged and staged diffs |
41
+ * | `raw-evidence-transcript-file` | one transcript, end to end |
42
+ * | `raw-evidence-sanitize` | masking, and the receipt it leaves |
43
+ * | `raw-evidence-facts` | the three endings, the pointers, the scan |
44
+ * | `raw-evidence-completeness` | the gap ledger |
45
+ * | `raw-evidence-manifest` | what we say about a file |
46
+ * | `raw-evidence-pack-store` | where the bytes live |
47
+ * | `raw-evidence-keys` | naming |
48
+ * | `raw-evidence-git-diff` | running git |
49
+ * | `raw-evidence-attribution-gaps` | step 2's scan results as ledger records |
50
+ *
51
+ * The readers deliberately share one shape — derive the names, guard, mask and
52
+ * record, hash, check reuse, check budget, then a single `write…ToPack` that
53
+ * puts the bytes down and adds the manifest entry. They do NOT share one
54
+ * implementation, because each pushes a different ledger row and logs a
55
+ * different line, and merging them would change what an operator reads.
56
+ */
5
57
  import path from "node:path";
6
- import { makeSourceAdapterIdentity, } from "./common.js";
7
- import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
8
- import { defaultCodexSessionDirs, } from "./codex-attribution.js";
9
58
  import { recordClaudeAttributionCompleteness, recordCodexAttributionCompleteness, } from "./raw-evidence-attribution-gaps.js";
10
- import { describeError } from "../health-detail.js";
11
- import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject, } from "../raw-evidence-staging.js";
12
- import { isSecretLikePath, safeKeySegment, sha256, shortHash, } from "./raw-evidence-keys.js";
13
- import { sanitizeTextEvidenceForUpload, } from "./raw-evidence-sanitize.js";
14
- import { countEvidenceEntries, makeEvidenceCompleteness, markBudgetCapApplied, markCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
15
- import { evidenceEntry, pointerFromEntry, RAW_EVIDENCE_BUCKET, } from "./raw-evidence-manifest.js";
16
- import { chmodPrivate, ensurePrivateDir, persistStagingState, promoteStagedPack, stageManifest, } from "./raw-evidence-pack-store.js";
17
- import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, REPO_ROOT_MISSING_REASON, RepoRootMissingError, repoRootExists, runGitDiff, } from "./raw-evidence-git-diff.js";
59
+ import { ensurePrivateDir } from "./raw-evidence-pack-store.js";
60
+ import { DEFAULT_SESSION_LIMIT, DEFAULT_SINCE_MINUTES, newStagingDir, openCollection, } from "./raw-evidence-collection-state.js";
61
+ import { finishWithEmptyPack, finishWithFailedPack, finishWithPromotedPack, } from "./raw-evidence-facts.js";
62
+ import { collectCodexJsonlFiles } from "./raw-evidence-codex-reader.js";
63
+ import { collectClaudeJsonlFiles } from "./raw-evidence-claude-reader.js";
64
+ import { collectGitDiffFiles } from "./raw-evidence-git-diff-reader.js";
18
65
  // Re-exported so every consumer keeps importing from `adapters/raw-evidence`.
19
66
  export { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./raw-evidence-manifest.js";
20
67
  export { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
21
- const DEFAULT_SINCE_MINUTES = 24 * 60;
22
- const DEFAULT_SESSION_LIMIT = 50;
23
- // Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
24
- // transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
25
- // so 2 GiB leaves room for several files without starving the sync; overflow
26
- // still defers and converges.
27
- export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
28
- export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
29
- const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;
30
- // ---------------------------------------------------------------------------
31
- // The pass
32
- // ---------------------------------------------------------------------------
68
+ export { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET, } from "./raw-evidence-collection-state.js";
33
69
  /**
34
70
  * Decide which bytes on this laptop are allowed to become durable evidence for
35
71
  * one repo, and put exactly those into one private, content-keyed pack.
@@ -64,15 +100,6 @@ export async function collectRawEvidencePack(context, options) {
64
100
  return await finishWithFailedPack(collection, { ...run, error });
65
101
  }
66
102
  }
67
- /**
68
- * Staging first, promotion second (BLI-3066). The pack id cannot be known until
69
- * the content is, so bytes land in a private per-attempt directory, and that
70
- * directory is then renamed to its content-keyed name — or dropped, when an
71
- * identical pack is already there.
72
- */
73
- function newStagingDir(rawEvidenceRoot) {
74
- return path.join(rawEvidenceRoot, `.staging-${process.pid}-${crypto.randomUUID().slice(0, 8)}`);
75
- }
76
103
  /**
77
104
  * Every kind of evidence this repo can offer, in the order it is read. Codex and
78
105
  * Claude transcripts are each switchable off by the caller; the git diffs are
@@ -93,414 +120,6 @@ async function collectEverySource(collection, options, sinceMinutes) {
93
120
  }
94
121
  await collectGitDiffFiles(collection, options.repoRoot);
95
122
  }
96
- async function openCollection(context, options, places) {
97
- const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
98
- const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
99
- const staging = await readRawEvidenceStagingState(options.stateDir);
100
- return {
101
- context,
102
- filesDir: path.join(places.stagingDir, "files"),
103
- rawEvidenceRoot: places.rawEvidenceRoot,
104
- packId: "",
105
- staging,
106
- heldSources: collectionHeldSources(staging, context.now, options),
107
- stagedReusedCount: 0,
108
- stagedNewCount: 0,
109
- deliveryHeldCount: 0,
110
- entries: [],
111
- skipped: [],
112
- truncated: [],
113
- failed: [],
114
- redacted: [],
115
- reused: [],
116
- scanned: new Map(),
117
- caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
118
- skipContentHashes: options.skipContentHashes ?? new Set(),
119
- budget: options.budget ?? {
120
- remainingBytes: byteBudget,
121
- remainingObjects: objectBudget,
122
- },
123
- index: { value: 0 },
124
- };
125
- }
126
- /**
127
- * Which sources this pass refuses to even read because their delivery is inside
128
- * a backoff window — none of them, when a person asked for this pass.
129
- *
130
- * The bypass is logged rather than assumed: an operator retry that quietly
131
- * ignored a hold would be as unreadable as the hold that quietly blocked it.
132
- */
133
- function collectionHeldSources(staging, now, options) {
134
- const held = heldSourceKeys(staging, now);
135
- if (deliveryBackoffApplies(options.deliveryMode))
136
- return held;
137
- if (held.size > 0) {
138
- console.error("[raw-evidence] delivery backoff bypassed for operator retry", JSON.stringify({
139
- reason: DELIVERY_BACKOFF_BYPASS_REASON,
140
- source_count: held.size,
141
- }));
142
- }
143
- return new Set();
144
- }
145
- /** Every cap this pass could hit, declared up front and flipped when applied. */
146
- function startingCaps(options) {
147
- return [
148
- {
149
- source: "raw_evidence",
150
- cap_type: "byte_budget",
151
- limit: options.byteBudget,
152
- observed: options.budget?.remainingBytes ?? options.byteBudget,
153
- applied: false,
154
- },
155
- {
156
- source: "raw_evidence",
157
- cap_type: "object_budget",
158
- limit: options.objectBudget,
159
- observed: options.budget?.remainingObjects ?? options.objectBudget,
160
- applied: false,
161
- },
162
- {
163
- source: "git_diff",
164
- cap_type: "max_bytes_per_diff",
165
- limit: MAX_GIT_DIFF_BYTES,
166
- applied: false,
167
- },
168
- {
169
- source: "git_diff",
170
- cap_type: "timeout_ms",
171
- limit: GIT_DIFF_TIMEOUT_MS,
172
- applied: false,
173
- },
174
- {
175
- source: "codex_jsonl",
176
- cap_type: "max_file_bytes",
177
- limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
178
- applied: false,
179
- },
180
- {
181
- source: "claude_jsonl",
182
- cap_type: "max_file_bytes",
183
- limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
184
- applied: false,
185
- },
186
- {
187
- source: "claude_jsonl_sidecar",
188
- cap_type: "max_file_bytes",
189
- limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
190
- applied: false,
191
- },
192
- ];
193
- }
194
- // ---------------------------------------------------------------------------
195
- // The three ways a pass ends
196
- // ---------------------------------------------------------------------------
197
- /**
198
- * Nothing collected: drop the staging directory instead of leaving an empty
199
- * pack behind. This used to leave one empty `work-*` dir per sync.
200
- */
201
- async function finishWithEmptyPack(collection, options, run) {
202
- await discardStagingDir(run.stagingDir);
203
- collection.packId = contentKeyedRawEvidencePackId({
204
- workContextId: collection.context.workContextId,
205
- contentHashes: [],
206
- });
207
- await persistStagingState(options.stateDir, collection.staging, collection.context.now.toISOString());
208
- const facts = makeEmptyPackFacts(collection, run.window);
209
- return {
210
- facts,
211
- scan: makeRawEvidenceScan({
212
- context: collection.context,
213
- startedAt: run.window.startedAt,
214
- status: "partial",
215
- facts,
216
- }),
217
- };
218
- }
219
- async function finishWithPromotedPack(collection, options, run) {
220
- const context = collection.context;
221
- const entries = collection.entries;
222
- // The pack is named by what is in it, never by when it was made. Identical
223
- // content on the next sync resolves to the identical directory, which is
224
- // the whole fix for BLI-3066.
225
- const packId = contentKeyedRawEvidencePackId({
226
- workContextId: context.workContextId,
227
- contentHashes: entries.map((entry) => entry.content_hash_sha256),
228
- });
229
- collection.packId = packId;
230
- const promotion = await promoteStagedPack({
231
- rawEvidenceRoot: collection.rawEvidenceRoot,
232
- stagingDir: run.stagingDir,
233
- packId,
234
- workContextId: context.workContextId,
235
- entries,
236
- });
237
- rebaseStagedEntriesOntoPack(collection, promotion.evidenceDir);
238
- const manifestPath = await addManifestToPack(collection, promotion, packId);
239
- await persistStagingState(options.stateDir, collection.staging, context.now.toISOString());
240
- logPackStaged(collection, promotion, packId);
241
- const facts = makePromotedPackFacts(collection, {
242
- packId,
243
- manifestPath,
244
- promotion,
245
- window: run.window,
246
- });
247
- return {
248
- facts,
249
- scan: makeRawEvidenceScan({
250
- context,
251
- startedAt: run.window.startedAt,
252
- // A pack of nothing but its own manifest carries no evidence, so it
253
- // reports partial however cleanly it was written.
254
- status: entries.length > 1 ? "ok" : "partial",
255
- facts,
256
- }),
257
- };
258
- }
259
- /** A pack with files: the same shell as an empty one, plus what is in it. */
260
- function makePromotedPackFacts(collection, place) {
261
- const entries = collection.entries;
262
- return {
263
- ...packFactsShell(collection, {
264
- packId: place.packId,
265
- manifestPath: place.manifestPath,
266
- evidenceDir: place.promotion.evidenceDir,
267
- window: place.window,
268
- }),
269
- stage_state: place.promotion.state,
270
- file_count: entries.length,
271
- byte_size: totalByteSize(entries),
272
- content_kinds: [...new Set(entries.map((entry) => entry.kind))],
273
- pointers: entries.map(pointerFromEntry),
274
- upload_files: entries.map(uploadFileFromEntry),
275
- };
276
- }
277
- /**
278
- * Write the manifest and make it an object of the pack like any other file, so
279
- * the uploader has one list to walk. Returns where it landed.
280
- */
281
- async function addManifestToPack(collection, promotion, packId) {
282
- const context = collection.context;
283
- const manifestPath = path.join(promotion.evidenceDir, "manifest.json");
284
- const manifestBytes = await stageManifest({
285
- context,
286
- packId,
287
- manifestPath,
288
- entries: collection.entries,
289
- skipped: collection.skipped,
290
- redacted: collection.redacted,
291
- reused: collection.reused,
292
- reusePack: promotion.state === "reused",
293
- });
294
- collection.entries.push(evidenceEntry({
295
- context,
296
- kind: "manifest",
297
- packId,
298
- localPath: manifestPath,
299
- relativePath: "manifest.json",
300
- mediaType: "application/json",
301
- redactedSummary: "Local raw evidence pack manifest.",
302
- bytes: manifestBytes,
303
- // The manifest carries its own content hash in its key, exactly like
304
- // every other object in the pack (BLI-3552). Without it the key was
305
- // `…/<packId>/manifest.json`, and a pack id is a function of the OTHER
306
- // files' hashes — not of the manifest's bytes, which also carry
307
- // `created_at`, the ordering-dependent `files[].relative_path`, `branch`,
308
- // and the skipped/redacted/reused ledgers. So the same key named
309
- // different bytes whenever the pack directory was not adopted verbatim
310
- // (a refill, a pruned or wiped state dir, a machine that had never seen
311
- // the pack). `begin` then answered `hash_mismatch_committed_object` on
312
- // every sync forever, because the new manifest is stable and the old one
313
- // is durable. `stageManifest`'s byte-stability trick still stands; it is
314
- // now a nice-to-have rather than the only thing between us and a loop.
315
- contentAddress: `manifest/${sha256(manifestBytes).slice(0, 16)}.json`,
316
- }));
317
- return manifestPath;
318
- }
319
- /** The success line. A sync that staged nothing new still has to say so. */
320
- function logPackStaged(collection, promotion, packId) {
321
- console.error("[raw-evidence] pack staged", JSON.stringify({
322
- pack_id: packId,
323
- stage_state: promotion.state,
324
- reason: stageReasonLabel(promotion.state, promotion.priorPackCount),
325
- prior_pack_count: promotion.priorPackCount,
326
- refilled_file_count: promotion.refilledFileCount,
327
- file_count: collection.entries.length,
328
- byte_size: totalByteSize(collection.entries),
329
- staged_new: collection.stagedNewCount,
330
- staged_reused: collection.stagedReusedCount,
331
- delivery_held: collection.deliveryHeldCount,
332
- }));
333
- }
334
- /**
335
- * A crashed pass. The staged-object index is deliberately NOT written: this
336
- * attempt's bookkeeping describes a directory that is about to be deleted.
337
- */
338
- async function finishWithFailedPack(collection, run) {
339
- collection.failed.push({
340
- kind: "raw_evidence",
341
- reason: "collection_failed",
342
- });
343
- // Staging is per-attempt scratch: a crashed pass must not leave a partial
344
- // directory behind to be counted, re-hashed or swept later.
345
- await discardStagingDir(run.stagingDir);
346
- collection.packId =
347
- collection.packId ||
348
- contentKeyedRawEvidencePackId({
349
- workContextId: collection.context.workContextId,
350
- contentHashes: collection.entries.map((entry) => entry.content_hash_sha256),
351
- });
352
- console.error("[raw-evidence] pack collection failed", JSON.stringify({
353
- pack_id: collection.packId,
354
- reason: "collection_failed",
355
- detail: run.error instanceof Error ? run.error.name : typeof run.error,
356
- collected_file_count: collection.entries.length,
357
- staged_new: collection.stagedNewCount,
358
- staged_reused: collection.stagedReusedCount,
359
- }));
360
- const facts = makeEmptyPackFacts(collection, run.window);
361
- return {
362
- facts,
363
- scan: makeRawEvidenceScan({
364
- context: collection.context,
365
- startedAt: run.window.startedAt,
366
- status: "failed",
367
- facts,
368
- }),
369
- };
370
- }
371
- /**
372
- * A pack with no files of its own: the empty pass and the crashed pass report
373
- * the same shape, differing only in the scan status and the reason already
374
- * logged. `evidence_completeness` still carries every gap this pass recorded.
375
- */
376
- function makeEmptyPackFacts(collection, scanWindow) {
377
- const evidenceDir = path.join(collection.rawEvidenceRoot, collection.packId);
378
- return {
379
- ...packFactsShell(collection, {
380
- packId: collection.packId,
381
- manifestPath: path.join(evidenceDir, "manifest.json"),
382
- evidenceDir,
383
- window: scanWindow,
384
- }),
385
- stage_state: "empty",
386
- file_count: 0,
387
- byte_size: 0,
388
- content_kinds: [],
389
- pointers: [],
390
- upload_files: [],
391
- };
392
- }
393
- /** The counts every ending reports identically, however the pass ended. */
394
- function packFactsShell(collection, place) {
395
- return {
396
- pack_id: place.packId,
397
- manifest_path: place.manifestPath,
398
- evidence_dir: place.evidenceDir,
399
- storage_bucket: RAW_EVIDENCE_BUCKET,
400
- skipped_count: countEvidenceEntries(collection.skipped),
401
- sanitized_count: collection.redacted.length,
402
- reused_count: collection.reused.length,
403
- staged_reused_count: collection.stagedReusedCount,
404
- staged_new_count: collection.stagedNewCount,
405
- delivery_held_count: collection.deliveryHeldCount,
406
- deferred_byte_budget_count: countDeferred(collection, "deferred_byte_budget"),
407
- deferred_object_budget_count: countDeferred(collection, "deferred_object_budget"),
408
- evidence_completeness: makeEvidenceCompleteness(collection, {
409
- startedAt: place.window.startedAt,
410
- finishedAt: place.window.finishedAt(),
411
- sinceMinutes: place.window.sinceMinutes,
412
- }),
413
- reused: collection.reused,
414
- };
415
- }
416
- function countDeferred(collection, reason) {
417
- return countEvidenceEntries(collection.skipped, (entry) => entry.reason === reason);
418
- }
419
- function totalByteSize(entries) {
420
- return entries.reduce((sum, entry) => sum + entry.byte_size, 0);
421
- }
422
- function uploadFileFromEntry(entry) {
423
- return {
424
- pointer: pointerFromEntry(entry),
425
- local_path: entry.local_path,
426
- kind: entry.kind,
427
- codex_session_id: entry.codex_session_id ?? null,
428
- ...(entry.artifact_metadata
429
- ? { artifact_metadata: entry.artifact_metadata }
430
- : {}),
431
- };
432
- }
433
- function stageReasonLabel(state, priorPackCount) {
434
- if (state === "reused")
435
- return "staged_reused";
436
- if (state === "restaged_incomplete")
437
- return "restaged_incomplete";
438
- return priorPackCount > 0 ? "restaged_content_changed" : "staged_new";
439
- }
440
- /**
441
- * Point every entry this pass staged at its home in the promoted pack, and
442
- * remember the content hash so the next sync can adopt the copy instead of
443
- * writing it again. Entries adopted from another pack keep their path.
444
- */
445
- function rebaseStagedEntriesOntoPack(collection, evidenceDir) {
446
- for (const entry of collection.entries) {
447
- if (!entry.staged_in_pack)
448
- continue;
449
- entry.local_path = path.join(evidenceDir, "files", path.basename(entry.local_path));
450
- recordStagedObject(collection.staging, entry.content_hash_sha256, {
451
- pack_id: collection.packId,
452
- relative_path: `files/${path.basename(entry.local_path)}`,
453
- byte_size: entry.byte_size,
454
- source_key: entry.source_key,
455
- staged_at: collection.context.now.toISOString(),
456
- });
457
- }
458
- }
459
- async function discardStagingDir(stagingDir) {
460
- await fs
461
- .rm(stagingDir, { recursive: true, force: true })
462
- .catch(() => undefined);
463
- }
464
- function makeRawEvidenceScan(options) {
465
- return SourceScanResultSchema.parse({
466
- adapter: makeSourceAdapterIdentity("collector_runtime", "raw-evidence-pack"),
467
- work_context_id: options.context.workContextId,
468
- status: options.status,
469
- started_at: options.startedAt,
470
- finished_at: options.context.now.toISOString(),
471
- events: [
472
- {
473
- source_event_id: `raw-evidence-pack:${options.facts.pack_id}`,
474
- event_type: "raw_evidence_pack_written",
475
- occurred_at: options.context.now.toISOString(),
476
- redaction: {
477
- privacy_classification: "remote_durable_raw_evidence",
478
- redaction_status: "raw_remote_durable",
479
- redacted_fields: ["local_path"],
480
- raw_evidence_pointer_ids: options.facts.pointers.map((pointer) => pointer.raw_evidence_pointer_id),
481
- redacted_summary: "Raw prompt/response/tool/session/diff evidence harvested to a private durable remote evidence pack.",
482
- },
483
- raw_evidence_pointers: options.facts.pointers,
484
- },
485
- ],
486
- diagnostic_labels: [
487
- `pack_id:${options.facts.pack_id}`,
488
- `files:${options.facts.file_count}`,
489
- `bytes:${options.facts.byte_size}`,
490
- `skipped:${options.facts.skipped_count}`,
491
- `reused:${options.facts.reused_count}`,
492
- `stage_state:${options.facts.stage_state}`,
493
- `staged_new:${options.facts.staged_new_count}`,
494
- `staged_reused:${options.facts.staged_reused_count}`,
495
- `delivery_held:${options.facts.delivery_held_count}`,
496
- `completeness:${options.facts.evidence_completeness.status}`,
497
- `truncated:${options.facts.evidence_completeness.totals.truncated_count}`,
498
- `deferred:${options.facts.evidence_completeness.totals.deferred_count}`,
499
- `failed:${options.facts.evidence_completeness.totals.failed_count}`,
500
- ...options.facts.content_kinds.map((kind) => `kind:${kind}`),
501
- ],
502
- });
503
- }
504
123
  // ---------------------------------------------------------------------------
505
124
  // What attribution already knew
506
125
  // ---------------------------------------------------------------------------
@@ -516,743 +135,4 @@ function recordAttributionCompleteness(collection, options) {
516
135
  if (options.claudeAttributionScan) {
517
136
  recordClaudeAttributionCompleteness(collection, options.claudeAttributionScan, new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []));
518
137
  }
519
- }
520
- // ---------------------------------------------------------------------------
521
- // Codex transcripts
522
- // ---------------------------------------------------------------------------
523
- async function collectCodexJsonlFiles(collection, options) {
524
- for (const candidate of await chooseCodexCandidates(collection, options)) {
525
- recordScanned(collection, "codex_jsonl");
526
- await collectOneCodexSession(collection, candidate);
527
- }
528
- }
529
- /**
530
- * Which Codex transcripts this pass will look at, and the record of how many it
531
- * had to leave out. Attribution's own list is never trimmed — the session limit
532
- * only guards the fallback walk, which can turn up every session on the machine.
533
- */
534
- async function chooseCodexCandidates(collection, options) {
535
- const attributed = Boolean(options.codexSessionFiles);
536
- const candidates = options.codexSessionFiles
537
- ? options.codexSessionFiles.map((file) => ({
538
- filePath: file.local_path,
539
- codexSessionId: file.codex_session_id,
540
- }))
541
- : (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
542
- collection.caps.push({
543
- source: "codex_jsonl",
544
- cap_type: "session_limit",
545
- limit: options.limit,
546
- observed: candidates.length,
547
- applied: !attributed && candidates.length > options.limit,
548
- });
549
- return attributed ? candidates : candidates.slice(0, options.limit);
550
- }
551
- /** The transcript, then the images it explicitly attached — never the reverse. */
552
- async function collectOneCodexSession(collection, candidate) {
553
- const codexSessionId = candidate.codexSessionId ?? shortHash(candidate.filePath);
554
- const transcriptAccepted = await collectOneEvidenceFile(collection, {
555
- filePath: candidate.filePath,
556
- kind: "codex_jsonl",
557
- sessionId: codexSessionId,
558
- mediaType: "application/jsonl",
559
- // Guard before read/toString: Codex files can exceed the buffered commit
560
- // ceiling just like Claude mains.
561
- maxFileBytes: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
562
- redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
563
- contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
564
- });
565
- if (!transcriptAccepted)
566
- return;
567
- await collectAgentImagesFromTranscript(collection, {
568
- filePath: candidate.filePath,
569
- source: "codex",
570
- sessionId: codexSessionId,
571
- kind: "codex_image_attachment",
572
- contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
573
- });
574
- }
575
- /** The unattributed fallback: every recent Codex transcript on this machine. */
576
- function walkRecentCodexJsonlFiles(collection, options) {
577
- const dirs = options.sessionsDirs ??
578
- (options.sessionsDir
579
- ? [options.sessionsDir]
580
- : defaultCodexSessionDirs(os.homedir()));
581
- return walkJsonlFiles(dirs, collection.context.now.getTime() - options.sinceMinutes * 60 * 1000);
582
- }
583
- // ---------------------------------------------------------------------------
584
- // Claude transcripts
585
- // ---------------------------------------------------------------------------
586
- async function collectClaudeJsonlFiles(collection, sessions) {
587
- recordScanned(collection, "claude_jsonl", sessions.reduce((count, session) => count + 1 + session.sidecar_files.length, 0));
588
- for (const session of sessions) {
589
- await collectOneClaudeSession(collection, session);
590
- }
591
- }
592
- async function collectOneClaudeSession(collection, session) {
593
- const sessionId = session.claude_session_id;
594
- const mainOutcome = await collectClaudeMainFile(collection, session);
595
- if (mainOutcome === "collected" || mainOutcome === "damped_reuse") {
596
- await collectAgentImagesFromTranscript(collection, {
597
- filePath: session.local_path,
598
- source: "claude_code",
599
- sessionId,
600
- kind: "claude_image_attachment",
601
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
602
- });
603
- }
604
- for (const sidecar of session.sidecar_files) {
605
- await collectOneClaudeSidecar(collection, session, sidecar.local_path);
606
- }
607
- }
608
- /**
609
- * What happens to a Claude main file, as a decision table:
610
- *
611
- * | condition | main bytes | outcome |
612
- * | --------------------- | ------------------------- | ------------------- |
613
- * | `main_file_oversized` | skipped `file_too_large` | `skipped_too_large` |
614
- * | `skip_main` (D9) | not re-collected | `damped_reuse` |
615
- * | otherwise | collected, or named skip | `collected` / `not_collected` |
616
- *
617
- * Its images are collected for every outcome except `skipped_too_large` and
618
- * `not_collected` — the caller decides that, this function only reports.
619
- *
620
- * D7: an oversized main was attributed via a streamed read but its bytes are
621
- * never uploaded (server commit assembles in memory). Its sidecars still
622
- * collect. D9 damped: the prior durable copy is still good enough, and the
623
- * session reports `reused_existing` from cursor state, so no skip is recorded.
624
- */
625
- async function collectClaudeMainFile(collection, session) {
626
- if (session.main_file_oversized) {
627
- collection.skipped.push({
628
- kind: "claude_jsonl",
629
- label: path.basename(session.local_path),
630
- reason: "file_too_large",
631
- });
632
- return "skipped_too_large";
633
- }
634
- if (session.skip_main)
635
- return "damped_reuse";
636
- const accepted = await collectOneEvidenceFile(collection, {
637
- filePath: session.local_path,
638
- kind: "claude_jsonl",
639
- sessionId: session.claude_session_id,
640
- mediaType: "application/jsonl",
641
- // Re-check size at collection: a main that grew past the cap between
642
- // attribution and collection is an honest file_too_large skip, not an
643
- // upload_failed at the chunk client.
644
- maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
645
- redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
646
- contentAddress: (hash16) => `claude/${safeKeySegment(session.claude_session_id)}/${hash16}.jsonl`,
647
- });
648
- return accepted ? "collected" : "not_collected";
649
- }
650
- async function collectOneClaudeSidecar(collection, session, sidecarPath) {
651
- const sessionId = session.claude_session_id;
652
- const stem = path.basename(sidecarPath).replace(/\.jsonl$/i, "");
653
- const safeSidecarId = isSecretLikePath(stem)
654
- ? "redacted-file-name"
655
- : safeKeySegment(stem);
656
- const sidecarAccepted = await collectOneEvidenceFile(collection, {
657
- filePath: sidecarPath,
658
- kind: "claude_jsonl_sidecar",
659
- sessionId,
660
- mediaType: "application/jsonl",
661
- maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
662
- redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
663
- contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
664
- });
665
- if (!sidecarAccepted)
666
- return;
667
- await collectAgentImagesFromTranscript(collection, {
668
- filePath: sidecarPath,
669
- source: "claude_code",
670
- sessionId,
671
- sidecarId: safeSidecarId,
672
- kind: "claude_image_attachment",
673
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
674
- });
675
- }
676
- async function collectAgentImagesFromTranscript(collection, options) {
677
- const result = await collectAgentImageEvidenceFromJsonlFile({
678
- filePath: options.filePath,
679
- source: options.source,
680
- sessionId: options.sessionId,
681
- sidecarId: options.sidecarId,
682
- });
683
- recordScanned(collection, options.kind, result.images.length + result.skipped.length);
684
- for (const skipped of result.skipped) {
685
- collection.skipped.push({
686
- kind: options.kind,
687
- label: skipped.label,
688
- reason: skipped.reason,
689
- });
690
- }
691
- for (const image of result.images) {
692
- await collectOneAgentImageFile(collection, {
693
- image,
694
- kind: options.kind,
695
- sessionId: options.sessionId,
696
- contentAddress: options.contentAddress,
697
- });
698
- }
699
- }
700
- async function collectOneAgentImageFile(collection, options) {
701
- const raw = options.image.bytes;
702
- const contentHash = sha256(raw);
703
- const metadata = {
704
- ...options.image.metadata,
705
- content_hash_sha256: contentHash,
706
- byte_size: raw.byteLength,
707
- };
708
- if (collection.skipContentHashes.has(contentHash)) {
709
- collection.reused.push({
710
- kind: options.kind,
711
- label: options.image.label,
712
- content_hash_sha256: contentHash,
713
- codex_session_id: options.sessionId,
714
- artifact_metadata: metadata,
715
- });
716
- return;
717
- }
718
- const deferReason = admitToBudget(collection.budget, raw.byteLength);
719
- if (deferReason) {
720
- markBudgetCapApplied(collection, deferReason);
721
- collection.skipped.push({
722
- kind: options.kind,
723
- label: options.image.label,
724
- reason: deferReason,
725
- });
726
- return;
727
- }
728
- await writeAgentImageToPack(collection, {
729
- ...options,
730
- contentHash,
731
- metadata,
732
- });
733
- }
734
- /** Put an accepted image's bytes on disk and add its manifest entry. */
735
- async function writeAgentImageToPack(collection, image) {
736
- const raw = image.image.bytes;
737
- collection.index.value += 1;
738
- const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-agent-image-${image.contentHash.slice(0, 16)}.${image.image.extension}`);
739
- const imageSourceKey = evidenceSourceKey({
740
- kind: image.kind,
741
- sessionId: image.sessionId,
742
- label: image.image.label,
743
- });
744
- const staged = await stageEvidenceBytes(collection, {
745
- contentHash: image.contentHash,
746
- bytes: raw,
747
- fileName: path.basename(relativePath),
748
- kind: image.kind,
749
- sourceKey: imageSourceKey,
750
- });
751
- collection.entries.push(evidenceEntry({
752
- context: collection.context,
753
- kind: image.kind,
754
- packId: collection.packId,
755
- localPath: staged.local_path,
756
- relativePath,
757
- stagedInPack: staged.staged_in_pack,
758
- sourceKey: imageSourceKey,
759
- mediaType: image.metadata.media_type,
760
- redactedSummary: "Raw image explicitly attached to an agent session, preserved in private durable storage.",
761
- bytes: raw,
762
- codexSessionId: image.sessionId,
763
- contentAddress: image.contentAddress(image.contentHash.slice(0, 16), image.image.extension),
764
- artifactMetadata: image.metadata,
765
- }));
766
- }
767
- // ---------------------------------------------------------------------------
768
- // One transcript, end to end
769
- // ---------------------------------------------------------------------------
770
- /**
771
- * Reads, secret-guards, content-addresses, budget-checks, and copies one
772
- * attributed transcript into the pack.
773
- *
774
- * Returns whether this file's *content* is accounted for, which is not the same
775
- * as "was staged": a transcript whose bytes are already durable remotely
776
- * returns true without staging anything, and its images are still worth
777
- * collecting. Every false return has pushed a named skip first.
778
- */
779
- async function collectOneEvidenceFile(collection, options) {
780
- const names = nameEvidenceFile(options);
781
- const skip = (reason) => {
782
- collection.skipped.push({
783
- kind: options.kind,
784
- label: names.evidenceLabel,
785
- reason,
786
- });
787
- return false;
788
- };
789
- // Delivery backoff is checked BEFORE the file is read. An object whose commit
790
- // has failed repeatedly costs nothing at all this cycle — no read, no hash,
791
- // no copy, no request — and the hold is a named, retryable gap so a held
792
- // session cannot make the sync look clean (BLI-3066).
793
- if (collection.heldSources.has(names.sourceKey)) {
794
- collection.deliveryHeldCount += 1;
795
- skip(DELIVERY_BACKOFF_HOLDING_REASON);
796
- logDeliveryBackoffHold(options.kind, names.sourceKey);
797
- return false;
798
- }
799
- const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
800
- if (read.status === "read_failed")
801
- return skip("file_read_failed");
802
- if (read.status === "too_large") {
803
- markCapApplied(collection, options.kind, "max_file_bytes");
804
- return skip("file_too_large");
805
- }
806
- const sanitized = maskSecretsInTranscript(collection, {
807
- raw: read.bytes,
808
- kind: options.kind,
809
- evidenceLabel: names.evidenceLabel,
810
- secretLikeFileName: names.secretLikeFileName,
811
- });
812
- const contentHash = sha256(sanitized.bytes);
813
- if (collection.skipContentHashes.has(contentHash)) {
814
- collection.reused.push({
815
- kind: options.kind,
816
- label: names.evidenceLabel,
817
- content_hash_sha256: contentHash,
818
- codex_session_id: options.sessionId,
819
- });
820
- return true;
821
- }
822
- const deferReason = admitToBudget(collection.budget, sanitized.bytes.byteLength);
823
- if (deferReason) {
824
- markBudgetCapApplied(collection, deferReason);
825
- return skip(deferReason);
826
- }
827
- await stageOneTranscript(collection, {
828
- ...options,
829
- names,
830
- sanitized,
831
- contentHash,
832
- });
833
- return true;
834
- }
835
- function nameEvidenceFile(options) {
836
- const fileName = path.basename(options.filePath);
837
- const secretLikeFileName = isSecretLikePath(fileName);
838
- return {
839
- secretLikeFileName,
840
- evidenceLabel: secretLikeFileName ? "[REDACTED_FILE_NAME]" : fileName,
841
- packedFileName: secretLikeFileName ? "redacted-file.jsonl" : fileName,
842
- sourceKey: evidenceSourceKey({
843
- kind: options.kind,
844
- sessionId: options.sessionId,
845
- sourcePath: options.filePath,
846
- }),
847
- };
848
- }
849
- function logDeliveryBackoffHold(kind, sourceKey) {
850
- console.error("[raw-evidence] delivery backoff holding source", JSON.stringify({
851
- reason: DELIVERY_BACKOFF_HOLDING_REASON,
852
- kind,
853
- source_key: sourceKey,
854
- }));
855
- }
856
- /**
857
- * Mask secret-like values in a transcript and record that it happened, because
858
- * masking is invisible in the uploaded bytes and an operator has to be able to
859
- * see that this file went up altered.
860
- */
861
- function maskSecretsInTranscript(collection, options) {
862
- const sanitized = sanitizeTextEvidenceForUpload({
863
- text: options.raw.toString("utf8"),
864
- originalBytes: options.raw,
865
- redactedFields: [`${options.kind}.body`],
866
- secretLikeFileName: options.secretLikeFileName,
867
- });
868
- if (sanitized.status === "redacted") {
869
- collection.redacted.push({
870
- kind: options.kind,
871
- label: options.evidenceLabel,
872
- redaction: sanitized.redaction,
873
- completenessLabel: sanitized.completenessLabel,
874
- });
875
- console.error("[raw-evidence] text evidence sanitized", JSON.stringify({
876
- kind: options.kind,
877
- mode: sanitized.completenessLabel,
878
- original_bytes: options.raw.byteLength,
879
- uploaded_bytes: sanitized.bytes.byteLength,
880
- }));
881
- }
882
- return sanitized;
883
- }
884
- /** Put an accepted transcript's bytes on disk and add its manifest entry. */
885
- async function stageOneTranscript(collection, file) {
886
- collection.index.value += 1;
887
- const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(file.filePath)}-${file.names.packedFileName}`);
888
- const staged = await stageEvidenceBytes(collection, {
889
- contentHash: file.contentHash,
890
- bytes: file.sanitized.bytes,
891
- fileName: path.basename(relativePath),
892
- kind: file.kind,
893
- sourceKey: file.names.sourceKey,
894
- });
895
- collection.entries.push(evidenceEntry({
896
- context: collection.context,
897
- kind: file.kind,
898
- packId: collection.packId,
899
- localPath: staged.local_path,
900
- relativePath,
901
- mediaType: file.mediaType,
902
- redactedSummary: file.sanitized.status === "redacted"
903
- ? `${file.redactedSummary} Secret-like values were deterministically redacted before upload.`
904
- : file.redactedSummary,
905
- // Both branches carry a record now (BLI-3277), so "was anything
906
- // replaced?" is the status, never the presence of `redaction`.
907
- redaction: file.sanitized.redaction,
908
- bytes: file.sanitized.bytes,
909
- codexSessionId: file.sessionId,
910
- contentAddress: file.contentAddress(file.contentHash.slice(0, 16)),
911
- stagedInPack: staged.staged_in_pack,
912
- sourceKey: file.names.sourceKey,
913
- }));
914
- }
915
- /**
916
- * Size is checked twice on purpose: once by `stat` so an oversized transcript
917
- * is never buffered at all, and once on the bytes actually read, because a live
918
- * session can grow between the two.
919
- */
920
- async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
921
- if (maxFileBytes) {
922
- let stat;
923
- try {
924
- stat = await fs.stat(filePath);
925
- }
926
- catch (error) {
927
- // The caller turns this into the `file_read_failed` skip label, which is
928
- // the one label an operator can do nothing with. The file was discovered
929
- // moments ago, so a failure here is a rotated session, a permission
930
- // problem or a dead symlink — three different answers (BLI-3238).
931
- console.error("[raw-evidence] evidence file could not be stat'd", JSON.stringify({
932
- reason: "file_read_failed",
933
- stage: "stat",
934
- ...describeError(error),
935
- }));
936
- return { status: "read_failed" };
937
- }
938
- if (stat.size > maxFileBytes)
939
- return { status: "too_large" };
940
- }
941
- let bytes;
942
- try {
943
- bytes = await fs.readFile(filePath);
944
- }
945
- catch (error) {
946
- console.error("[raw-evidence] evidence file could not be read", JSON.stringify({
947
- reason: "file_read_failed",
948
- stage: "read",
949
- ...describeError(error),
950
- }));
951
- return { status: "read_failed" };
952
- }
953
- if (maxFileBytes && bytes.byteLength > maxFileBytes) {
954
- return { status: "too_large" };
955
- }
956
- return { status: "ok", bytes };
957
- }
958
- // ---------------------------------------------------------------------------
959
- // Staging and budgets
960
- // ---------------------------------------------------------------------------
961
- /**
962
- * Put these bytes on disk once.
963
- *
964
- * If an earlier sync already staged this exact content and the copy is still
965
- * there, that copy is used — the uploader only needs a readable path, and it
966
- * does not care which pack directory holds it. This is the branch that stops
967
- * one 334 MB rollout from becoming 559 copies while its commit keeps failing.
968
- */
969
- async function stageEvidenceBytes(collection, options) {
970
- const existing = await resolveStagedObject(collection.rawEvidenceRoot, collection.staging, options.contentHash);
971
- if (existing) {
972
- collection.stagedReusedCount += 1;
973
- console.error("[raw-evidence] staged copy reused", JSON.stringify({
974
- reason: "staged_reused",
975
- kind: options.kind,
976
- content_hash_prefix: options.contentHash.slice(0, 16),
977
- byte_size: existing.entry.byte_size,
978
- pack_id: existing.entry.pack_id,
979
- }));
980
- return { local_path: existing.local_path, staged_in_pack: false };
981
- }
982
- const destination = path.join(collection.filesDir, options.fileName);
983
- await fs.writeFile(destination, options.bytes, { mode: 0o600 });
984
- await chmodPrivate(destination, 0o600);
985
- collection.stagedNewCount += 1;
986
- return { local_path: destination, staged_in_pack: true };
987
- }
988
- /**
989
- * Decrements the per-sync budget when a file fits, or returns a deferred-skip
990
- * reason when it does not. The object budget bounds request count; the byte
991
- * budget bounds buffered bytes.
992
- */
993
- function admitToBudget(budget, byteLength) {
994
- if (budget.remainingObjects <= 0)
995
- return "deferred_object_budget";
996
- if (byteLength > budget.remainingBytes)
997
- return "deferred_byte_budget";
998
- budget.remainingObjects -= 1;
999
- budget.remainingBytes -= byteLength;
1000
- return null;
1001
- }
1002
- // ---------------------------------------------------------------------------
1003
- // Git diffs
1004
- // ---------------------------------------------------------------------------
1005
- const GIT_DIFF_TARGETS = [
1006
- { label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
1007
- { label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
1008
- ];
1009
- async function collectGitDiffFiles(collection, repoRoot) {
1010
- if (!(await repoRootExists(repoRoot))) {
1011
- skipEveryDiffTargetForMissingRoot(collection);
1012
- return;
1013
- }
1014
- for (const target of GIT_DIFF_TARGETS) {
1015
- recordScanned(collection, "git_diff");
1016
- const diff = await runOneGitDiff(collection, target, repoRoot);
1017
- if (!diff)
1018
- continue;
1019
- if (diff.truncated)
1020
- recordGitDiffTruncation(collection, diff);
1021
- // An empty diff is not a gap: there was simply nothing to record. The
1022
- // truncation marker above still stands even when zero bytes survived.
1023
- if (!diff.stdout.trim())
1024
- continue;
1025
- await stageOneGitDiff(collection, {
1026
- label: target.label,
1027
- diffText: diff.stdout,
1028
- truncated: diff.truncated,
1029
- });
1030
- }
1031
- }
1032
- /**
1033
- * BLI-3551: the root is asked about once per pass, before either target, so a
1034
- * pruned worktree costs ONE line instead of one per diff target per session per
1035
- * tick — and the line says the folder is gone rather than accusing git of
1036
- * failing. Both targets still get their own named gap.
1037
- */
1038
- function skipEveryDiffTargetForMissingRoot(collection) {
1039
- console.error("[raw-evidence] git diff skipped, repository root is no longer on disk", JSON.stringify({
1040
- reason: REPO_ROOT_MISSING_REASON,
1041
- diff_targets_skipped: GIT_DIFF_TARGETS.length,
1042
- next_action: "nothing to do; the diff returns when the worktree is restored or the session ages out",
1043
- }));
1044
- for (const target of GIT_DIFF_TARGETS) {
1045
- recordScanned(collection, "git_diff");
1046
- collection.skipped.push({
1047
- kind: "git_diff",
1048
- label: target.label,
1049
- reason: REPO_ROOT_MISSING_REASON,
1050
- });
1051
- }
1052
- }
1053
- /**
1054
- * Run one diff, or record why it produced nothing. Null means the gap is
1055
- * already named in the ledger, so the caller only has to move on.
1056
- */
1057
- async function runOneGitDiff(collection, target, repoRoot) {
1058
- try {
1059
- return await runGitDiff(target.args, repoRoot);
1060
- }
1061
- catch (error) {
1062
- // The root was there a moment ago and is not now (or a second collector
1063
- // pruned it mid-tick). Same named outcome, still not a git failure.
1064
- if (error instanceof RepoRootMissingError) {
1065
- collection.skipped.push({
1066
- kind: "git_diff",
1067
- label: target.label,
1068
- reason: REPO_ROOT_MISSING_REASON,
1069
- });
1070
- return null;
1071
- }
1072
- // `git_diff_failed` is the skip label and stays. It covers git not being
1073
- // installed, the folder not being a repo, a locked index and a diff that
1074
- // exceeded the child-process buffer — and the diff is half the evidence
1075
- // for what someone actually changed, so losing it quietly matters.
1076
- console.error("[raw-evidence] git diff failed", JSON.stringify({
1077
- reason: "git_diff_failed",
1078
- diff_target: target.label,
1079
- ...describeError(error),
1080
- }));
1081
- collection.skipped.push({
1082
- kind: "git_diff",
1083
- label: target.label,
1084
- reason: "git_diff_failed",
1085
- });
1086
- return null;
1087
- }
1088
- }
1089
- /** A diff that hit its size or time cap is partial evidence, and says so. */
1090
- function recordGitDiffTruncation(collection, diff) {
1091
- markCapApplied(collection, "git_diff", diff.truncationCapType);
1092
- collection.truncated.push({
1093
- kind: "git_diff",
1094
- reason: diff.truncationReason,
1095
- ...(diff.truncationCapType === "max_bytes_per_diff"
1096
- ? { max_bytes: MAX_GIT_DIFF_BYTES }
1097
- : {}),
1098
- observed_bytes: diff.observedBytes,
1099
- included_bytes: Buffer.byteLength(diff.stdout, "utf8"),
1100
- });
1101
- }
1102
- async function stageOneGitDiff(collection, target) {
1103
- const sanitized = maskSecretsInGitDiff(collection, target);
1104
- const contentHash = sha256(sanitized.bytes);
1105
- if (collection.skipContentHashes.has(contentHash)) {
1106
- collection.reused.push({
1107
- kind: "git_diff",
1108
- label: target.label,
1109
- content_hash_sha256: contentHash,
1110
- codex_session_id: null,
1111
- });
1112
- return;
1113
- }
1114
- const deferReason = admitToBudget(collection.budget, sanitized.bytes.byteLength);
1115
- if (deferReason) {
1116
- markBudgetCapApplied(collection, deferReason);
1117
- collection.skipped.push({
1118
- kind: "git_diff",
1119
- label: target.label,
1120
- reason: deferReason,
1121
- });
1122
- return;
1123
- }
1124
- await writeGitDiffToPack(collection, { target, sanitized, contentHash });
1125
- }
1126
- /**
1127
- * Mask secret-like values in a diff and record that it happened. A diff carries
1128
- * whatever a person pasted into a config file, so this is the branch that most
1129
- * often fires — and an operator has to see that the bytes went up altered.
1130
- */
1131
- function maskSecretsInGitDiff(collection, target) {
1132
- const sanitized = sanitizeTextEvidenceForUpload({
1133
- text: target.diffText,
1134
- redactedFields: [`git_diff.${target.label}`],
1135
- });
1136
- if (sanitized.status === "redacted") {
1137
- collection.redacted.push({
1138
- kind: "git_diff",
1139
- label: target.label,
1140
- redaction: sanitized.redaction,
1141
- completenessLabel: sanitized.completenessLabel,
1142
- });
1143
- console.error("[raw-evidence] git diff sanitized", JSON.stringify({
1144
- mode: sanitized.completenessLabel,
1145
- original_bytes: Buffer.byteLength(target.diffText, "utf8"),
1146
- uploaded_bytes: sanitized.bytes.byteLength,
1147
- }));
1148
- }
1149
- return sanitized;
1150
- }
1151
- /** Put an accepted diff's bytes on disk and add its manifest entry. */
1152
- async function writeGitDiffToPack(collection, diff) {
1153
- const label = diff.target.label;
1154
- const relativePath = path.join("files", `git-${label}.diff`);
1155
- const diffSourceKey = evidenceSourceKey({
1156
- kind: "git_diff",
1157
- sessionId: collection.context.workContextId,
1158
- label,
1159
- });
1160
- const staged = await stageEvidenceBytes(collection, {
1161
- contentHash: diff.contentHash,
1162
- bytes: diff.sanitized.bytes,
1163
- fileName: path.basename(relativePath),
1164
- kind: "git_diff",
1165
- sourceKey: diffSourceKey,
1166
- });
1167
- collection.entries.push(evidenceEntry({
1168
- context: collection.context,
1169
- kind: "git_diff",
1170
- packId: collection.packId,
1171
- localPath: staged.local_path,
1172
- relativePath,
1173
- stagedInPack: staged.staged_in_pack,
1174
- sourceKey: diffSourceKey,
1175
- mediaType: "text/x-diff",
1176
- redactedSummary: gitDiffSummary(label, {
1177
- redacted: diff.sanitized.status === "redacted",
1178
- truncated: diff.target.truncated,
1179
- }),
1180
- redaction: diff.sanitized.redaction,
1181
- bytes: diff.sanitized.bytes,
1182
- contentAddress: `git-diff/${label}-${diff.contentHash.slice(0, 16)}.diff`,
1183
- }));
1184
- }
1185
- function gitDiffSummary(label, state) {
1186
- if (state.redacted) {
1187
- return `Raw git ${label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`;
1188
- }
1189
- if (state.truncated) {
1190
- return `Raw git ${label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`;
1191
- }
1192
- return `Raw git ${label} diff preserved locally with env/secret paths excluded.`;
1193
- }
1194
- // ---------------------------------------------------------------------------
1195
- // Walking the session store
1196
- // ---------------------------------------------------------------------------
1197
- /**
1198
- * Newest-first `.jsonl` files under these directories, modified since `cutoffMs`.
1199
- *
1200
- * Secret-like directory and file names are never descended into or opened;
1201
- * symlinked duplicates are collapsed by real path so one transcript reachable
1202
- * two ways is collected once.
1203
- */
1204
- async function walkJsonlFiles(dir, cutoffMs) {
1205
- const out = [];
1206
- const stack = Array.isArray(dir) ? [...dir] : [dir];
1207
- const seen = new Set();
1208
- // Counted rather than logged per directory: a wide walk can hit many, and
1209
- // the useful signal is "N directories in the session store were skipped and
1210
- // here is the first reason", not N near-identical lines (BLI-3238).
1211
- let unreadableDirCount = 0;
1212
- let firstUnreadableDir = null;
1213
- while (stack.length > 0) {
1214
- const current = stack.pop();
1215
- if (!current || isSecretLikePath(current))
1216
- continue;
1217
- let entries;
1218
- try {
1219
- entries = await fs.readdir(current, { withFileTypes: true });
1220
- }
1221
- catch (error) {
1222
- // A directory that cannot be listed hides every session under it, and
1223
- // the walk's only visible effect is a smaller file count.
1224
- unreadableDirCount += 1;
1225
- firstUnreadableDir ??= describeError(error);
1226
- continue;
1227
- }
1228
- for (const entry of entries) {
1229
- const full = path.join(current, entry.name);
1230
- if (isSecretLikePath(full))
1231
- continue;
1232
- if (entry.isDirectory()) {
1233
- stack.push(full);
1234
- continue;
1235
- }
1236
- if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
1237
- continue;
1238
- const stat = await fs.stat(full);
1239
- if (stat.mtimeMs < cutoffMs)
1240
- continue;
1241
- const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
1242
- if (seen.has(dedupeKey))
1243
- continue;
1244
- seen.add(dedupeKey);
1245
- out.push({ file: full, mtimeMs: stat.mtimeMs });
1246
- }
1247
- }
1248
- if (unreadableDirCount > 0) {
1249
- console.error("[raw-evidence] session-store directories skipped during the walk", JSON.stringify({
1250
- reason: "session_dir_unreadable",
1251
- unreadable_dir_count: unreadableDirCount,
1252
- found_file_count: out.length,
1253
- ...firstUnreadableDir,
1254
- }));
1255
- }
1256
- out.sort((a, b) => b.mtimeMs - a.mtimeMs);
1257
- return out.map((entry) => entry.file);
1258
138
  }