@bli-cockpit/cli 0.2.48 → 0.2.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/adapters/raw-evidence-attribution-gaps.js +133 -0
  2. package/dist/adapters/raw-evidence-claude-reader.js +108 -0
  3. package/dist/adapters/raw-evidence-codex-reader.js +147 -0
  4. package/dist/adapters/raw-evidence-collection-state.js +199 -0
  5. package/dist/adapters/raw-evidence-facts.js +338 -0
  6. package/dist/adapters/raw-evidence-git-diff-reader.js +187 -0
  7. package/dist/adapters/raw-evidence-image-reader.js +107 -0
  8. package/dist/adapters/raw-evidence-sanitize.js +56 -0
  9. package/dist/adapters/raw-evidence-transcript-file.js +182 -0
  10. package/dist/adapters/raw-evidence.js +94 -1203
  11. package/dist/autostart-contract.js +79 -0
  12. package/dist/autostart-darwin-plist.js +265 -0
  13. package/dist/autostart-darwin.js +171 -0
  14. package/dist/autostart-windows-scripts.js +310 -0
  15. package/dist/autostart-windows-task-xml.js +260 -0
  16. package/dist/autostart-windows.js +237 -0
  17. package/dist/autostart-xml.js +23 -0
  18. package/dist/autostart.js +35 -1148
  19. package/dist/commands/agent-rules-command.js +55 -0
  20. package/dist/commands/agent-session-report.js +290 -0
  21. package/dist/commands/analyze.js +131 -0
  22. package/dist/commands/autostart-command.js +105 -0
  23. package/dist/commands/backfill-batches.js +34 -0
  24. package/dist/commands/backfill-candidates.js +54 -0
  25. package/dist/commands/backfill-checkpoint.js +101 -0
  26. package/dist/commands/backfill-command-line.js +70 -0
  27. package/dist/commands/backfill-evidence-outcomes.js +104 -0
  28. package/dist/commands/backfill-issues.js +265 -0
  29. package/dist/commands/backfill-output.js +75 -0
  30. package/dist/commands/backfill-plan.js +71 -0
  31. package/dist/commands/backfill-reasons.js +107 -0
  32. package/dist/commands/backfill-report.js +298 -0
  33. package/dist/commands/backfill-result.js +150 -0
  34. package/dist/commands/backfill-scan.js +274 -0
  35. package/dist/commands/backfill-scope.js +114 -0
  36. package/dist/commands/backfill-session-report.js +145 -0
  37. package/dist/commands/backfill-types.js +1 -0
  38. package/dist/commands/backfill-upload.js +212 -0
  39. package/dist/commands/backfill.js +58 -1705
  40. package/dist/commands/cli-io.js +13 -0
  41. package/dist/commands/doctor.js +57 -0
  42. package/dist/commands/jarvis-trace.js +184 -0
  43. package/dist/commands/jarvis.js +323 -7
  44. package/dist/commands/local-arg-values.js +169 -0
  45. package/dist/commands/local-args-collector.js +604 -0
  46. package/dist/commands/local-args-tower.js +891 -0
  47. package/dist/commands/local-args.js +10 -1549
  48. package/dist/commands/local-help.js +30 -5
  49. package/dist/commands/local.js +21 -1786
  50. package/dist/commands/login.js +53 -0
  51. package/dist/commands/logout.js +66 -0
  52. package/dist/commands/memory-install-claude.js +294 -0
  53. package/dist/commands/memory-install-codex.js +205 -0
  54. package/dist/commands/memory-install-contract.js +231 -0
  55. package/dist/commands/memory-install-files.js +63 -0
  56. package/dist/commands/memory-install-skills.js +121 -0
  57. package/dist/commands/memory-install-toml.js +265 -0
  58. package/dist/commands/memory-install.js +378 -0
  59. package/dist/commands/onboard-receipts.js +66 -0
  60. package/dist/commands/onboard-report.js +274 -0
  61. package/dist/commands/onboard.js +449 -0
  62. package/dist/commands/ops-render.js +36 -0
  63. package/dist/commands/public-root.js +1 -1
  64. package/dist/commands/serve.js +13 -0
  65. package/dist/commands/session-sync.js +513 -534
  66. package/dist/commands/settings-render.js +28 -0
  67. package/dist/commands/settings.js +66 -2
  68. package/dist/commands/start.js +47 -0
  69. package/dist/commands/sync-followups.js +308 -0
  70. package/dist/commands/sync.js +387 -0
  71. package/dist/local-state-attributed-target.js +75 -0
  72. package/dist/local-state-config.js +147 -0
  73. package/dist/local-state-files.js +59 -0
  74. package/dist/local-state-identity.js +73 -0
  75. package/dist/local-state-pairing.js +263 -0
  76. package/dist/local-state-paths.js +61 -0
  77. package/dist/local-state-session.js +68 -0
  78. package/dist/local-state-status.js +163 -0
  79. package/dist/local-state-work-context.js +190 -0
  80. package/dist/local-state.js +34 -848
  81. package/dist/tower-client.js +3 -2
  82. package/dist/tower-stream.js +76 -6
  83. package/package.json +2 -1
@@ -1,472 +1,124 @@
1
- import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, SourceScanResultSchema, } from "@bli-cockpit/telemetry-core";
2
- import crypto from "node:crypto";
3
- import fs from "node:fs/promises";
4
- import os from "node:os";
1
+ /**
2
+ * Collect one repo's raw evidence into a private, content-keyed pack.
3
+ *
4
+ * This is the content guard between "a transcript exists on this laptop" and
5
+ * "these bytes may become durable evidence". Attribution before it only reads
6
+ * metadata; the server commit after it is the second guard.
7
+ *
8
+ * The pass, in order:
9
+ *
10
+ * 1. **Open a staging directory.** The pack's id comes from its content, so it
11
+ * cannot be known yet (BLI-3066). Bytes land in `.staging-<pid>-<rand>`.
12
+ * 2. **Record what attribution already knew** — every session it scanned and
13
+ * every one it could not name, as completeness records, before a single byte
14
+ * is read.
15
+ * 3. **Collect** every source: Codex transcripts, Claude transcripts and their
16
+ * subagent sidecars, images explicitly attached to either, and the two git
17
+ * diffs.
18
+ * 4. **Finish** one of three ways — empty, promoted, or failed — each reporting
19
+ * the same facts, pointers and scan.
20
+ *
21
+ * Three doctrines hold everywhere in this family:
22
+ *
23
+ * - **Nothing gets in the way of collecting a session.** Redaction masks and
24
+ * continues; it never drops a file.
25
+ * - **A failure must name itself.** Every path that declines a file records a
26
+ * reason label in the gap ledger.
27
+ * - **Never succeed silently either.** Staging, reuse, holds and failures all
28
+ * log metadata — ids, sizes, counts, reason labels. Never content, never a
29
+ * local path.
30
+ *
31
+ * Who owns what (BLI-3586 moved the bodies out; every public name is still
32
+ * exported from this file, so no consumer's import changed):
33
+ *
34
+ * | module | owns |
35
+ * | ------------------------------- | --------------------------------------- |
36
+ * | `raw-evidence-collection-state` | what a pass carries, its budget, staging bytes |
37
+ * | `raw-evidence-codex-reader` | Codex transcripts, and the capped fallback walk |
38
+ * | `raw-evidence-claude-reader` | Claude mains, sidecars, the D7/D9 outcomes |
39
+ * | `raw-evidence-image-reader` | images attached to either transcript |
40
+ * | `raw-evidence-git-diff-reader` | the unstaged and staged diffs |
41
+ * | `raw-evidence-transcript-file` | one transcript, end to end |
42
+ * | `raw-evidence-sanitize` | masking, and the receipt it leaves |
43
+ * | `raw-evidence-facts` | the three endings, the pointers, the scan |
44
+ * | `raw-evidence-completeness` | the gap ledger |
45
+ * | `raw-evidence-manifest` | what we say about a file |
46
+ * | `raw-evidence-pack-store` | where the bytes live |
47
+ * | `raw-evidence-keys` | naming |
48
+ * | `raw-evidence-git-diff` | running git |
49
+ * | `raw-evidence-attribution-gaps` | step 2's scan results as ledger records |
50
+ *
51
+ * The readers deliberately share one shape — derive the names, guard, mask and
52
+ * record, hash, check reuse, check budget, then a single `write…ToPack` that
53
+ * puts the bytes down and adds the manifest entry. They do NOT share one
54
+ * implementation, because each pushes a different ledger row and logs a
55
+ * different line, and merging them would change what an operator reads.
56
+ */
5
57
  import path from "node:path";
6
- import { makeSourceAdapterIdentity, } from "./common.js";
7
- import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
8
- import { defaultCodexSessionDirs, } from "./codex-attribution.js";
9
- import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-policy.js";
10
- import { describeError } from "../health-detail.js";
11
- import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject, } from "../raw-evidence-staging.js";
12
- import { isSecretLikePath, safeKeySegment, sha256, shortHash, } from "./raw-evidence-keys.js";
13
- import { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
14
- import { countEvidenceEntries, makeEvidenceCompleteness, markBudgetCapApplied, markCapApplied, recordScanned, recordSkipCount, recordTruncationCount, } from "./raw-evidence-completeness.js";
15
- import { evidenceEntry, pointerFromEntry, RAW_EVIDENCE_BUCKET, } from "./raw-evidence-manifest.js";
16
- import { chmodPrivate, ensurePrivateDir, persistStagingState, promoteStagedPack, stageManifest, } from "./raw-evidence-pack-store.js";
17
- import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, REPO_ROOT_MISSING_REASON, RepoRootMissingError, repoRootExists, runGitDiff, } from "./raw-evidence-git-diff.js";
58
+ import { recordClaudeAttributionCompleteness, recordCodexAttributionCompleteness, } from "./raw-evidence-attribution-gaps.js";
59
+ import { ensurePrivateDir } from "./raw-evidence-pack-store.js";
60
+ import { DEFAULT_SESSION_LIMIT, DEFAULT_SINCE_MINUTES, newStagingDir, openCollection, } from "./raw-evidence-collection-state.js";
61
+ import { finishWithEmptyPack, finishWithFailedPack, finishWithPromotedPack, } from "./raw-evidence-facts.js";
62
+ import { collectCodexJsonlFiles } from "./raw-evidence-codex-reader.js";
63
+ import { collectClaudeJsonlFiles } from "./raw-evidence-claude-reader.js";
64
+ import { collectGitDiffFiles } from "./raw-evidence-git-diff-reader.js";
18
65
  // Re-exported so every consumer keeps importing from `adapters/raw-evidence`.
19
66
  export { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./raw-evidence-manifest.js";
20
67
  export { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
21
- const DEFAULT_SINCE_MINUTES = 24 * 60;
22
- const DEFAULT_SESSION_LIMIT = 50;
23
- // Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
24
- // transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
25
- // so 2 GiB leaves room for several files without starving the sync; overflow
26
- // still defers and converges.
27
- export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
28
- export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
29
- const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;
30
- // ---------------------------------------------------------------------------
31
- // The pass
32
- // ---------------------------------------------------------------------------
68
+ export { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET, } from "./raw-evidence-collection-state.js";
69
+ /**
70
+ * Decide which bytes on this laptop are allowed to become durable evidence for
71
+ * one repo, and put exactly those into one private, content-keyed pack.
72
+ *
73
+ * Never throws: a crash mid-pass still returns facts and a `failed` scan, so a
74
+ * broken sync is reported rather than silently absent.
75
+ */
33
76
  export async function collectRawEvidencePack(context, options) {
34
- const startedAt = context.now.toISOString();
35
77
  const rawEvidenceRoot = path.join(options.stateDir, "raw-evidence");
36
- // Staging first, promotion second (BLI-3066). The pack id cannot be known
37
- // until the content is, so bytes land in a private staging directory and the
38
- // directory is then renamed to its content-keyed name — or dropped, when an
39
- // identical pack is already there.
40
- const stagingDir = path.join(rawEvidenceRoot, `.staging-${process.pid}-${crypto.randomUUID().slice(0, 8)}`);
78
+ const stagingDir = newStagingDir(rawEvidenceRoot);
41
79
  const sinceMinutes = options.sinceMinutes ?? DEFAULT_SINCE_MINUTES;
42
80
  const collection = await openCollection(context, options, {
43
81
  rawEvidenceRoot,
44
82
  stagingDir,
45
83
  });
46
84
  const scanWindow = {
47
- startedAt,
85
+ startedAt: context.now.toISOString(),
48
86
  finishedAt: () => context.now.toISOString(),
49
87
  sinceMinutes,
50
88
  };
89
+ const run = { stagingDir, window: scanWindow };
51
90
  try {
52
91
  await ensurePrivateDir(stagingDir);
53
92
  await ensurePrivateDir(collection.filesDir);
54
93
  recordAttributionCompleteness(collection, options);
55
- if (options.includeCodexJsonl !== false) {
56
- await collectCodexJsonlFiles(collection, {
57
- codexSessionFiles: options.codexSessionFiles,
58
- sessionsDir: options.sessionsDir,
59
- sessionsDirs: options.sessionsDirs,
60
- sinceMinutes,
61
- limit: options.sessionLimit ?? DEFAULT_SESSION_LIMIT,
62
- });
63
- }
64
- if (options.includeClaudeJsonl !== false && options.claudeSessionFiles) {
65
- await collectClaudeJsonlFiles(collection, options.claudeSessionFiles);
66
- }
67
- await collectGitDiffFiles(collection, options.repoRoot);
68
- if (collection.entries.length === 0) {
69
- return await finishWithEmptyPack(collection, options, {
70
- stagingDir,
71
- window: scanWindow,
72
- });
73
- }
74
- return await finishWithPromotedPack(collection, options, {
75
- stagingDir,
76
- window: scanWindow,
77
- });
94
+ await collectEverySource(collection, options, sinceMinutes);
95
+ return collection.entries.length === 0
96
+ ? await finishWithEmptyPack(collection, options, run)
97
+ : await finishWithPromotedPack(collection, options, run);
78
98
  }
79
99
  catch (error) {
80
- return await finishWithFailedPack(collection, {
81
- stagingDir,
82
- window: scanWindow,
83
- error,
84
- });
85
- }
86
- }
87
- async function openCollection(context, options, places) {
88
- const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
89
- const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
90
- const staging = await readRawEvidenceStagingState(options.stateDir);
91
- return {
92
- context,
93
- filesDir: path.join(places.stagingDir, "files"),
94
- rawEvidenceRoot: places.rawEvidenceRoot,
95
- packId: "",
96
- staging,
97
- heldSources: collectionHeldSources(staging, context.now, options),
98
- stagedReusedCount: 0,
99
- stagedNewCount: 0,
100
- deliveryHeldCount: 0,
101
- entries: [],
102
- skipped: [],
103
- truncated: [],
104
- failed: [],
105
- redacted: [],
106
- reused: [],
107
- scanned: new Map(),
108
- caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
109
- skipContentHashes: options.skipContentHashes ?? new Set(),
110
- budget: options.budget ?? {
111
- remainingBytes: byteBudget,
112
- remainingObjects: objectBudget,
113
- },
114
- index: { value: 0 },
115
- };
116
- }
117
- /**
118
- * Which sources this pass refuses to even read because their delivery is inside
119
- * a backoff window — none of them, when a person asked for this pass.
120
- *
121
- * The bypass is logged rather than assumed: an operator retry that quietly
122
- * ignored a hold would be as unreadable as the hold that quietly blocked it.
123
- */
124
- function collectionHeldSources(staging, now, options) {
125
- const held = heldSourceKeys(staging, now);
126
- if (deliveryBackoffApplies(options.deliveryMode))
127
- return held;
128
- if (held.size > 0) {
129
- console.error("[raw-evidence] delivery backoff bypassed for operator retry", JSON.stringify({
130
- reason: DELIVERY_BACKOFF_BYPASS_REASON,
131
- source_count: held.size,
132
- }));
100
+ return await finishWithFailedPack(collection, { ...run, error });
133
101
  }
134
- return new Set();
135
- }
136
- /** Every cap this pass could hit, declared up front and flipped when applied. */
137
- function startingCaps(options) {
138
- return [
139
- {
140
- source: "raw_evidence",
141
- cap_type: "byte_budget",
142
- limit: options.byteBudget,
143
- observed: options.budget?.remainingBytes ?? options.byteBudget,
144
- applied: false,
145
- },
146
- {
147
- source: "raw_evidence",
148
- cap_type: "object_budget",
149
- limit: options.objectBudget,
150
- observed: options.budget?.remainingObjects ?? options.objectBudget,
151
- applied: false,
152
- },
153
- {
154
- source: "git_diff",
155
- cap_type: "max_bytes_per_diff",
156
- limit: MAX_GIT_DIFF_BYTES,
157
- applied: false,
158
- },
159
- {
160
- source: "git_diff",
161
- cap_type: "timeout_ms",
162
- limit: GIT_DIFF_TIMEOUT_MS,
163
- applied: false,
164
- },
165
- {
166
- source: "codex_jsonl",
167
- cap_type: "max_file_bytes",
168
- limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
169
- applied: false,
170
- },
171
- {
172
- source: "claude_jsonl",
173
- cap_type: "max_file_bytes",
174
- limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
175
- applied: false,
176
- },
177
- {
178
- source: "claude_jsonl_sidecar",
179
- cap_type: "max_file_bytes",
180
- limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
181
- applied: false,
182
- },
183
- ];
184
- }
185
- // ---------------------------------------------------------------------------
186
- // The three ways a pass ends
187
- // ---------------------------------------------------------------------------
188
- /**
189
- * Nothing collected: drop the staging directory instead of leaving an empty
190
- * pack behind. This used to leave one empty `work-*` dir per sync.
191
- */
192
- async function finishWithEmptyPack(collection, options, run) {
193
- await discardStagingDir(run.stagingDir);
194
- collection.packId = contentKeyedRawEvidencePackId({
195
- workContextId: collection.context.workContextId,
196
- contentHashes: [],
197
- });
198
- await persistStagingState(options.stateDir, collection.staging, collection.context.now.toISOString());
199
- const facts = makeEmptyPackFacts(collection, run.window);
200
- return {
201
- facts,
202
- scan: makeRawEvidenceScan({
203
- context: collection.context,
204
- startedAt: run.window.startedAt,
205
- status: "partial",
206
- facts,
207
- }),
208
- };
209
- }
210
- async function finishWithPromotedPack(collection, options, run) {
211
- const context = collection.context;
212
- const entries = collection.entries;
213
- // The pack is named by what is in it, never by when it was made. Identical
214
- // content on the next sync resolves to the identical directory, which is
215
- // the whole fix for BLI-3066.
216
- const packId = contentKeyedRawEvidencePackId({
217
- workContextId: context.workContextId,
218
- contentHashes: entries.map((entry) => entry.content_hash_sha256),
219
- });
220
- collection.packId = packId;
221
- const promotion = await promoteStagedPack({
222
- rawEvidenceRoot: collection.rawEvidenceRoot,
223
- stagingDir: run.stagingDir,
224
- packId,
225
- workContextId: context.workContextId,
226
- entries,
227
- });
228
- const evidenceDir = promotion.evidenceDir;
229
- rebaseStagedEntriesOntoPack(collection, evidenceDir);
230
- const manifestPath = path.join(evidenceDir, "manifest.json");
231
- const manifestBytes = await stageManifest({
232
- context,
233
- packId,
234
- manifestPath,
235
- entries,
236
- skipped: collection.skipped,
237
- redacted: collection.redacted,
238
- reused: collection.reused,
239
- reusePack: promotion.state === "reused",
240
- });
241
- entries.push(evidenceEntry({
242
- context,
243
- kind: "manifest",
244
- packId,
245
- localPath: manifestPath,
246
- relativePath: "manifest.json",
247
- mediaType: "application/json",
248
- redactedSummary: "Local raw evidence pack manifest.",
249
- bytes: manifestBytes,
250
- // The manifest carries its own content hash in its key, exactly like
251
- // every other object in the pack (BLI-3552). Without it the key was
252
- // `…/<packId>/manifest.json`, and a pack id is a function of the OTHER
253
- // files' hashes — not of the manifest's bytes, which also carry
254
- // `created_at`, the ordering-dependent `files[].relative_path`, `branch`,
255
- // and the skipped/redacted/reused ledgers. So the same key named
256
- // different bytes whenever the pack directory was not adopted verbatim
257
- // (a refill, a pruned or wiped state dir, a machine that had never seen
258
- // the pack). `begin` then answered `hash_mismatch_committed_object` on
259
- // every sync forever, because the new manifest is stable and the old one
260
- // is durable. `stageManifest`'s byte-stability trick still stands; it is
261
- // now a nice-to-have rather than the only thing between us and a loop.
262
- contentAddress: `manifest/${sha256(manifestBytes).slice(0, 16)}.json`,
263
- }));
264
- await persistStagingState(options.stateDir, collection.staging, context.now.toISOString());
265
- console.error("[raw-evidence] pack staged", JSON.stringify({
266
- pack_id: packId,
267
- stage_state: promotion.state,
268
- reason: stageReasonLabel(promotion.state, promotion.priorPackCount),
269
- prior_pack_count: promotion.priorPackCount,
270
- refilled_file_count: promotion.refilledFileCount,
271
- file_count: entries.length,
272
- byte_size: totalByteSize(entries),
273
- staged_new: collection.stagedNewCount,
274
- staged_reused: collection.stagedReusedCount,
275
- delivery_held: collection.deliveryHeldCount,
276
- }));
277
- const facts = {
278
- ...packFactsShell(collection, {
279
- packId,
280
- manifestPath,
281
- evidenceDir,
282
- window: run.window,
283
- }),
284
- stage_state: promotion.state,
285
- file_count: entries.length,
286
- byte_size: totalByteSize(entries),
287
- content_kinds: [...new Set(entries.map((entry) => entry.kind))],
288
- pointers: entries.map(pointerFromEntry),
289
- upload_files: entries.map(uploadFileFromEntry),
290
- };
291
- return {
292
- facts,
293
- scan: makeRawEvidenceScan({
294
- context,
295
- startedAt: run.window.startedAt,
296
- status: entries.length > 1 ? "ok" : "partial",
297
- facts,
298
- }),
299
- };
300
- }
301
- /**
302
- * A crashed pass. The staged-object index is deliberately NOT written: this
303
- * attempt's bookkeeping describes a directory that is about to be deleted.
304
- */
305
- async function finishWithFailedPack(collection, run) {
306
- collection.failed.push({
307
- kind: "raw_evidence",
308
- reason: "collection_failed",
309
- });
310
- // Staging is per-attempt scratch: a crashed pass must not leave a partial
311
- // directory behind to be counted, re-hashed or swept later.
312
- await discardStagingDir(run.stagingDir);
313
- collection.packId =
314
- collection.packId ||
315
- contentKeyedRawEvidencePackId({
316
- workContextId: collection.context.workContextId,
317
- contentHashes: collection.entries.map((entry) => entry.content_hash_sha256),
318
- });
319
- console.error("[raw-evidence] pack collection failed", JSON.stringify({
320
- pack_id: collection.packId,
321
- reason: "collection_failed",
322
- detail: run.error instanceof Error ? run.error.name : typeof run.error,
323
- collected_file_count: collection.entries.length,
324
- staged_new: collection.stagedNewCount,
325
- staged_reused: collection.stagedReusedCount,
326
- }));
327
- const facts = makeEmptyPackFacts(collection, run.window);
328
- return {
329
- facts,
330
- scan: makeRawEvidenceScan({
331
- context: collection.context,
332
- startedAt: run.window.startedAt,
333
- status: "failed",
334
- facts,
335
- }),
336
- };
337
- }
338
- /**
339
- * A pack with no files of its own: the empty pass and the crashed pass report
340
- * the same shape, differing only in the scan status and the reason already
341
- * logged. `evidence_completeness` still carries every gap this pass recorded.
342
- */
343
- function makeEmptyPackFacts(collection, scanWindow) {
344
- const evidenceDir = path.join(collection.rawEvidenceRoot, collection.packId);
345
- return {
346
- ...packFactsShell(collection, {
347
- packId: collection.packId,
348
- manifestPath: path.join(evidenceDir, "manifest.json"),
349
- evidenceDir,
350
- window: scanWindow,
351
- }),
352
- stage_state: "empty",
353
- file_count: 0,
354
- byte_size: 0,
355
- content_kinds: [],
356
- pointers: [],
357
- upload_files: [],
358
- };
359
- }
360
- /** The counts every ending reports identically, however the pass ended. */
361
- function packFactsShell(collection, place) {
362
- return {
363
- pack_id: place.packId,
364
- manifest_path: place.manifestPath,
365
- evidence_dir: place.evidenceDir,
366
- storage_bucket: RAW_EVIDENCE_BUCKET,
367
- skipped_count: countEvidenceEntries(collection.skipped),
368
- sanitized_count: collection.redacted.length,
369
- reused_count: collection.reused.length,
370
- staged_reused_count: collection.stagedReusedCount,
371
- staged_new_count: collection.stagedNewCount,
372
- delivery_held_count: collection.deliveryHeldCount,
373
- deferred_byte_budget_count: countDeferred(collection, "deferred_byte_budget"),
374
- deferred_object_budget_count: countDeferred(collection, "deferred_object_budget"),
375
- evidence_completeness: makeEvidenceCompleteness(collection, {
376
- startedAt: place.window.startedAt,
377
- finishedAt: place.window.finishedAt(),
378
- sinceMinutes: place.window.sinceMinutes,
379
- }),
380
- reused: collection.reused,
381
- };
382
- }
383
- function countDeferred(collection, reason) {
384
- return countEvidenceEntries(collection.skipped, (entry) => entry.reason === reason);
385
- }
386
- function totalByteSize(entries) {
387
- return entries.reduce((sum, entry) => sum + entry.byte_size, 0);
388
- }
389
- function uploadFileFromEntry(entry) {
390
- return {
391
- pointer: pointerFromEntry(entry),
392
- local_path: entry.local_path,
393
- kind: entry.kind,
394
- codex_session_id: entry.codex_session_id ?? null,
395
- ...(entry.artifact_metadata
396
- ? { artifact_metadata: entry.artifact_metadata }
397
- : {}),
398
- };
399
- }
400
- function stageReasonLabel(state, priorPackCount) {
401
- if (state === "reused")
402
- return "staged_reused";
403
- if (state === "restaged_incomplete")
404
- return "restaged_incomplete";
405
- return priorPackCount > 0 ? "restaged_content_changed" : "staged_new";
406
102
  }
407
103
  /**
408
- * Point every entry this pass staged at its home in the promoted pack, and
409
- * remember the content hash so the next sync can adopt the copy instead of
410
- * writing it again. Entries adopted from another pack keep their path.
104
+ * Every kind of evidence this repo can offer, in the order it is read. Codex and
105
+ * Claude transcripts are each switchable off by the caller; the git diffs are
106
+ * not, because they are half the evidence of what someone actually changed.
411
107
  */
412
- function rebaseStagedEntriesOntoPack(collection, evidenceDir) {
413
- for (const entry of collection.entries) {
414
- if (!entry.staged_in_pack)
415
- continue;
416
- entry.local_path = path.join(evidenceDir, "files", path.basename(entry.local_path));
417
- recordStagedObject(collection.staging, entry.content_hash_sha256, {
418
- pack_id: collection.packId,
419
- relative_path: `files/${path.basename(entry.local_path)}`,
420
- byte_size: entry.byte_size,
421
- source_key: entry.source_key,
422
- staged_at: collection.context.now.toISOString(),
108
+ async function collectEverySource(collection, options, sinceMinutes) {
109
+ if (options.includeCodexJsonl !== false) {
110
+ await collectCodexJsonlFiles(collection, {
111
+ codexSessionFiles: options.codexSessionFiles,
112
+ sessionsDir: options.sessionsDir,
113
+ sessionsDirs: options.sessionsDirs,
114
+ sinceMinutes,
115
+ limit: options.sessionLimit ?? DEFAULT_SESSION_LIMIT,
423
116
  });
424
117
  }
425
- }
426
- async function discardStagingDir(stagingDir) {
427
- await fs
428
- .rm(stagingDir, { recursive: true, force: true })
429
- .catch(() => undefined);
430
- }
431
- function makeRawEvidenceScan(options) {
432
- return SourceScanResultSchema.parse({
433
- adapter: makeSourceAdapterIdentity("collector_runtime", "raw-evidence-pack"),
434
- work_context_id: options.context.workContextId,
435
- status: options.status,
436
- started_at: options.startedAt,
437
- finished_at: options.context.now.toISOString(),
438
- events: [
439
- {
440
- source_event_id: `raw-evidence-pack:${options.facts.pack_id}`,
441
- event_type: "raw_evidence_pack_written",
442
- occurred_at: options.context.now.toISOString(),
443
- redaction: {
444
- privacy_classification: "remote_durable_raw_evidence",
445
- redaction_status: "raw_remote_durable",
446
- redacted_fields: ["local_path"],
447
- raw_evidence_pointer_ids: options.facts.pointers.map((pointer) => pointer.raw_evidence_pointer_id),
448
- redacted_summary: "Raw prompt/response/tool/session/diff evidence harvested to a private durable remote evidence pack.",
449
- },
450
- raw_evidence_pointers: options.facts.pointers,
451
- },
452
- ],
453
- diagnostic_labels: [
454
- `pack_id:${options.facts.pack_id}`,
455
- `files:${options.facts.file_count}`,
456
- `bytes:${options.facts.byte_size}`,
457
- `skipped:${options.facts.skipped_count}`,
458
- `reused:${options.facts.reused_count}`,
459
- `stage_state:${options.facts.stage_state}`,
460
- `staged_new:${options.facts.staged_new_count}`,
461
- `staged_reused:${options.facts.staged_reused_count}`,
462
- `delivery_held:${options.facts.delivery_held_count}`,
463
- `completeness:${options.facts.evidence_completeness.status}`,
464
- `truncated:${options.facts.evidence_completeness.totals.truncated_count}`,
465
- `deferred:${options.facts.evidence_completeness.totals.deferred_count}`,
466
- `failed:${options.facts.evidence_completeness.totals.failed_count}`,
467
- ...options.facts.content_kinds.map((kind) => `kind:${kind}`),
468
- ],
469
- });
118
+ if (options.includeClaudeJsonl !== false && options.claudeSessionFiles) {
119
+ await collectClaudeJsonlFiles(collection, options.claudeSessionFiles);
120
+ }
121
+ await collectGitDiffFiles(collection, options.repoRoot);
470
122
  }
471
123
  // ---------------------------------------------------------------------------
472
124
  // What attribution already knew
@@ -483,765 +135,4 @@ function recordAttributionCompleteness(collection, options) {
483
135
  if (options.claudeAttributionScan) {
484
136
  recordClaudeAttributionCompleteness(collection, options.claudeAttributionScan, new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []));
485
137
  }
486
- }
487
- function recordCodexAttributionCompleteness(collection, scan, selectedPaths) {
488
- recordScanned(collection, "codex_attribution", scan.scanned_file_count);
489
- collection.caps.push({
490
- source: "codex_attribution",
491
- cap_type: "scan_window_minutes",
492
- limit: scan.since_minutes,
493
- observed: scan.since_minutes,
494
- applied: false,
495
- }, {
496
- source: "codex_attribution",
497
- cap_type: "session_limit",
498
- limit: scan.session_limit,
499
- observed: scan.discovered_file_count,
500
- applied: scan.session_limit_applied,
501
- });
502
- recordSkipCount(collection, "codex_attribution", "session_limit_overflow", Math.max(0, scan.discovered_file_count - scan.scanned_file_count));
503
- recordSkipCount(collection, "codex_attribution", "directory_read_failed", scan.directory_read_failed_count);
504
- recordSkipCount(collection, "codex_attribution", "file_stat_failed", scan.stat_failed_count);
505
- recordSkipCount(collection, "codex_attribution", "secret_like_directory", scan.secret_path_skipped_count);
506
- recordAttributionResultSkips(collection, "codex_attribution", scan.results, selectedPaths);
507
- }
508
- function recordClaudeAttributionCompleteness(collection, scan, selectedPaths) {
509
- recordScanned(collection, "claude_attribution", scan.scanned_session_count);
510
- if (scan.disabled_reason) {
511
- recordSkipCount(collection, "claude_attribution", scan.disabled_reason, 1);
512
- }
513
- collection.caps.push({
514
- source: "claude_attribution",
515
- cap_type: "scan_window_minutes",
516
- limit: scan.since_minutes,
517
- observed: scan.since_minutes,
518
- applied: false,
519
- }, {
520
- source: "claude_attribution",
521
- cap_type: "session_limit",
522
- limit: scan.session_limit,
523
- observed: scan.discovered_session_count,
524
- applied: scan.session_limit_applied,
525
- }, {
526
- source: "claude_attribution",
527
- cap_type: "max_file_bytes",
528
- limit: scan.max_file_bytes,
529
- applied: scan.counts.mains_oversized > 0 ||
530
- scan.results.some((result) => result.reason === "file_too_large"),
531
- }, {
532
- source: "claude_attribution",
533
- cap_type: "max_sidecar_files",
534
- limit: scan.max_sidecar_files,
535
- observed: scan.max_sidecar_files + scan.counts.sidecars_capped,
536
- applied: scan.counts.sidecars_capped > 0,
537
- }, {
538
- source: "claude_attribution",
539
- cap_type: "max_line_buffer_bytes",
540
- limit: scan.max_line_buffer_bytes,
541
- applied: scan.counts.oversized_lines_skipped > 0,
542
- });
543
- recordSkipCount(collection, "claude_attribution", "session_limit_overflow", Math.max(0, scan.discovered_session_count - scan.scanned_session_count));
544
- recordSkipCount(collection, "claude_attribution", "secret_like_project_dir", scan.project_dirs_skipped);
545
- recordSkipCount(collection, "claude_attribution", "project_dir_read_failed", scan.project_dir_read_failed_count);
546
- recordSkipCount(collection, "claude_attribution", "session_stat_failed", scan.session_stat_failed_count);
547
- recordSkipCount(collection, "claude_attribution", "sidecar_dir_read_failed", scan.sidecar_dir_read_failed_count);
548
- recordSkipCount(collection, "claude_attribution", "sidecar_stat_failed", scan.sidecar_stat_failed_count);
549
- recordSkipCount(collection, "claude_attribution", "sidecar_limit_overflow", scan.counts.sidecars_capped);
550
- recordTruncationCount(collection, "claude_attribution", "oversized_jsonl_line", scan.counts.oversized_lines_skipped, { max_bytes: scan.max_line_buffer_bytes });
551
- recordAttributionResultSkips(collection, "claude_attribution", scan.results, selectedPaths);
552
- for (const result of scan.results) {
553
- for (const sidecar of result.sidecar_files) {
554
- if (!sidecar.skipped_reason)
555
- continue;
556
- recordSkipCount(collection, "claude_attribution", `sidecar_${sidecar.skipped_reason}`, 1);
557
- }
558
- }
559
- }
560
- function recordAttributionResultSkips(collection, source, results, selectedPaths) {
561
- for (const result of results) {
562
- if (isAttributionAccountedFor(result, selectedPaths))
563
- continue;
564
- const reason = result.state === "skipped"
565
- ? result.reason
566
- : `attribution_${result.state}:${result.reason}`;
567
- recordSkipCount(collection, source, reason, 1);
568
- }
569
- }
570
- /**
571
- * Not every unattributed result is a gap. A selected session is being collected
572
- * by this pass, and a live-sync-safe synthetic target is owned by another
573
- * workspace's pack — neither is missing evidence.
574
- */
575
- function isAttributionAccountedFor(result, selectedPaths) {
576
- if (result.state === "attributed")
577
- return true;
578
- if (selectedPaths.has(result.file_path))
579
- return true;
580
- return (result.worktree !== null &&
581
- isLiveRawEvidenceSyncAttribution(result.state, result.reason, true));
582
- }
583
- // ---------------------------------------------------------------------------
584
- // Codex transcripts
585
- // ---------------------------------------------------------------------------
586
- async function collectCodexJsonlFiles(collection, options) {
587
- const attributed = Boolean(options.codexSessionFiles);
588
- const resolvedCandidates = options.codexSessionFiles
589
- ? options.codexSessionFiles.map((file) => ({
590
- filePath: file.local_path,
591
- codexSessionId: file.codex_session_id,
592
- }))
593
- : (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
594
- collection.caps.push({
595
- source: "codex_jsonl",
596
- cap_type: "session_limit",
597
- limit: options.limit,
598
- observed: resolvedCandidates.length,
599
- applied: !attributed && resolvedCandidates.length > options.limit,
600
- });
601
- const candidates = attributed
602
- ? resolvedCandidates
603
- : resolvedCandidates.slice(0, options.limit);
604
- for (const candidate of candidates) {
605
- recordScanned(collection, "codex_jsonl");
606
- const codexSessionId = candidate.codexSessionId ?? shortHash(candidate.filePath);
607
- const transcriptAccepted = await collectOneEvidenceFile(collection, {
608
- filePath: candidate.filePath,
609
- kind: "codex_jsonl",
610
- sessionId: codexSessionId,
611
- mediaType: "application/jsonl",
612
- // Guard before read/toString: Codex files can exceed the buffered commit
613
- // ceiling just like Claude mains.
614
- maxFileBytes: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
615
- redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
616
- contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
617
- });
618
- if (!transcriptAccepted)
619
- continue;
620
- await collectAgentImagesFromTranscript(collection, {
621
- filePath: candidate.filePath,
622
- source: "codex",
623
- sessionId: codexSessionId,
624
- kind: "codex_image_attachment",
625
- contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
626
- });
627
- }
628
- }
629
- /** The unattributed fallback: every recent Codex transcript on this machine. */
630
- function walkRecentCodexJsonlFiles(collection, options) {
631
- const dirs = options.sessionsDirs ??
632
- (options.sessionsDir
633
- ? [options.sessionsDir]
634
- : defaultCodexSessionDirs(os.homedir()));
635
- return walkJsonlFiles(dirs, collection.context.now.getTime() - options.sinceMinutes * 60 * 1000);
636
- }
637
- // ---------------------------------------------------------------------------
638
- // Claude transcripts
639
- // ---------------------------------------------------------------------------
640
- async function collectClaudeJsonlFiles(collection, sessions) {
641
- recordScanned(collection, "claude_jsonl", sessions.reduce((count, session) => count + 1 + session.sidecar_files.length, 0));
642
- for (const session of sessions) {
643
- await collectOneClaudeSession(collection, session);
644
- }
645
- }
646
- async function collectOneClaudeSession(collection, session) {
647
- const sessionId = session.claude_session_id;
648
- const mainOutcome = await collectClaudeMainFile(collection, session);
649
- if (mainOutcome === "collected" || mainOutcome === "damped_reuse") {
650
- await collectAgentImagesFromTranscript(collection, {
651
- filePath: session.local_path,
652
- source: "claude_code",
653
- sessionId,
654
- kind: "claude_image_attachment",
655
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
656
- });
657
- }
658
- for (const sidecar of session.sidecar_files) {
659
- await collectOneClaudeSidecar(collection, session, sidecar.local_path);
660
- }
661
- }
662
- /**
663
- * What happens to a Claude main file, as a decision table:
664
- *
665
- * | condition | main bytes | outcome |
666
- * | --------------------- | ------------------------- | ------------------- |
667
- * | `main_file_oversized` | skipped `file_too_large` | `skipped_too_large` |
668
- * | `skip_main` (D9) | not re-collected | `damped_reuse` |
669
- * | otherwise | collected, or named skip | `collected` / `not_collected` |
670
- *
671
- * Its images are collected for every outcome except `skipped_too_large` and
672
- * `not_collected` — the caller decides that, this function only reports.
673
- *
674
- * D7: an oversized main was attributed via a streamed read but its bytes are
675
- * never uploaded (server commit assembles in memory). Its sidecars still
676
- * collect. D9 damped: the prior durable copy is still good enough, and the
677
- * session reports `reused_existing` from cursor state, so no skip is recorded.
678
- */
679
- async function collectClaudeMainFile(collection, session) {
680
- if (session.main_file_oversized) {
681
- collection.skipped.push({
682
- kind: "claude_jsonl",
683
- label: path.basename(session.local_path),
684
- reason: "file_too_large",
685
- });
686
- return "skipped_too_large";
687
- }
688
- if (session.skip_main)
689
- return "damped_reuse";
690
- const accepted = await collectOneEvidenceFile(collection, {
691
- filePath: session.local_path,
692
- kind: "claude_jsonl",
693
- sessionId: session.claude_session_id,
694
- mediaType: "application/jsonl",
695
- // Re-check size at collection: a main that grew past the cap between
696
- // attribution and collection is an honest file_too_large skip, not an
697
- // upload_failed at the chunk client.
698
- maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
699
- redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
700
- contentAddress: (hash16) => `claude/${safeKeySegment(session.claude_session_id)}/${hash16}.jsonl`,
701
- });
702
- return accepted ? "collected" : "not_collected";
703
- }
704
- async function collectOneClaudeSidecar(collection, session, sidecarPath) {
705
- const sessionId = session.claude_session_id;
706
- const stem = path.basename(sidecarPath).replace(/\.jsonl$/i, "");
707
- const safeSidecarId = isSecretLikePath(stem)
708
- ? "redacted-file-name"
709
- : safeKeySegment(stem);
710
- const sidecarAccepted = await collectOneEvidenceFile(collection, {
711
- filePath: sidecarPath,
712
- kind: "claude_jsonl_sidecar",
713
- sessionId,
714
- mediaType: "application/jsonl",
715
- maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
716
- redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
717
- contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
718
- });
719
- if (!sidecarAccepted)
720
- return;
721
- await collectAgentImagesFromTranscript(collection, {
722
- filePath: sidecarPath,
723
- source: "claude_code",
724
- sessionId,
725
- sidecarId: safeSidecarId,
726
- kind: "claude_image_attachment",
727
- contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
728
- });
729
- }
730
- async function collectAgentImagesFromTranscript(collection, options) {
731
- const result = await collectAgentImageEvidenceFromJsonlFile({
732
- filePath: options.filePath,
733
- source: options.source,
734
- sessionId: options.sessionId,
735
- sidecarId: options.sidecarId,
736
- });
737
- recordScanned(collection, options.kind, result.images.length + result.skipped.length);
738
- for (const skipped of result.skipped) {
739
- collection.skipped.push({
740
- kind: options.kind,
741
- label: skipped.label,
742
- reason: skipped.reason,
743
- });
744
- }
745
- for (const image of result.images) {
746
- await collectOneAgentImageFile(collection, {
747
- image,
748
- kind: options.kind,
749
- sessionId: options.sessionId,
750
- contentAddress: options.contentAddress,
751
- });
752
- }
753
- }
754
- async function collectOneAgentImageFile(collection, options) {
755
- const raw = options.image.bytes;
756
- const contentHash = sha256(raw);
757
- const metadata = {
758
- ...options.image.metadata,
759
- content_hash_sha256: contentHash,
760
- byte_size: raw.byteLength,
761
- };
762
- if (collection.skipContentHashes.has(contentHash)) {
763
- collection.reused.push({
764
- kind: options.kind,
765
- label: options.image.label,
766
- content_hash_sha256: contentHash,
767
- codex_session_id: options.sessionId,
768
- artifact_metadata: metadata,
769
- });
770
- return;
771
- }
772
- const deferReason = admitToBudget(collection.budget, raw.byteLength);
773
- if (deferReason) {
774
- markBudgetCapApplied(collection, deferReason);
775
- collection.skipped.push({
776
- kind: options.kind,
777
- label: options.image.label,
778
- reason: deferReason,
779
- });
780
- return;
781
- }
782
- collection.index.value += 1;
783
- const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-agent-image-${contentHash.slice(0, 16)}.${options.image.extension}`);
784
- const imageSourceKey = evidenceSourceKey({
785
- kind: options.kind,
786
- sessionId: options.sessionId,
787
- label: options.image.label,
788
- });
789
- const staged = await stageEvidenceBytes(collection, {
790
- contentHash,
791
- bytes: raw,
792
- fileName: path.basename(relativePath),
793
- kind: options.kind,
794
- sourceKey: imageSourceKey,
795
- });
796
- collection.entries.push(evidenceEntry({
797
- context: collection.context,
798
- kind: options.kind,
799
- packId: collection.packId,
800
- localPath: staged.local_path,
801
- relativePath,
802
- stagedInPack: staged.staged_in_pack,
803
- sourceKey: imageSourceKey,
804
- mediaType: metadata.media_type,
805
- redactedSummary: "Raw image explicitly attached to an agent session, preserved in private durable storage.",
806
- bytes: raw,
807
- codexSessionId: options.sessionId,
808
- contentAddress: options.contentAddress(contentHash.slice(0, 16), options.image.extension),
809
- artifactMetadata: metadata,
810
- }));
811
- }
812
- // ---------------------------------------------------------------------------
813
- // One transcript, end to end
814
- // ---------------------------------------------------------------------------
815
- /**
816
- * Reads, secret-guards, content-addresses, budget-checks, and copies one
817
- * attributed transcript into the pack.
818
- *
819
- * Returns whether this file's *content* is accounted for, which is not the same
820
- * as "was staged": a transcript whose bytes are already durable remotely
821
- * returns true without staging anything, and its images are still worth
822
- * collecting. Every false return has pushed a named skip first.
823
- */
824
- async function collectOneEvidenceFile(collection, options) {
825
- const fileName = path.basename(options.filePath);
826
- const secretLikeFileName = isSecretLikePath(fileName);
827
- const evidenceLabel = secretLikeFileName ? "[REDACTED_FILE_NAME]" : fileName;
828
- const packedFileName = secretLikeFileName ? "redacted-file.jsonl" : fileName;
829
- const sourceKey = evidenceSourceKey({
830
- kind: options.kind,
831
- sessionId: options.sessionId,
832
- sourcePath: options.filePath,
833
- });
834
- const skip = (reason) => {
835
- collection.skipped.push({
836
- kind: options.kind,
837
- label: evidenceLabel,
838
- reason,
839
- });
840
- return false;
841
- };
842
- // Delivery backoff is checked BEFORE the file is read. An object whose commit
843
- // has failed repeatedly costs nothing at all this cycle — no read, no hash,
844
- // no copy, no request — and the hold is a named, retryable gap so a held
845
- // session cannot make the sync look clean (BLI-3066).
846
- if (collection.heldSources.has(sourceKey)) {
847
- collection.deliveryHeldCount += 1;
848
- skip(DELIVERY_BACKOFF_HOLDING_REASON);
849
- console.error("[raw-evidence] delivery backoff holding source", JSON.stringify({
850
- reason: DELIVERY_BACKOFF_HOLDING_REASON,
851
- kind: options.kind,
852
- source_key: sourceKey,
853
- }));
854
- return false;
855
- }
856
- const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
857
- if (read.status === "read_failed")
858
- return skip("file_read_failed");
859
- if (read.status === "too_large") {
860
- markCapApplied(collection, options.kind, "max_file_bytes");
861
- return skip("file_too_large");
862
- }
863
- const raw = read.bytes;
864
- const sanitized = sanitizeTextEvidenceForUpload({
865
- text: raw.toString("utf8"),
866
- originalBytes: raw,
867
- redactedFields: [`${options.kind}.body`],
868
- secretLikeFileName,
869
- });
870
- if (sanitized.status === "redacted") {
871
- collection.redacted.push({
872
- kind: options.kind,
873
- label: evidenceLabel,
874
- redaction: sanitized.redaction,
875
- completenessLabel: sanitized.completenessLabel,
876
- });
877
- console.error("[raw-evidence] text evidence sanitized", JSON.stringify({
878
- kind: options.kind,
879
- mode: sanitized.completenessLabel,
880
- original_bytes: raw.byteLength,
881
- uploaded_bytes: sanitized.bytes.byteLength,
882
- }));
883
- }
884
- const evidenceBytes = sanitized.bytes;
885
- // Both branches carry a record now (BLI-3277), so "was anything replaced?" is
886
- // the status, never the presence of `redaction`.
887
- const redaction = sanitized.redaction;
888
- const wasRedacted = sanitized.status === "redacted";
889
- const contentHash = sha256(evidenceBytes);
890
- if (collection.skipContentHashes.has(contentHash)) {
891
- collection.reused.push({
892
- kind: options.kind,
893
- label: evidenceLabel,
894
- content_hash_sha256: contentHash,
895
- codex_session_id: options.sessionId,
896
- });
897
- return true;
898
- }
899
- const deferReason = admitToBudget(collection.budget, evidenceBytes.byteLength);
900
- if (deferReason) {
901
- markBudgetCapApplied(collection, deferReason);
902
- return skip(deferReason);
903
- }
904
- collection.index.value += 1;
905
- const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(options.filePath)}-${packedFileName}`);
906
- const staged = await stageEvidenceBytes(collection, {
907
- contentHash,
908
- bytes: evidenceBytes,
909
- fileName: path.basename(relativePath),
910
- kind: options.kind,
911
- sourceKey,
912
- });
913
- collection.entries.push(evidenceEntry({
914
- context: collection.context,
915
- kind: options.kind,
916
- packId: collection.packId,
917
- localPath: staged.local_path,
918
- relativePath,
919
- mediaType: options.mediaType,
920
- redactedSummary: wasRedacted
921
- ? `${options.redactedSummary} Secret-like values were deterministically redacted before upload.`
922
- : options.redactedSummary,
923
- redaction,
924
- bytes: evidenceBytes,
925
- codexSessionId: options.sessionId,
926
- contentAddress: options.contentAddress(contentHash.slice(0, 16)),
927
- stagedInPack: staged.staged_in_pack,
928
- sourceKey,
929
- }));
930
- return true;
931
- }
932
- /**
933
- * Size is checked twice on purpose: once by `stat` so an oversized transcript
934
- * is never buffered at all, and once on the bytes actually read, because a live
935
- * session can grow between the two.
936
- */
937
- async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
938
- if (maxFileBytes) {
939
- let stat;
940
- try {
941
- stat = await fs.stat(filePath);
942
- }
943
- catch (error) {
944
- // The caller turns this into the `file_read_failed` skip label, which is
945
- // the one label an operator can do nothing with. The file was discovered
946
- // moments ago, so a failure here is a rotated session, a permission
947
- // problem or a dead symlink — three different answers (BLI-3238).
948
- console.error("[raw-evidence] evidence file could not be stat'd", JSON.stringify({
949
- reason: "file_read_failed",
950
- stage: "stat",
951
- ...describeError(error),
952
- }));
953
- return { status: "read_failed" };
954
- }
955
- if (stat.size > maxFileBytes)
956
- return { status: "too_large" };
957
- }
958
- let bytes;
959
- try {
960
- bytes = await fs.readFile(filePath);
961
- }
962
- catch (error) {
963
- console.error("[raw-evidence] evidence file could not be read", JSON.stringify({
964
- reason: "file_read_failed",
965
- stage: "read",
966
- ...describeError(error),
967
- }));
968
- return { status: "read_failed" };
969
- }
970
- if (maxFileBytes && bytes.byteLength > maxFileBytes) {
971
- return { status: "too_large" };
972
- }
973
- return { status: "ok", bytes };
974
- }
975
- // ---------------------------------------------------------------------------
976
- // Staging and budgets
977
- // ---------------------------------------------------------------------------
978
- /**
979
- * Put these bytes on disk once.
980
- *
981
- * If an earlier sync already staged this exact content and the copy is still
982
- * there, that copy is used — the uploader only needs a readable path, and it
983
- * does not care which pack directory holds it. This is the branch that stops
984
- * one 334 MB rollout from becoming 559 copies while its commit keeps failing.
985
- */
986
- async function stageEvidenceBytes(collection, options) {
987
- const existing = await resolveStagedObject(collection.rawEvidenceRoot, collection.staging, options.contentHash);
988
- if (existing) {
989
- collection.stagedReusedCount += 1;
990
- console.error("[raw-evidence] staged copy reused", JSON.stringify({
991
- reason: "staged_reused",
992
- kind: options.kind,
993
- content_hash_prefix: options.contentHash.slice(0, 16),
994
- byte_size: existing.entry.byte_size,
995
- pack_id: existing.entry.pack_id,
996
- }));
997
- return { local_path: existing.local_path, staged_in_pack: false };
998
- }
999
- const destination = path.join(collection.filesDir, options.fileName);
1000
- await fs.writeFile(destination, options.bytes, { mode: 0o600 });
1001
- await chmodPrivate(destination, 0o600);
1002
- collection.stagedNewCount += 1;
1003
- return { local_path: destination, staged_in_pack: true };
1004
- }
1005
- /**
1006
- * Decrements the per-sync budget when a file fits, or returns a deferred-skip
1007
- * reason when it does not. The object budget bounds request count; the byte
1008
- * budget bounds buffered bytes.
1009
- */
1010
- function admitToBudget(budget, byteLength) {
1011
- if (budget.remainingObjects <= 0)
1012
- return "deferred_object_budget";
1013
- if (byteLength > budget.remainingBytes)
1014
- return "deferred_byte_budget";
1015
- budget.remainingObjects -= 1;
1016
- budget.remainingBytes -= byteLength;
1017
- return null;
1018
- }
1019
- // ---------------------------------------------------------------------------
1020
- // Git diffs
1021
- // ---------------------------------------------------------------------------
1022
- const GIT_DIFF_TARGETS = [
1023
- { label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
1024
- { label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
1025
- ];
1026
- async function collectGitDiffFiles(collection, repoRoot) {
1027
- // BLI-3551: asked once per root, before either target, so a pruned worktree
1028
- // costs ONE line instead of one per diff target per session per tick — and
1029
- // the line says the folder is gone rather than accusing git of failing.
1030
- if (!(await repoRootExists(repoRoot))) {
1031
- console.error("[raw-evidence] git diff skipped, repository root is no longer on disk", JSON.stringify({
1032
- reason: REPO_ROOT_MISSING_REASON,
1033
- diff_targets_skipped: GIT_DIFF_TARGETS.length,
1034
- next_action: "nothing to do; the diff returns when the worktree is restored or the session ages out",
1035
- }));
1036
- for (const target of GIT_DIFF_TARGETS) {
1037
- recordScanned(collection, "git_diff");
1038
- collection.skipped.push({
1039
- kind: "git_diff",
1040
- label: target.label,
1041
- reason: REPO_ROOT_MISSING_REASON,
1042
- });
1043
- }
1044
- return;
1045
- }
1046
- for (const target of GIT_DIFF_TARGETS) {
1047
- recordScanned(collection, "git_diff");
1048
- let diff;
1049
- try {
1050
- diff = await runGitDiff(target.args, repoRoot);
1051
- }
1052
- catch (error) {
1053
- // The root was there a moment ago and is not now (or a second collector
1054
- // pruned it mid-tick). Same named outcome, still not a git failure.
1055
- if (error instanceof RepoRootMissingError) {
1056
- collection.skipped.push({
1057
- kind: "git_diff",
1058
- label: target.label,
1059
- reason: REPO_ROOT_MISSING_REASON,
1060
- });
1061
- continue;
1062
- }
1063
- // `git_diff_failed` is the skip label and stays. It covers git not being
1064
- // installed, the folder not being a repo, a locked index and a diff that
1065
- // exceeded the child-process buffer — and the diff is half the evidence
1066
- // for what someone actually changed, so losing it quietly matters.
1067
- console.error("[raw-evidence] git diff failed", JSON.stringify({
1068
- reason: "git_diff_failed",
1069
- diff_target: target.label,
1070
- ...describeError(error),
1071
- }));
1072
- collection.skipped.push({
1073
- kind: "git_diff",
1074
- label: target.label,
1075
- reason: "git_diff_failed",
1076
- });
1077
- continue;
1078
- }
1079
- if (diff.truncated) {
1080
- markCapApplied(collection, "git_diff", diff.truncationCapType);
1081
- collection.truncated.push({
1082
- kind: "git_diff",
1083
- reason: diff.truncationReason,
1084
- ...(diff.truncationCapType === "max_bytes_per_diff"
1085
- ? { max_bytes: MAX_GIT_DIFF_BYTES }
1086
- : {}),
1087
- observed_bytes: diff.observedBytes,
1088
- included_bytes: Buffer.byteLength(diff.stdout, "utf8"),
1089
- });
1090
- }
1091
- // An empty diff is not a gap: there was simply nothing to record. The
1092
- // truncation marker above still stands even when zero bytes survived.
1093
- if (!diff.stdout.trim())
1094
- continue;
1095
- await stageOneGitDiff(collection, {
1096
- label: target.label,
1097
- diffText: diff.stdout,
1098
- truncated: diff.truncated,
1099
- });
1100
- }
1101
- }
1102
- async function stageOneGitDiff(collection, target) {
1103
- const sanitized = sanitizeTextEvidenceForUpload({
1104
- text: target.diffText,
1105
- redactedFields: [`git_diff.${target.label}`],
1106
- });
1107
- if (sanitized.status === "redacted") {
1108
- collection.redacted.push({
1109
- kind: "git_diff",
1110
- label: target.label,
1111
- redaction: sanitized.redaction,
1112
- completenessLabel: sanitized.completenessLabel,
1113
- });
1114
- console.error("[raw-evidence] git diff sanitized", JSON.stringify({
1115
- mode: sanitized.completenessLabel,
1116
- original_bytes: Buffer.byteLength(target.diffText, "utf8"),
1117
- uploaded_bytes: sanitized.bytes.byteLength,
1118
- }));
1119
- }
1120
- const raw = sanitized.bytes;
1121
- const redaction = sanitized.redaction;
1122
- const wasRedacted = sanitized.status === "redacted";
1123
- const contentHash = sha256(raw);
1124
- if (collection.skipContentHashes.has(contentHash)) {
1125
- collection.reused.push({
1126
- kind: "git_diff",
1127
- label: target.label,
1128
- content_hash_sha256: contentHash,
1129
- codex_session_id: null,
1130
- });
1131
- return;
1132
- }
1133
- const deferReason = admitToBudget(collection.budget, raw.byteLength);
1134
- if (deferReason) {
1135
- markBudgetCapApplied(collection, deferReason);
1136
- collection.skipped.push({
1137
- kind: "git_diff",
1138
- label: target.label,
1139
- reason: deferReason,
1140
- });
1141
- return;
1142
- }
1143
- const relativePath = path.join("files", `git-${target.label}.diff`);
1144
- const diffSourceKey = evidenceSourceKey({
1145
- kind: "git_diff",
1146
- sessionId: collection.context.workContextId,
1147
- label: target.label,
1148
- });
1149
- const staged = await stageEvidenceBytes(collection, {
1150
- contentHash,
1151
- bytes: raw,
1152
- fileName: path.basename(relativePath),
1153
- kind: "git_diff",
1154
- sourceKey: diffSourceKey,
1155
- });
1156
- collection.entries.push(evidenceEntry({
1157
- context: collection.context,
1158
- kind: "git_diff",
1159
- packId: collection.packId,
1160
- localPath: staged.local_path,
1161
- relativePath,
1162
- stagedInPack: staged.staged_in_pack,
1163
- sourceKey: diffSourceKey,
1164
- mediaType: "text/x-diff",
1165
- redactedSummary: gitDiffSummary(target.label, {
1166
- redacted: wasRedacted,
1167
- truncated: target.truncated,
1168
- }),
1169
- redaction,
1170
- bytes: raw,
1171
- contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
1172
- }));
1173
- }
1174
- function gitDiffSummary(label, state) {
1175
- if (state.redacted) {
1176
- return `Raw git ${label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`;
1177
- }
1178
- if (state.truncated) {
1179
- return `Raw git ${label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`;
1180
- }
1181
- return `Raw git ${label} diff preserved locally with env/secret paths excluded.`;
1182
- }
1183
- // ---------------------------------------------------------------------------
1184
- // Walking the session store
1185
- // ---------------------------------------------------------------------------
1186
- /**
1187
- * Newest-first `.jsonl` files under these directories, modified since `cutoffMs`.
1188
- *
1189
- * Secret-like directory and file names are never descended into or opened;
1190
- * symlinked duplicates are collapsed by real path so one transcript reachable
1191
- * two ways is collected once.
1192
- */
1193
- async function walkJsonlFiles(dir, cutoffMs) {
1194
- const out = [];
1195
- const stack = Array.isArray(dir) ? [...dir] : [dir];
1196
- const seen = new Set();
1197
- // Counted rather than logged per directory: a wide walk can hit many, and
1198
- // the useful signal is "N directories in the session store were skipped and
1199
- // here is the first reason", not N near-identical lines (BLI-3238).
1200
- let unreadableDirCount = 0;
1201
- let firstUnreadableDir = null;
1202
- while (stack.length > 0) {
1203
- const current = stack.pop();
1204
- if (!current || isSecretLikePath(current))
1205
- continue;
1206
- let entries;
1207
- try {
1208
- entries = await fs.readdir(current, { withFileTypes: true });
1209
- }
1210
- catch (error) {
1211
- // A directory that cannot be listed hides every session under it, and
1212
- // the walk's only visible effect is a smaller file count.
1213
- unreadableDirCount += 1;
1214
- firstUnreadableDir ??= describeError(error);
1215
- continue;
1216
- }
1217
- for (const entry of entries) {
1218
- const full = path.join(current, entry.name);
1219
- if (isSecretLikePath(full))
1220
- continue;
1221
- if (entry.isDirectory()) {
1222
- stack.push(full);
1223
- continue;
1224
- }
1225
- if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
1226
- continue;
1227
- const stat = await fs.stat(full);
1228
- if (stat.mtimeMs < cutoffMs)
1229
- continue;
1230
- const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
1231
- if (seen.has(dedupeKey))
1232
- continue;
1233
- seen.add(dedupeKey);
1234
- out.push({ file: full, mtimeMs: stat.mtimeMs });
1235
- }
1236
- }
1237
- if (unreadableDirCount > 0) {
1238
- console.error("[raw-evidence] session-store directories skipped during the walk", JSON.stringify({
1239
- reason: "session_dir_unreadable",
1240
- unreadable_dir_count: unreadableDirCount,
1241
- found_file_count: out.length,
1242
- ...firstUnreadableDir,
1243
- }));
1244
- }
1245
- out.sort((a, b) => b.mtimeMs - a.mtimeMs);
1246
- return out.map((entry) => entry.file);
1247
138
  }