@bli-cockpit/cli 0.2.49 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-claude-reader.js +108 -0
- package/dist/adapters/raw-evidence-codex-reader.js +147 -0
- package/dist/adapters/raw-evidence-collection-state.js +199 -0
- package/dist/adapters/raw-evidence-facts.js +338 -0
- package/dist/adapters/raw-evidence-git-diff-reader.js +187 -0
- package/dist/adapters/raw-evidence-image-reader.js +107 -0
- package/dist/adapters/raw-evidence-sanitize.js +56 -0
- package/dist/adapters/raw-evidence-transcript-file.js +182 -0
- package/dist/adapters/raw-evidence.js +63 -1183
- package/dist/commands/backfill-batches.js +34 -0
- package/dist/commands/backfill-candidates.js +54 -0
- package/dist/commands/backfill-checkpoint.js +101 -0
- package/dist/commands/backfill-command-line.js +70 -0
- package/dist/commands/backfill-evidence-outcomes.js +104 -0
- package/dist/commands/backfill-issues.js +265 -0
- package/dist/commands/backfill-output.js +75 -0
- package/dist/commands/backfill-plan.js +71 -0
- package/dist/commands/backfill-reasons.js +107 -0
- package/dist/commands/backfill-report.js +298 -0
- package/dist/commands/backfill-result.js +150 -0
- package/dist/commands/backfill-scan.js +274 -0
- package/dist/commands/backfill-scope.js +114 -0
- package/dist/commands/backfill-session-report.js +145 -0
- package/dist/commands/backfill-types.js +1 -0
- package/dist/commands/backfill-upload.js +212 -0
- package/dist/commands/backfill.js +41 -1961
- package/dist/commands/doctor.js +57 -0
- package/dist/commands/jarvis-trace.js +184 -0
- package/dist/commands/jarvis.js +144 -4
- package/dist/commands/local-args-collector.js +26 -0
- package/dist/commands/local-args-tower.js +21 -0
- package/dist/commands/local-args.js +3 -1
- package/dist/commands/local-help.js +19 -2
- package/dist/commands/local.js +3 -0
- package/dist/commands/memory-install-claude.js +294 -0
- package/dist/commands/memory-install-codex.js +205 -0
- package/dist/commands/memory-install-contract.js +231 -0
- package/dist/commands/memory-install-files.js +63 -0
- package/dist/commands/memory-install-skills.js +121 -0
- package/dist/commands/memory-install-toml.js +265 -0
- package/dist/commands/memory-install.js +378 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-followups.js +105 -0
- package/dist/commands/sync.js +7 -1
- package/dist/local-state-attributed-target.js +75 -0
- package/dist/local-state-config.js +147 -0
- package/dist/local-state-files.js +59 -0
- package/dist/local-state-identity.js +73 -0
- package/dist/local-state-pairing.js +263 -0
- package/dist/local-state-paths.js +61 -0
- package/dist/local-state-session.js +68 -0
- package/dist/local-state-status.js +163 -0
- package/dist/local-state-work-context.js +190 -0
- package/dist/local-state.js +34 -848
- package/dist/tower-client.js +3 -2
- package/dist/tower-stream.js +57 -3
- package/package.json +2 -1
|
@@ -1,35 +1,71 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Collect one repo's raw evidence into a private, content-keyed pack.
|
|
3
|
+
*
|
|
4
|
+
* This is the content guard between "a transcript exists on this laptop" and
|
|
5
|
+
* "these bytes may become durable evidence". Attribution before it only reads
|
|
6
|
+
* metadata; the server commit after it is the second guard.
|
|
7
|
+
*
|
|
8
|
+
* The pass, in order:
|
|
9
|
+
*
|
|
10
|
+
* 1. **Open a staging directory.** The pack's id comes from its content, so it
|
|
11
|
+
* cannot be known yet (BLI-3066). Bytes land in `.staging-<pid>-<rand>`.
|
|
12
|
+
* 2. **Record what attribution already knew** — every session it scanned and
|
|
13
|
+
* every one it could not name, as completeness records, before a single byte
|
|
14
|
+
* is read.
|
|
15
|
+
* 3. **Collect** every source: Codex transcripts, Claude transcripts and their
|
|
16
|
+
* subagent sidecars, images explicitly attached to either, and the two git
|
|
17
|
+
* diffs.
|
|
18
|
+
* 4. **Finish** one of three ways — empty, promoted, or failed — each reporting
|
|
19
|
+
* the same facts, pointers and scan.
|
|
20
|
+
*
|
|
21
|
+
* Three doctrines hold everywhere in this family:
|
|
22
|
+
*
|
|
23
|
+
* - **Nothing gets in the way of collecting a session.** Redaction masks and
|
|
24
|
+
* continues; it never drops a file.
|
|
25
|
+
* - **A failure must name itself.** Every path that declines a file records a
|
|
26
|
+
* reason label in the gap ledger.
|
|
27
|
+
* - **Never succeed silently either.** Staging, reuse, holds and failures all
|
|
28
|
+
* log metadata — ids, sizes, counts, reason labels. Never content, never a
|
|
29
|
+
* local path.
|
|
30
|
+
*
|
|
31
|
+
* Who owns what (BLI-3586 moved the bodies out; every public name is still
|
|
32
|
+
* exported from this file, so no consumer's import changed):
|
|
33
|
+
*
|
|
34
|
+
* | module | owns |
|
|
35
|
+
* | ------------------------------- | --------------------------------------- |
|
|
36
|
+
* | `raw-evidence-collection-state` | what a pass carries, its budget, staging bytes |
|
|
37
|
+
* | `raw-evidence-codex-reader` | Codex transcripts, and the capped fallback walk |
|
|
38
|
+
* | `raw-evidence-claude-reader` | Claude mains, sidecars, the D7/D9 outcomes |
|
|
39
|
+
* | `raw-evidence-image-reader` | images attached to either transcript |
|
|
40
|
+
* | `raw-evidence-git-diff-reader` | the unstaged and staged diffs |
|
|
41
|
+
* | `raw-evidence-transcript-file` | one transcript, end to end |
|
|
42
|
+
* | `raw-evidence-sanitize` | masking, and the receipt it leaves |
|
|
43
|
+
* | `raw-evidence-facts` | the three endings, the pointers, the scan |
|
|
44
|
+
* | `raw-evidence-completeness` | the gap ledger |
|
|
45
|
+
* | `raw-evidence-manifest` | what we say about a file |
|
|
46
|
+
* | `raw-evidence-pack-store` | where the bytes live |
|
|
47
|
+
* | `raw-evidence-keys` | naming |
|
|
48
|
+
* | `raw-evidence-git-diff` | running git |
|
|
49
|
+
* | `raw-evidence-attribution-gaps` | step 2's scan results as ledger records |
|
|
50
|
+
*
|
|
51
|
+
* The readers deliberately share one shape — derive the names, guard, mask and
|
|
52
|
+
* record, hash, check reuse, check budget, then a single `write…ToPack` that
|
|
53
|
+
* puts the bytes down and adds the manifest entry. They do NOT share one
|
|
54
|
+
* implementation, because each pushes a different ledger row and logs a
|
|
55
|
+
* different line, and merging them would change what an operator reads.
|
|
56
|
+
*/
|
|
5
57
|
import path from "node:path";
|
|
6
|
-
import { makeSourceAdapterIdentity, } from "./common.js";
|
|
7
|
-
import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
|
|
8
|
-
import { defaultCodexSessionDirs, } from "./codex-attribution.js";
|
|
9
58
|
import { recordClaudeAttributionCompleteness, recordCodexAttributionCompleteness, } from "./raw-evidence-attribution-gaps.js";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import { chmodPrivate, ensurePrivateDir, persistStagingState, promoteStagedPack, stageManifest, } from "./raw-evidence-pack-store.js";
|
|
17
|
-
import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, REPO_ROOT_MISSING_REASON, RepoRootMissingError, repoRootExists, runGitDiff, } from "./raw-evidence-git-diff.js";
|
|
59
|
+
import { ensurePrivateDir } from "./raw-evidence-pack-store.js";
|
|
60
|
+
import { DEFAULT_SESSION_LIMIT, DEFAULT_SINCE_MINUTES, newStagingDir, openCollection, } from "./raw-evidence-collection-state.js";
|
|
61
|
+
import { finishWithEmptyPack, finishWithFailedPack, finishWithPromotedPack, } from "./raw-evidence-facts.js";
|
|
62
|
+
import { collectCodexJsonlFiles } from "./raw-evidence-codex-reader.js";
|
|
63
|
+
import { collectClaudeJsonlFiles } from "./raw-evidence-claude-reader.js";
|
|
64
|
+
import { collectGitDiffFiles } from "./raw-evidence-git-diff-reader.js";
|
|
18
65
|
// Re-exported so every consumer keeps importing from `adapters/raw-evidence`.
|
|
19
66
|
export { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./raw-evidence-manifest.js";
|
|
20
67
|
export { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
|
|
21
|
-
|
|
22
|
-
const DEFAULT_SESSION_LIMIT = 50;
|
|
23
|
-
// Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
|
|
24
|
-
// transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
|
|
25
|
-
// so 2 GiB leaves room for several files without starving the sync; overflow
|
|
26
|
-
// still defers and converges.
|
|
27
|
-
export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
|
|
28
|
-
export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
|
|
29
|
-
const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;
|
|
30
|
-
// ---------------------------------------------------------------------------
|
|
31
|
-
// The pass
|
|
32
|
-
// ---------------------------------------------------------------------------
|
|
68
|
+
export { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET, } from "./raw-evidence-collection-state.js";
|
|
33
69
|
/**
|
|
34
70
|
* Decide which bytes on this laptop are allowed to become durable evidence for
|
|
35
71
|
* one repo, and put exactly those into one private, content-keyed pack.
|
|
@@ -64,15 +100,6 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
64
100
|
return await finishWithFailedPack(collection, { ...run, error });
|
|
65
101
|
}
|
|
66
102
|
}
|
|
67
|
-
/**
|
|
68
|
-
* Staging first, promotion second (BLI-3066). The pack id cannot be known until
|
|
69
|
-
* the content is, so bytes land in a private per-attempt directory, and that
|
|
70
|
-
* directory is then renamed to its content-keyed name — or dropped, when an
|
|
71
|
-
* identical pack is already there.
|
|
72
|
-
*/
|
|
73
|
-
function newStagingDir(rawEvidenceRoot) {
|
|
74
|
-
return path.join(rawEvidenceRoot, `.staging-${process.pid}-${crypto.randomUUID().slice(0, 8)}`);
|
|
75
|
-
}
|
|
76
103
|
/**
|
|
77
104
|
* Every kind of evidence this repo can offer, in the order it is read. Codex and
|
|
78
105
|
* Claude transcripts are each switchable off by the caller; the git diffs are
|
|
@@ -93,414 +120,6 @@ async function collectEverySource(collection, options, sinceMinutes) {
|
|
|
93
120
|
}
|
|
94
121
|
await collectGitDiffFiles(collection, options.repoRoot);
|
|
95
122
|
}
|
|
96
|
-
async function openCollection(context, options, places) {
|
|
97
|
-
const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
|
|
98
|
-
const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
|
|
99
|
-
const staging = await readRawEvidenceStagingState(options.stateDir);
|
|
100
|
-
return {
|
|
101
|
-
context,
|
|
102
|
-
filesDir: path.join(places.stagingDir, "files"),
|
|
103
|
-
rawEvidenceRoot: places.rawEvidenceRoot,
|
|
104
|
-
packId: "",
|
|
105
|
-
staging,
|
|
106
|
-
heldSources: collectionHeldSources(staging, context.now, options),
|
|
107
|
-
stagedReusedCount: 0,
|
|
108
|
-
stagedNewCount: 0,
|
|
109
|
-
deliveryHeldCount: 0,
|
|
110
|
-
entries: [],
|
|
111
|
-
skipped: [],
|
|
112
|
-
truncated: [],
|
|
113
|
-
failed: [],
|
|
114
|
-
redacted: [],
|
|
115
|
-
reused: [],
|
|
116
|
-
scanned: new Map(),
|
|
117
|
-
caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
|
|
118
|
-
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
119
|
-
budget: options.budget ?? {
|
|
120
|
-
remainingBytes: byteBudget,
|
|
121
|
-
remainingObjects: objectBudget,
|
|
122
|
-
},
|
|
123
|
-
index: { value: 0 },
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Which sources this pass refuses to even read because their delivery is inside
|
|
128
|
-
* a backoff window — none of them, when a person asked for this pass.
|
|
129
|
-
*
|
|
130
|
-
* The bypass is logged rather than assumed: an operator retry that quietly
|
|
131
|
-
* ignored a hold would be as unreadable as the hold that quietly blocked it.
|
|
132
|
-
*/
|
|
133
|
-
function collectionHeldSources(staging, now, options) {
|
|
134
|
-
const held = heldSourceKeys(staging, now);
|
|
135
|
-
if (deliveryBackoffApplies(options.deliveryMode))
|
|
136
|
-
return held;
|
|
137
|
-
if (held.size > 0) {
|
|
138
|
-
console.error("[raw-evidence] delivery backoff bypassed for operator retry", JSON.stringify({
|
|
139
|
-
reason: DELIVERY_BACKOFF_BYPASS_REASON,
|
|
140
|
-
source_count: held.size,
|
|
141
|
-
}));
|
|
142
|
-
}
|
|
143
|
-
return new Set();
|
|
144
|
-
}
|
|
145
|
-
/** Every cap this pass could hit, declared up front and flipped when applied. */
|
|
146
|
-
function startingCaps(options) {
|
|
147
|
-
return [
|
|
148
|
-
{
|
|
149
|
-
source: "raw_evidence",
|
|
150
|
-
cap_type: "byte_budget",
|
|
151
|
-
limit: options.byteBudget,
|
|
152
|
-
observed: options.budget?.remainingBytes ?? options.byteBudget,
|
|
153
|
-
applied: false,
|
|
154
|
-
},
|
|
155
|
-
{
|
|
156
|
-
source: "raw_evidence",
|
|
157
|
-
cap_type: "object_budget",
|
|
158
|
-
limit: options.objectBudget,
|
|
159
|
-
observed: options.budget?.remainingObjects ?? options.objectBudget,
|
|
160
|
-
applied: false,
|
|
161
|
-
},
|
|
162
|
-
{
|
|
163
|
-
source: "git_diff",
|
|
164
|
-
cap_type: "max_bytes_per_diff",
|
|
165
|
-
limit: MAX_GIT_DIFF_BYTES,
|
|
166
|
-
applied: false,
|
|
167
|
-
},
|
|
168
|
-
{
|
|
169
|
-
source: "git_diff",
|
|
170
|
-
cap_type: "timeout_ms",
|
|
171
|
-
limit: GIT_DIFF_TIMEOUT_MS,
|
|
172
|
-
applied: false,
|
|
173
|
-
},
|
|
174
|
-
{
|
|
175
|
-
source: "codex_jsonl",
|
|
176
|
-
cap_type: "max_file_bytes",
|
|
177
|
-
limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
178
|
-
applied: false,
|
|
179
|
-
},
|
|
180
|
-
{
|
|
181
|
-
source: "claude_jsonl",
|
|
182
|
-
cap_type: "max_file_bytes",
|
|
183
|
-
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
184
|
-
applied: false,
|
|
185
|
-
},
|
|
186
|
-
{
|
|
187
|
-
source: "claude_jsonl_sidecar",
|
|
188
|
-
cap_type: "max_file_bytes",
|
|
189
|
-
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
190
|
-
applied: false,
|
|
191
|
-
},
|
|
192
|
-
];
|
|
193
|
-
}
|
|
194
|
-
// ---------------------------------------------------------------------------
|
|
195
|
-
// The three ways a pass ends
|
|
196
|
-
// ---------------------------------------------------------------------------
|
|
197
|
-
/**
|
|
198
|
-
* Nothing collected: drop the staging directory instead of leaving an empty
|
|
199
|
-
* pack behind. This used to leave one empty `work-*` dir per sync.
|
|
200
|
-
*/
|
|
201
|
-
async function finishWithEmptyPack(collection, options, run) {
|
|
202
|
-
await discardStagingDir(run.stagingDir);
|
|
203
|
-
collection.packId = contentKeyedRawEvidencePackId({
|
|
204
|
-
workContextId: collection.context.workContextId,
|
|
205
|
-
contentHashes: [],
|
|
206
|
-
});
|
|
207
|
-
await persistStagingState(options.stateDir, collection.staging, collection.context.now.toISOString());
|
|
208
|
-
const facts = makeEmptyPackFacts(collection, run.window);
|
|
209
|
-
return {
|
|
210
|
-
facts,
|
|
211
|
-
scan: makeRawEvidenceScan({
|
|
212
|
-
context: collection.context,
|
|
213
|
-
startedAt: run.window.startedAt,
|
|
214
|
-
status: "partial",
|
|
215
|
-
facts,
|
|
216
|
-
}),
|
|
217
|
-
};
|
|
218
|
-
}
|
|
219
|
-
async function finishWithPromotedPack(collection, options, run) {
|
|
220
|
-
const context = collection.context;
|
|
221
|
-
const entries = collection.entries;
|
|
222
|
-
// The pack is named by what is in it, never by when it was made. Identical
|
|
223
|
-
// content on the next sync resolves to the identical directory, which is
|
|
224
|
-
// the whole fix for BLI-3066.
|
|
225
|
-
const packId = contentKeyedRawEvidencePackId({
|
|
226
|
-
workContextId: context.workContextId,
|
|
227
|
-
contentHashes: entries.map((entry) => entry.content_hash_sha256),
|
|
228
|
-
});
|
|
229
|
-
collection.packId = packId;
|
|
230
|
-
const promotion = await promoteStagedPack({
|
|
231
|
-
rawEvidenceRoot: collection.rawEvidenceRoot,
|
|
232
|
-
stagingDir: run.stagingDir,
|
|
233
|
-
packId,
|
|
234
|
-
workContextId: context.workContextId,
|
|
235
|
-
entries,
|
|
236
|
-
});
|
|
237
|
-
rebaseStagedEntriesOntoPack(collection, promotion.evidenceDir);
|
|
238
|
-
const manifestPath = await addManifestToPack(collection, promotion, packId);
|
|
239
|
-
await persistStagingState(options.stateDir, collection.staging, context.now.toISOString());
|
|
240
|
-
logPackStaged(collection, promotion, packId);
|
|
241
|
-
const facts = makePromotedPackFacts(collection, {
|
|
242
|
-
packId,
|
|
243
|
-
manifestPath,
|
|
244
|
-
promotion,
|
|
245
|
-
window: run.window,
|
|
246
|
-
});
|
|
247
|
-
return {
|
|
248
|
-
facts,
|
|
249
|
-
scan: makeRawEvidenceScan({
|
|
250
|
-
context,
|
|
251
|
-
startedAt: run.window.startedAt,
|
|
252
|
-
// A pack of nothing but its own manifest carries no evidence, so it
|
|
253
|
-
// reports partial however cleanly it was written.
|
|
254
|
-
status: entries.length > 1 ? "ok" : "partial",
|
|
255
|
-
facts,
|
|
256
|
-
}),
|
|
257
|
-
};
|
|
258
|
-
}
|
|
259
|
-
/** A pack with files: the same shell as an empty one, plus what is in it. */
|
|
260
|
-
function makePromotedPackFacts(collection, place) {
|
|
261
|
-
const entries = collection.entries;
|
|
262
|
-
return {
|
|
263
|
-
...packFactsShell(collection, {
|
|
264
|
-
packId: place.packId,
|
|
265
|
-
manifestPath: place.manifestPath,
|
|
266
|
-
evidenceDir: place.promotion.evidenceDir,
|
|
267
|
-
window: place.window,
|
|
268
|
-
}),
|
|
269
|
-
stage_state: place.promotion.state,
|
|
270
|
-
file_count: entries.length,
|
|
271
|
-
byte_size: totalByteSize(entries),
|
|
272
|
-
content_kinds: [...new Set(entries.map((entry) => entry.kind))],
|
|
273
|
-
pointers: entries.map(pointerFromEntry),
|
|
274
|
-
upload_files: entries.map(uploadFileFromEntry),
|
|
275
|
-
};
|
|
276
|
-
}
|
|
277
|
-
/**
|
|
278
|
-
* Write the manifest and make it an object of the pack like any other file, so
|
|
279
|
-
* the uploader has one list to walk. Returns where it landed.
|
|
280
|
-
*/
|
|
281
|
-
async function addManifestToPack(collection, promotion, packId) {
|
|
282
|
-
const context = collection.context;
|
|
283
|
-
const manifestPath = path.join(promotion.evidenceDir, "manifest.json");
|
|
284
|
-
const manifestBytes = await stageManifest({
|
|
285
|
-
context,
|
|
286
|
-
packId,
|
|
287
|
-
manifestPath,
|
|
288
|
-
entries: collection.entries,
|
|
289
|
-
skipped: collection.skipped,
|
|
290
|
-
redacted: collection.redacted,
|
|
291
|
-
reused: collection.reused,
|
|
292
|
-
reusePack: promotion.state === "reused",
|
|
293
|
-
});
|
|
294
|
-
collection.entries.push(evidenceEntry({
|
|
295
|
-
context,
|
|
296
|
-
kind: "manifest",
|
|
297
|
-
packId,
|
|
298
|
-
localPath: manifestPath,
|
|
299
|
-
relativePath: "manifest.json",
|
|
300
|
-
mediaType: "application/json",
|
|
301
|
-
redactedSummary: "Local raw evidence pack manifest.",
|
|
302
|
-
bytes: manifestBytes,
|
|
303
|
-
// The manifest carries its own content hash in its key, exactly like
|
|
304
|
-
// every other object in the pack (BLI-3552). Without it the key was
|
|
305
|
-
// `…/<packId>/manifest.json`, and a pack id is a function of the OTHER
|
|
306
|
-
// files' hashes — not of the manifest's bytes, which also carry
|
|
307
|
-
// `created_at`, the ordering-dependent `files[].relative_path`, `branch`,
|
|
308
|
-
// and the skipped/redacted/reused ledgers. So the same key named
|
|
309
|
-
// different bytes whenever the pack directory was not adopted verbatim
|
|
310
|
-
// (a refill, a pruned or wiped state dir, a machine that had never seen
|
|
311
|
-
// the pack). `begin` then answered `hash_mismatch_committed_object` on
|
|
312
|
-
// every sync forever, because the new manifest is stable and the old one
|
|
313
|
-
// is durable. `stageManifest`'s byte-stability trick still stands; it is
|
|
314
|
-
// now a nice-to-have rather than the only thing between us and a loop.
|
|
315
|
-
contentAddress: `manifest/${sha256(manifestBytes).slice(0, 16)}.json`,
|
|
316
|
-
}));
|
|
317
|
-
return manifestPath;
|
|
318
|
-
}
|
|
319
|
-
/** The success line. A sync that staged nothing new still has to say so. */
|
|
320
|
-
function logPackStaged(collection, promotion, packId) {
|
|
321
|
-
console.error("[raw-evidence] pack staged", JSON.stringify({
|
|
322
|
-
pack_id: packId,
|
|
323
|
-
stage_state: promotion.state,
|
|
324
|
-
reason: stageReasonLabel(promotion.state, promotion.priorPackCount),
|
|
325
|
-
prior_pack_count: promotion.priorPackCount,
|
|
326
|
-
refilled_file_count: promotion.refilledFileCount,
|
|
327
|
-
file_count: collection.entries.length,
|
|
328
|
-
byte_size: totalByteSize(collection.entries),
|
|
329
|
-
staged_new: collection.stagedNewCount,
|
|
330
|
-
staged_reused: collection.stagedReusedCount,
|
|
331
|
-
delivery_held: collection.deliveryHeldCount,
|
|
332
|
-
}));
|
|
333
|
-
}
|
|
334
|
-
/**
|
|
335
|
-
* A crashed pass. The staged-object index is deliberately NOT written: this
|
|
336
|
-
* attempt's bookkeeping describes a directory that is about to be deleted.
|
|
337
|
-
*/
|
|
338
|
-
async function finishWithFailedPack(collection, run) {
|
|
339
|
-
collection.failed.push({
|
|
340
|
-
kind: "raw_evidence",
|
|
341
|
-
reason: "collection_failed",
|
|
342
|
-
});
|
|
343
|
-
// Staging is per-attempt scratch: a crashed pass must not leave a partial
|
|
344
|
-
// directory behind to be counted, re-hashed or swept later.
|
|
345
|
-
await discardStagingDir(run.stagingDir);
|
|
346
|
-
collection.packId =
|
|
347
|
-
collection.packId ||
|
|
348
|
-
contentKeyedRawEvidencePackId({
|
|
349
|
-
workContextId: collection.context.workContextId,
|
|
350
|
-
contentHashes: collection.entries.map((entry) => entry.content_hash_sha256),
|
|
351
|
-
});
|
|
352
|
-
console.error("[raw-evidence] pack collection failed", JSON.stringify({
|
|
353
|
-
pack_id: collection.packId,
|
|
354
|
-
reason: "collection_failed",
|
|
355
|
-
detail: run.error instanceof Error ? run.error.name : typeof run.error,
|
|
356
|
-
collected_file_count: collection.entries.length,
|
|
357
|
-
staged_new: collection.stagedNewCount,
|
|
358
|
-
staged_reused: collection.stagedReusedCount,
|
|
359
|
-
}));
|
|
360
|
-
const facts = makeEmptyPackFacts(collection, run.window);
|
|
361
|
-
return {
|
|
362
|
-
facts,
|
|
363
|
-
scan: makeRawEvidenceScan({
|
|
364
|
-
context: collection.context,
|
|
365
|
-
startedAt: run.window.startedAt,
|
|
366
|
-
status: "failed",
|
|
367
|
-
facts,
|
|
368
|
-
}),
|
|
369
|
-
};
|
|
370
|
-
}
|
|
371
|
-
/**
|
|
372
|
-
* A pack with no files of its own: the empty pass and the crashed pass report
|
|
373
|
-
* the same shape, differing only in the scan status and the reason already
|
|
374
|
-
* logged. `evidence_completeness` still carries every gap this pass recorded.
|
|
375
|
-
*/
|
|
376
|
-
function makeEmptyPackFacts(collection, scanWindow) {
|
|
377
|
-
const evidenceDir = path.join(collection.rawEvidenceRoot, collection.packId);
|
|
378
|
-
return {
|
|
379
|
-
...packFactsShell(collection, {
|
|
380
|
-
packId: collection.packId,
|
|
381
|
-
manifestPath: path.join(evidenceDir, "manifest.json"),
|
|
382
|
-
evidenceDir,
|
|
383
|
-
window: scanWindow,
|
|
384
|
-
}),
|
|
385
|
-
stage_state: "empty",
|
|
386
|
-
file_count: 0,
|
|
387
|
-
byte_size: 0,
|
|
388
|
-
content_kinds: [],
|
|
389
|
-
pointers: [],
|
|
390
|
-
upload_files: [],
|
|
391
|
-
};
|
|
392
|
-
}
|
|
393
|
-
/** The counts every ending reports identically, however the pass ended. */
|
|
394
|
-
function packFactsShell(collection, place) {
|
|
395
|
-
return {
|
|
396
|
-
pack_id: place.packId,
|
|
397
|
-
manifest_path: place.manifestPath,
|
|
398
|
-
evidence_dir: place.evidenceDir,
|
|
399
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
400
|
-
skipped_count: countEvidenceEntries(collection.skipped),
|
|
401
|
-
sanitized_count: collection.redacted.length,
|
|
402
|
-
reused_count: collection.reused.length,
|
|
403
|
-
staged_reused_count: collection.stagedReusedCount,
|
|
404
|
-
staged_new_count: collection.stagedNewCount,
|
|
405
|
-
delivery_held_count: collection.deliveryHeldCount,
|
|
406
|
-
deferred_byte_budget_count: countDeferred(collection, "deferred_byte_budget"),
|
|
407
|
-
deferred_object_budget_count: countDeferred(collection, "deferred_object_budget"),
|
|
408
|
-
evidence_completeness: makeEvidenceCompleteness(collection, {
|
|
409
|
-
startedAt: place.window.startedAt,
|
|
410
|
-
finishedAt: place.window.finishedAt(),
|
|
411
|
-
sinceMinutes: place.window.sinceMinutes,
|
|
412
|
-
}),
|
|
413
|
-
reused: collection.reused,
|
|
414
|
-
};
|
|
415
|
-
}
|
|
416
|
-
function countDeferred(collection, reason) {
|
|
417
|
-
return countEvidenceEntries(collection.skipped, (entry) => entry.reason === reason);
|
|
418
|
-
}
|
|
419
|
-
function totalByteSize(entries) {
|
|
420
|
-
return entries.reduce((sum, entry) => sum + entry.byte_size, 0);
|
|
421
|
-
}
|
|
422
|
-
function uploadFileFromEntry(entry) {
|
|
423
|
-
return {
|
|
424
|
-
pointer: pointerFromEntry(entry),
|
|
425
|
-
local_path: entry.local_path,
|
|
426
|
-
kind: entry.kind,
|
|
427
|
-
codex_session_id: entry.codex_session_id ?? null,
|
|
428
|
-
...(entry.artifact_metadata
|
|
429
|
-
? { artifact_metadata: entry.artifact_metadata }
|
|
430
|
-
: {}),
|
|
431
|
-
};
|
|
432
|
-
}
|
|
433
|
-
function stageReasonLabel(state, priorPackCount) {
|
|
434
|
-
if (state === "reused")
|
|
435
|
-
return "staged_reused";
|
|
436
|
-
if (state === "restaged_incomplete")
|
|
437
|
-
return "restaged_incomplete";
|
|
438
|
-
return priorPackCount > 0 ? "restaged_content_changed" : "staged_new";
|
|
439
|
-
}
|
|
440
|
-
/**
|
|
441
|
-
* Point every entry this pass staged at its home in the promoted pack, and
|
|
442
|
-
* remember the content hash so the next sync can adopt the copy instead of
|
|
443
|
-
* writing it again. Entries adopted from another pack keep their path.
|
|
444
|
-
*/
|
|
445
|
-
function rebaseStagedEntriesOntoPack(collection, evidenceDir) {
|
|
446
|
-
for (const entry of collection.entries) {
|
|
447
|
-
if (!entry.staged_in_pack)
|
|
448
|
-
continue;
|
|
449
|
-
entry.local_path = path.join(evidenceDir, "files", path.basename(entry.local_path));
|
|
450
|
-
recordStagedObject(collection.staging, entry.content_hash_sha256, {
|
|
451
|
-
pack_id: collection.packId,
|
|
452
|
-
relative_path: `files/${path.basename(entry.local_path)}`,
|
|
453
|
-
byte_size: entry.byte_size,
|
|
454
|
-
source_key: entry.source_key,
|
|
455
|
-
staged_at: collection.context.now.toISOString(),
|
|
456
|
-
});
|
|
457
|
-
}
|
|
458
|
-
}
|
|
459
|
-
async function discardStagingDir(stagingDir) {
|
|
460
|
-
await fs
|
|
461
|
-
.rm(stagingDir, { recursive: true, force: true })
|
|
462
|
-
.catch(() => undefined);
|
|
463
|
-
}
|
|
464
|
-
function makeRawEvidenceScan(options) {
|
|
465
|
-
return SourceScanResultSchema.parse({
|
|
466
|
-
adapter: makeSourceAdapterIdentity("collector_runtime", "raw-evidence-pack"),
|
|
467
|
-
work_context_id: options.context.workContextId,
|
|
468
|
-
status: options.status,
|
|
469
|
-
started_at: options.startedAt,
|
|
470
|
-
finished_at: options.context.now.toISOString(),
|
|
471
|
-
events: [
|
|
472
|
-
{
|
|
473
|
-
source_event_id: `raw-evidence-pack:${options.facts.pack_id}`,
|
|
474
|
-
event_type: "raw_evidence_pack_written",
|
|
475
|
-
occurred_at: options.context.now.toISOString(),
|
|
476
|
-
redaction: {
|
|
477
|
-
privacy_classification: "remote_durable_raw_evidence",
|
|
478
|
-
redaction_status: "raw_remote_durable",
|
|
479
|
-
redacted_fields: ["local_path"],
|
|
480
|
-
raw_evidence_pointer_ids: options.facts.pointers.map((pointer) => pointer.raw_evidence_pointer_id),
|
|
481
|
-
redacted_summary: "Raw prompt/response/tool/session/diff evidence harvested to a private durable remote evidence pack.",
|
|
482
|
-
},
|
|
483
|
-
raw_evidence_pointers: options.facts.pointers,
|
|
484
|
-
},
|
|
485
|
-
],
|
|
486
|
-
diagnostic_labels: [
|
|
487
|
-
`pack_id:${options.facts.pack_id}`,
|
|
488
|
-
`files:${options.facts.file_count}`,
|
|
489
|
-
`bytes:${options.facts.byte_size}`,
|
|
490
|
-
`skipped:${options.facts.skipped_count}`,
|
|
491
|
-
`reused:${options.facts.reused_count}`,
|
|
492
|
-
`stage_state:${options.facts.stage_state}`,
|
|
493
|
-
`staged_new:${options.facts.staged_new_count}`,
|
|
494
|
-
`staged_reused:${options.facts.staged_reused_count}`,
|
|
495
|
-
`delivery_held:${options.facts.delivery_held_count}`,
|
|
496
|
-
`completeness:${options.facts.evidence_completeness.status}`,
|
|
497
|
-
`truncated:${options.facts.evidence_completeness.totals.truncated_count}`,
|
|
498
|
-
`deferred:${options.facts.evidence_completeness.totals.deferred_count}`,
|
|
499
|
-
`failed:${options.facts.evidence_completeness.totals.failed_count}`,
|
|
500
|
-
...options.facts.content_kinds.map((kind) => `kind:${kind}`),
|
|
501
|
-
],
|
|
502
|
-
});
|
|
503
|
-
}
|
|
504
123
|
// ---------------------------------------------------------------------------
|
|
505
124
|
// What attribution already knew
|
|
506
125
|
// ---------------------------------------------------------------------------
|
|
@@ -516,743 +135,4 @@ function recordAttributionCompleteness(collection, options) {
|
|
|
516
135
|
if (options.claudeAttributionScan) {
|
|
517
136
|
recordClaudeAttributionCompleteness(collection, options.claudeAttributionScan, new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []));
|
|
518
137
|
}
|
|
519
|
-
}
|
|
520
|
-
// ---------------------------------------------------------------------------
|
|
521
|
-
// Codex transcripts
|
|
522
|
-
// ---------------------------------------------------------------------------
|
|
523
|
-
async function collectCodexJsonlFiles(collection, options) {
|
|
524
|
-
for (const candidate of await chooseCodexCandidates(collection, options)) {
|
|
525
|
-
recordScanned(collection, "codex_jsonl");
|
|
526
|
-
await collectOneCodexSession(collection, candidate);
|
|
527
|
-
}
|
|
528
|
-
}
|
|
529
|
-
/**
|
|
530
|
-
* Which Codex transcripts this pass will look at, and the record of how many it
|
|
531
|
-
* had to leave out. Attribution's own list is never trimmed — the session limit
|
|
532
|
-
* only guards the fallback walk, which can turn up every session on the machine.
|
|
533
|
-
*/
|
|
534
|
-
async function chooseCodexCandidates(collection, options) {
|
|
535
|
-
const attributed = Boolean(options.codexSessionFiles);
|
|
536
|
-
const candidates = options.codexSessionFiles
|
|
537
|
-
? options.codexSessionFiles.map((file) => ({
|
|
538
|
-
filePath: file.local_path,
|
|
539
|
-
codexSessionId: file.codex_session_id,
|
|
540
|
-
}))
|
|
541
|
-
: (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
|
|
542
|
-
collection.caps.push({
|
|
543
|
-
source: "codex_jsonl",
|
|
544
|
-
cap_type: "session_limit",
|
|
545
|
-
limit: options.limit,
|
|
546
|
-
observed: candidates.length,
|
|
547
|
-
applied: !attributed && candidates.length > options.limit,
|
|
548
|
-
});
|
|
549
|
-
return attributed ? candidates : candidates.slice(0, options.limit);
|
|
550
|
-
}
|
|
551
|
-
/** The transcript, then the images it explicitly attached — never the reverse. */
|
|
552
|
-
async function collectOneCodexSession(collection, candidate) {
|
|
553
|
-
const codexSessionId = candidate.codexSessionId ?? shortHash(candidate.filePath);
|
|
554
|
-
const transcriptAccepted = await collectOneEvidenceFile(collection, {
|
|
555
|
-
filePath: candidate.filePath,
|
|
556
|
-
kind: "codex_jsonl",
|
|
557
|
-
sessionId: codexSessionId,
|
|
558
|
-
mediaType: "application/jsonl",
|
|
559
|
-
// Guard before read/toString: Codex files can exceed the buffered commit
|
|
560
|
-
// ceiling just like Claude mains.
|
|
561
|
-
maxFileBytes: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
562
|
-
redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
563
|
-
contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
|
|
564
|
-
});
|
|
565
|
-
if (!transcriptAccepted)
|
|
566
|
-
return;
|
|
567
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
568
|
-
filePath: candidate.filePath,
|
|
569
|
-
source: "codex",
|
|
570
|
-
sessionId: codexSessionId,
|
|
571
|
-
kind: "codex_image_attachment",
|
|
572
|
-
contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
|
|
573
|
-
});
|
|
574
|
-
}
|
|
575
|
-
/** The unattributed fallback: every recent Codex transcript on this machine. */
|
|
576
|
-
function walkRecentCodexJsonlFiles(collection, options) {
|
|
577
|
-
const dirs = options.sessionsDirs ??
|
|
578
|
-
(options.sessionsDir
|
|
579
|
-
? [options.sessionsDir]
|
|
580
|
-
: defaultCodexSessionDirs(os.homedir()));
|
|
581
|
-
return walkJsonlFiles(dirs, collection.context.now.getTime() - options.sinceMinutes * 60 * 1000);
|
|
582
|
-
}
|
|
583
|
-
// ---------------------------------------------------------------------------
|
|
584
|
-
// Claude transcripts
|
|
585
|
-
// ---------------------------------------------------------------------------
|
|
586
|
-
async function collectClaudeJsonlFiles(collection, sessions) {
|
|
587
|
-
recordScanned(collection, "claude_jsonl", sessions.reduce((count, session) => count + 1 + session.sidecar_files.length, 0));
|
|
588
|
-
for (const session of sessions) {
|
|
589
|
-
await collectOneClaudeSession(collection, session);
|
|
590
|
-
}
|
|
591
|
-
}
|
|
592
|
-
async function collectOneClaudeSession(collection, session) {
|
|
593
|
-
const sessionId = session.claude_session_id;
|
|
594
|
-
const mainOutcome = await collectClaudeMainFile(collection, session);
|
|
595
|
-
if (mainOutcome === "collected" || mainOutcome === "damped_reuse") {
|
|
596
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
597
|
-
filePath: session.local_path,
|
|
598
|
-
source: "claude_code",
|
|
599
|
-
sessionId,
|
|
600
|
-
kind: "claude_image_attachment",
|
|
601
|
-
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
|
|
602
|
-
});
|
|
603
|
-
}
|
|
604
|
-
for (const sidecar of session.sidecar_files) {
|
|
605
|
-
await collectOneClaudeSidecar(collection, session, sidecar.local_path);
|
|
606
|
-
}
|
|
607
|
-
}
|
|
608
|
-
/**
|
|
609
|
-
* What happens to a Claude main file, as a decision table:
|
|
610
|
-
*
|
|
611
|
-
* | condition | main bytes | outcome |
|
|
612
|
-
* | --------------------- | ------------------------- | ------------------- |
|
|
613
|
-
* | `main_file_oversized` | skipped `file_too_large` | `skipped_too_large` |
|
|
614
|
-
* | `skip_main` (D9) | not re-collected | `damped_reuse` |
|
|
615
|
-
* | otherwise | collected, or named skip | `collected` / `not_collected` |
|
|
616
|
-
*
|
|
617
|
-
* Its images are collected for every outcome except `skipped_too_large` and
|
|
618
|
-
* `not_collected` — the caller decides that, this function only reports.
|
|
619
|
-
*
|
|
620
|
-
* D7: an oversized main was attributed via a streamed read but its bytes are
|
|
621
|
-
* never uploaded (server commit assembles in memory). Its sidecars still
|
|
622
|
-
* collect. D9 damped: the prior durable copy is still good enough, and the
|
|
623
|
-
* session reports `reused_existing` from cursor state, so no skip is recorded.
|
|
624
|
-
*/
|
|
625
|
-
async function collectClaudeMainFile(collection, session) {
|
|
626
|
-
if (session.main_file_oversized) {
|
|
627
|
-
collection.skipped.push({
|
|
628
|
-
kind: "claude_jsonl",
|
|
629
|
-
label: path.basename(session.local_path),
|
|
630
|
-
reason: "file_too_large",
|
|
631
|
-
});
|
|
632
|
-
return "skipped_too_large";
|
|
633
|
-
}
|
|
634
|
-
if (session.skip_main)
|
|
635
|
-
return "damped_reuse";
|
|
636
|
-
const accepted = await collectOneEvidenceFile(collection, {
|
|
637
|
-
filePath: session.local_path,
|
|
638
|
-
kind: "claude_jsonl",
|
|
639
|
-
sessionId: session.claude_session_id,
|
|
640
|
-
mediaType: "application/jsonl",
|
|
641
|
-
// Re-check size at collection: a main that grew past the cap between
|
|
642
|
-
// attribution and collection is an honest file_too_large skip, not an
|
|
643
|
-
// upload_failed at the chunk client.
|
|
644
|
-
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
645
|
-
redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
646
|
-
contentAddress: (hash16) => `claude/${safeKeySegment(session.claude_session_id)}/${hash16}.jsonl`,
|
|
647
|
-
});
|
|
648
|
-
return accepted ? "collected" : "not_collected";
|
|
649
|
-
}
|
|
650
|
-
async function collectOneClaudeSidecar(collection, session, sidecarPath) {
|
|
651
|
-
const sessionId = session.claude_session_id;
|
|
652
|
-
const stem = path.basename(sidecarPath).replace(/\.jsonl$/i, "");
|
|
653
|
-
const safeSidecarId = isSecretLikePath(stem)
|
|
654
|
-
? "redacted-file-name"
|
|
655
|
-
: safeKeySegment(stem);
|
|
656
|
-
const sidecarAccepted = await collectOneEvidenceFile(collection, {
|
|
657
|
-
filePath: sidecarPath,
|
|
658
|
-
kind: "claude_jsonl_sidecar",
|
|
659
|
-
sessionId,
|
|
660
|
-
mediaType: "application/jsonl",
|
|
661
|
-
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
662
|
-
redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
663
|
-
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
|
|
664
|
-
});
|
|
665
|
-
if (!sidecarAccepted)
|
|
666
|
-
return;
|
|
667
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
668
|
-
filePath: sidecarPath,
|
|
669
|
-
source: "claude_code",
|
|
670
|
-
sessionId,
|
|
671
|
-
sidecarId: safeSidecarId,
|
|
672
|
-
kind: "claude_image_attachment",
|
|
673
|
-
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
|
|
674
|
-
});
|
|
675
|
-
}
|
|
676
|
-
async function collectAgentImagesFromTranscript(collection, options) {
|
|
677
|
-
const result = await collectAgentImageEvidenceFromJsonlFile({
|
|
678
|
-
filePath: options.filePath,
|
|
679
|
-
source: options.source,
|
|
680
|
-
sessionId: options.sessionId,
|
|
681
|
-
sidecarId: options.sidecarId,
|
|
682
|
-
});
|
|
683
|
-
recordScanned(collection, options.kind, result.images.length + result.skipped.length);
|
|
684
|
-
for (const skipped of result.skipped) {
|
|
685
|
-
collection.skipped.push({
|
|
686
|
-
kind: options.kind,
|
|
687
|
-
label: skipped.label,
|
|
688
|
-
reason: skipped.reason,
|
|
689
|
-
});
|
|
690
|
-
}
|
|
691
|
-
for (const image of result.images) {
|
|
692
|
-
await collectOneAgentImageFile(collection, {
|
|
693
|
-
image,
|
|
694
|
-
kind: options.kind,
|
|
695
|
-
sessionId: options.sessionId,
|
|
696
|
-
contentAddress: options.contentAddress,
|
|
697
|
-
});
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
async function collectOneAgentImageFile(collection, options) {
|
|
701
|
-
const raw = options.image.bytes;
|
|
702
|
-
const contentHash = sha256(raw);
|
|
703
|
-
const metadata = {
|
|
704
|
-
...options.image.metadata,
|
|
705
|
-
content_hash_sha256: contentHash,
|
|
706
|
-
byte_size: raw.byteLength,
|
|
707
|
-
};
|
|
708
|
-
if (collection.skipContentHashes.has(contentHash)) {
|
|
709
|
-
collection.reused.push({
|
|
710
|
-
kind: options.kind,
|
|
711
|
-
label: options.image.label,
|
|
712
|
-
content_hash_sha256: contentHash,
|
|
713
|
-
codex_session_id: options.sessionId,
|
|
714
|
-
artifact_metadata: metadata,
|
|
715
|
-
});
|
|
716
|
-
return;
|
|
717
|
-
}
|
|
718
|
-
const deferReason = admitToBudget(collection.budget, raw.byteLength);
|
|
719
|
-
if (deferReason) {
|
|
720
|
-
markBudgetCapApplied(collection, deferReason);
|
|
721
|
-
collection.skipped.push({
|
|
722
|
-
kind: options.kind,
|
|
723
|
-
label: options.image.label,
|
|
724
|
-
reason: deferReason,
|
|
725
|
-
});
|
|
726
|
-
return;
|
|
727
|
-
}
|
|
728
|
-
await writeAgentImageToPack(collection, {
|
|
729
|
-
...options,
|
|
730
|
-
contentHash,
|
|
731
|
-
metadata,
|
|
732
|
-
});
|
|
733
|
-
}
|
|
734
|
-
/** Put an accepted image's bytes on disk and add its manifest entry. */
|
|
735
|
-
async function writeAgentImageToPack(collection, image) {
|
|
736
|
-
const raw = image.image.bytes;
|
|
737
|
-
collection.index.value += 1;
|
|
738
|
-
const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-agent-image-${image.contentHash.slice(0, 16)}.${image.image.extension}`);
|
|
739
|
-
const imageSourceKey = evidenceSourceKey({
|
|
740
|
-
kind: image.kind,
|
|
741
|
-
sessionId: image.sessionId,
|
|
742
|
-
label: image.image.label,
|
|
743
|
-
});
|
|
744
|
-
const staged = await stageEvidenceBytes(collection, {
|
|
745
|
-
contentHash: image.contentHash,
|
|
746
|
-
bytes: raw,
|
|
747
|
-
fileName: path.basename(relativePath),
|
|
748
|
-
kind: image.kind,
|
|
749
|
-
sourceKey: imageSourceKey,
|
|
750
|
-
});
|
|
751
|
-
collection.entries.push(evidenceEntry({
|
|
752
|
-
context: collection.context,
|
|
753
|
-
kind: image.kind,
|
|
754
|
-
packId: collection.packId,
|
|
755
|
-
localPath: staged.local_path,
|
|
756
|
-
relativePath,
|
|
757
|
-
stagedInPack: staged.staged_in_pack,
|
|
758
|
-
sourceKey: imageSourceKey,
|
|
759
|
-
mediaType: image.metadata.media_type,
|
|
760
|
-
redactedSummary: "Raw image explicitly attached to an agent session, preserved in private durable storage.",
|
|
761
|
-
bytes: raw,
|
|
762
|
-
codexSessionId: image.sessionId,
|
|
763
|
-
contentAddress: image.contentAddress(image.contentHash.slice(0, 16), image.image.extension),
|
|
764
|
-
artifactMetadata: image.metadata,
|
|
765
|
-
}));
|
|
766
|
-
}
|
|
767
|
-
// ---------------------------------------------------------------------------
|
|
768
|
-
// One transcript, end to end
|
|
769
|
-
// ---------------------------------------------------------------------------
|
|
770
|
-
/**
|
|
771
|
-
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
772
|
-
* attributed transcript into the pack.
|
|
773
|
-
*
|
|
774
|
-
* Returns whether this file's *content* is accounted for, which is not the same
|
|
775
|
-
* as "was staged": a transcript whose bytes are already durable remotely
|
|
776
|
-
* returns true without staging anything, and its images are still worth
|
|
777
|
-
* collecting. Every false return has pushed a named skip first.
|
|
778
|
-
*/
|
|
779
|
-
async function collectOneEvidenceFile(collection, options) {
|
|
780
|
-
const names = nameEvidenceFile(options);
|
|
781
|
-
const skip = (reason) => {
|
|
782
|
-
collection.skipped.push({
|
|
783
|
-
kind: options.kind,
|
|
784
|
-
label: names.evidenceLabel,
|
|
785
|
-
reason,
|
|
786
|
-
});
|
|
787
|
-
return false;
|
|
788
|
-
};
|
|
789
|
-
// Delivery backoff is checked BEFORE the file is read. An object whose commit
|
|
790
|
-
// has failed repeatedly costs nothing at all this cycle — no read, no hash,
|
|
791
|
-
// no copy, no request — and the hold is a named, retryable gap so a held
|
|
792
|
-
// session cannot make the sync look clean (BLI-3066).
|
|
793
|
-
if (collection.heldSources.has(names.sourceKey)) {
|
|
794
|
-
collection.deliveryHeldCount += 1;
|
|
795
|
-
skip(DELIVERY_BACKOFF_HOLDING_REASON);
|
|
796
|
-
logDeliveryBackoffHold(options.kind, names.sourceKey);
|
|
797
|
-
return false;
|
|
798
|
-
}
|
|
799
|
-
const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
|
|
800
|
-
if (read.status === "read_failed")
|
|
801
|
-
return skip("file_read_failed");
|
|
802
|
-
if (read.status === "too_large") {
|
|
803
|
-
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
804
|
-
return skip("file_too_large");
|
|
805
|
-
}
|
|
806
|
-
const sanitized = maskSecretsInTranscript(collection, {
|
|
807
|
-
raw: read.bytes,
|
|
808
|
-
kind: options.kind,
|
|
809
|
-
evidenceLabel: names.evidenceLabel,
|
|
810
|
-
secretLikeFileName: names.secretLikeFileName,
|
|
811
|
-
});
|
|
812
|
-
const contentHash = sha256(sanitized.bytes);
|
|
813
|
-
if (collection.skipContentHashes.has(contentHash)) {
|
|
814
|
-
collection.reused.push({
|
|
815
|
-
kind: options.kind,
|
|
816
|
-
label: names.evidenceLabel,
|
|
817
|
-
content_hash_sha256: contentHash,
|
|
818
|
-
codex_session_id: options.sessionId,
|
|
819
|
-
});
|
|
820
|
-
return true;
|
|
821
|
-
}
|
|
822
|
-
const deferReason = admitToBudget(collection.budget, sanitized.bytes.byteLength);
|
|
823
|
-
if (deferReason) {
|
|
824
|
-
markBudgetCapApplied(collection, deferReason);
|
|
825
|
-
return skip(deferReason);
|
|
826
|
-
}
|
|
827
|
-
await stageOneTranscript(collection, {
|
|
828
|
-
...options,
|
|
829
|
-
names,
|
|
830
|
-
sanitized,
|
|
831
|
-
contentHash,
|
|
832
|
-
});
|
|
833
|
-
return true;
|
|
834
|
-
}
|
|
835
|
-
function nameEvidenceFile(options) {
|
|
836
|
-
const fileName = path.basename(options.filePath);
|
|
837
|
-
const secretLikeFileName = isSecretLikePath(fileName);
|
|
838
|
-
return {
|
|
839
|
-
secretLikeFileName,
|
|
840
|
-
evidenceLabel: secretLikeFileName ? "[REDACTED_FILE_NAME]" : fileName,
|
|
841
|
-
packedFileName: secretLikeFileName ? "redacted-file.jsonl" : fileName,
|
|
842
|
-
sourceKey: evidenceSourceKey({
|
|
843
|
-
kind: options.kind,
|
|
844
|
-
sessionId: options.sessionId,
|
|
845
|
-
sourcePath: options.filePath,
|
|
846
|
-
}),
|
|
847
|
-
};
|
|
848
|
-
}
|
|
849
|
-
function logDeliveryBackoffHold(kind, sourceKey) {
|
|
850
|
-
console.error("[raw-evidence] delivery backoff holding source", JSON.stringify({
|
|
851
|
-
reason: DELIVERY_BACKOFF_HOLDING_REASON,
|
|
852
|
-
kind,
|
|
853
|
-
source_key: sourceKey,
|
|
854
|
-
}));
|
|
855
|
-
}
|
|
856
|
-
/**
|
|
857
|
-
* Mask secret-like values in a transcript and record that it happened, because
|
|
858
|
-
* masking is invisible in the uploaded bytes and an operator has to be able to
|
|
859
|
-
* see that this file went up altered.
|
|
860
|
-
*/
|
|
861
|
-
function maskSecretsInTranscript(collection, options) {
|
|
862
|
-
const sanitized = sanitizeTextEvidenceForUpload({
|
|
863
|
-
text: options.raw.toString("utf8"),
|
|
864
|
-
originalBytes: options.raw,
|
|
865
|
-
redactedFields: [`${options.kind}.body`],
|
|
866
|
-
secretLikeFileName: options.secretLikeFileName,
|
|
867
|
-
});
|
|
868
|
-
if (sanitized.status === "redacted") {
|
|
869
|
-
collection.redacted.push({
|
|
870
|
-
kind: options.kind,
|
|
871
|
-
label: options.evidenceLabel,
|
|
872
|
-
redaction: sanitized.redaction,
|
|
873
|
-
completenessLabel: sanitized.completenessLabel,
|
|
874
|
-
});
|
|
875
|
-
console.error("[raw-evidence] text evidence sanitized", JSON.stringify({
|
|
876
|
-
kind: options.kind,
|
|
877
|
-
mode: sanitized.completenessLabel,
|
|
878
|
-
original_bytes: options.raw.byteLength,
|
|
879
|
-
uploaded_bytes: sanitized.bytes.byteLength,
|
|
880
|
-
}));
|
|
881
|
-
}
|
|
882
|
-
return sanitized;
|
|
883
|
-
}
|
|
884
|
-
/** Put an accepted transcript's bytes on disk and add its manifest entry. */
|
|
885
|
-
async function stageOneTranscript(collection, file) {
|
|
886
|
-
collection.index.value += 1;
|
|
887
|
-
const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(file.filePath)}-${file.names.packedFileName}`);
|
|
888
|
-
const staged = await stageEvidenceBytes(collection, {
|
|
889
|
-
contentHash: file.contentHash,
|
|
890
|
-
bytes: file.sanitized.bytes,
|
|
891
|
-
fileName: path.basename(relativePath),
|
|
892
|
-
kind: file.kind,
|
|
893
|
-
sourceKey: file.names.sourceKey,
|
|
894
|
-
});
|
|
895
|
-
collection.entries.push(evidenceEntry({
|
|
896
|
-
context: collection.context,
|
|
897
|
-
kind: file.kind,
|
|
898
|
-
packId: collection.packId,
|
|
899
|
-
localPath: staged.local_path,
|
|
900
|
-
relativePath,
|
|
901
|
-
mediaType: file.mediaType,
|
|
902
|
-
redactedSummary: file.sanitized.status === "redacted"
|
|
903
|
-
? `${file.redactedSummary} Secret-like values were deterministically redacted before upload.`
|
|
904
|
-
: file.redactedSummary,
|
|
905
|
-
// Both branches carry a record now (BLI-3277), so "was anything
|
|
906
|
-
// replaced?" is the status, never the presence of `redaction`.
|
|
907
|
-
redaction: file.sanitized.redaction,
|
|
908
|
-
bytes: file.sanitized.bytes,
|
|
909
|
-
codexSessionId: file.sessionId,
|
|
910
|
-
contentAddress: file.contentAddress(file.contentHash.slice(0, 16)),
|
|
911
|
-
stagedInPack: staged.staged_in_pack,
|
|
912
|
-
sourceKey: file.names.sourceKey,
|
|
913
|
-
}));
|
|
914
|
-
}
|
|
915
|
-
/**
|
|
916
|
-
* Size is checked twice on purpose: once by `stat` so an oversized transcript
|
|
917
|
-
* is never buffered at all, and once on the bytes actually read, because a live
|
|
918
|
-
* session can grow between the two.
|
|
919
|
-
*/
|
|
920
|
-
async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
|
|
921
|
-
if (maxFileBytes) {
|
|
922
|
-
let stat;
|
|
923
|
-
try {
|
|
924
|
-
stat = await fs.stat(filePath);
|
|
925
|
-
}
|
|
926
|
-
catch (error) {
|
|
927
|
-
// The caller turns this into the `file_read_failed` skip label, which is
|
|
928
|
-
// the one label an operator can do nothing with. The file was discovered
|
|
929
|
-
// moments ago, so a failure here is a rotated session, a permission
|
|
930
|
-
// problem or a dead symlink — three different answers (BLI-3238).
|
|
931
|
-
console.error("[raw-evidence] evidence file could not be stat'd", JSON.stringify({
|
|
932
|
-
reason: "file_read_failed",
|
|
933
|
-
stage: "stat",
|
|
934
|
-
...describeError(error),
|
|
935
|
-
}));
|
|
936
|
-
return { status: "read_failed" };
|
|
937
|
-
}
|
|
938
|
-
if (stat.size > maxFileBytes)
|
|
939
|
-
return { status: "too_large" };
|
|
940
|
-
}
|
|
941
|
-
let bytes;
|
|
942
|
-
try {
|
|
943
|
-
bytes = await fs.readFile(filePath);
|
|
944
|
-
}
|
|
945
|
-
catch (error) {
|
|
946
|
-
console.error("[raw-evidence] evidence file could not be read", JSON.stringify({
|
|
947
|
-
reason: "file_read_failed",
|
|
948
|
-
stage: "read",
|
|
949
|
-
...describeError(error),
|
|
950
|
-
}));
|
|
951
|
-
return { status: "read_failed" };
|
|
952
|
-
}
|
|
953
|
-
if (maxFileBytes && bytes.byteLength > maxFileBytes) {
|
|
954
|
-
return { status: "too_large" };
|
|
955
|
-
}
|
|
956
|
-
return { status: "ok", bytes };
|
|
957
|
-
}
|
|
958
|
-
// ---------------------------------------------------------------------------
|
|
959
|
-
// Staging and budgets
|
|
960
|
-
// ---------------------------------------------------------------------------
|
|
961
|
-
/**
|
|
962
|
-
* Put these bytes on disk once.
|
|
963
|
-
*
|
|
964
|
-
* If an earlier sync already staged this exact content and the copy is still
|
|
965
|
-
* there, that copy is used — the uploader only needs a readable path, and it
|
|
966
|
-
* does not care which pack directory holds it. This is the branch that stops
|
|
967
|
-
* one 334 MB rollout from becoming 559 copies while its commit keeps failing.
|
|
968
|
-
*/
|
|
969
|
-
async function stageEvidenceBytes(collection, options) {
|
|
970
|
-
const existing = await resolveStagedObject(collection.rawEvidenceRoot, collection.staging, options.contentHash);
|
|
971
|
-
if (existing) {
|
|
972
|
-
collection.stagedReusedCount += 1;
|
|
973
|
-
console.error("[raw-evidence] staged copy reused", JSON.stringify({
|
|
974
|
-
reason: "staged_reused",
|
|
975
|
-
kind: options.kind,
|
|
976
|
-
content_hash_prefix: options.contentHash.slice(0, 16),
|
|
977
|
-
byte_size: existing.entry.byte_size,
|
|
978
|
-
pack_id: existing.entry.pack_id,
|
|
979
|
-
}));
|
|
980
|
-
return { local_path: existing.local_path, staged_in_pack: false };
|
|
981
|
-
}
|
|
982
|
-
const destination = path.join(collection.filesDir, options.fileName);
|
|
983
|
-
await fs.writeFile(destination, options.bytes, { mode: 0o600 });
|
|
984
|
-
await chmodPrivate(destination, 0o600);
|
|
985
|
-
collection.stagedNewCount += 1;
|
|
986
|
-
return { local_path: destination, staged_in_pack: true };
|
|
987
|
-
}
|
|
988
|
-
/**
|
|
989
|
-
* Decrements the per-sync budget when a file fits, or returns a deferred-skip
|
|
990
|
-
* reason when it does not. The object budget bounds request count; the byte
|
|
991
|
-
* budget bounds buffered bytes.
|
|
992
|
-
*/
|
|
993
|
-
function admitToBudget(budget, byteLength) {
|
|
994
|
-
if (budget.remainingObjects <= 0)
|
|
995
|
-
return "deferred_object_budget";
|
|
996
|
-
if (byteLength > budget.remainingBytes)
|
|
997
|
-
return "deferred_byte_budget";
|
|
998
|
-
budget.remainingObjects -= 1;
|
|
999
|
-
budget.remainingBytes -= byteLength;
|
|
1000
|
-
return null;
|
|
1001
|
-
}
|
|
1002
|
-
// ---------------------------------------------------------------------------
|
|
1003
|
-
// Git diffs
|
|
1004
|
-
// ---------------------------------------------------------------------------
|
|
1005
|
-
const GIT_DIFF_TARGETS = [
|
|
1006
|
-
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
1007
|
-
{ label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
|
|
1008
|
-
];
|
|
1009
|
-
async function collectGitDiffFiles(collection, repoRoot) {
|
|
1010
|
-
if (!(await repoRootExists(repoRoot))) {
|
|
1011
|
-
skipEveryDiffTargetForMissingRoot(collection);
|
|
1012
|
-
return;
|
|
1013
|
-
}
|
|
1014
|
-
for (const target of GIT_DIFF_TARGETS) {
|
|
1015
|
-
recordScanned(collection, "git_diff");
|
|
1016
|
-
const diff = await runOneGitDiff(collection, target, repoRoot);
|
|
1017
|
-
if (!diff)
|
|
1018
|
-
continue;
|
|
1019
|
-
if (diff.truncated)
|
|
1020
|
-
recordGitDiffTruncation(collection, diff);
|
|
1021
|
-
// An empty diff is not a gap: there was simply nothing to record. The
|
|
1022
|
-
// truncation marker above still stands even when zero bytes survived.
|
|
1023
|
-
if (!diff.stdout.trim())
|
|
1024
|
-
continue;
|
|
1025
|
-
await stageOneGitDiff(collection, {
|
|
1026
|
-
label: target.label,
|
|
1027
|
-
diffText: diff.stdout,
|
|
1028
|
-
truncated: diff.truncated,
|
|
1029
|
-
});
|
|
1030
|
-
}
|
|
1031
|
-
}
|
|
1032
|
-
/**
|
|
1033
|
-
* BLI-3551: the root is asked about once per pass, before either target, so a
|
|
1034
|
-
* pruned worktree costs ONE line instead of one per diff target per session per
|
|
1035
|
-
* tick — and the line says the folder is gone rather than accusing git of
|
|
1036
|
-
* failing. Both targets still get their own named gap.
|
|
1037
|
-
*/
|
|
1038
|
-
function skipEveryDiffTargetForMissingRoot(collection) {
|
|
1039
|
-
console.error("[raw-evidence] git diff skipped, repository root is no longer on disk", JSON.stringify({
|
|
1040
|
-
reason: REPO_ROOT_MISSING_REASON,
|
|
1041
|
-
diff_targets_skipped: GIT_DIFF_TARGETS.length,
|
|
1042
|
-
next_action: "nothing to do; the diff returns when the worktree is restored or the session ages out",
|
|
1043
|
-
}));
|
|
1044
|
-
for (const target of GIT_DIFF_TARGETS) {
|
|
1045
|
-
recordScanned(collection, "git_diff");
|
|
1046
|
-
collection.skipped.push({
|
|
1047
|
-
kind: "git_diff",
|
|
1048
|
-
label: target.label,
|
|
1049
|
-
reason: REPO_ROOT_MISSING_REASON,
|
|
1050
|
-
});
|
|
1051
|
-
}
|
|
1052
|
-
}
|
|
1053
|
-
/**
|
|
1054
|
-
* Run one diff, or record why it produced nothing. Null means the gap is
|
|
1055
|
-
* already named in the ledger, so the caller only has to move on.
|
|
1056
|
-
*/
|
|
1057
|
-
async function runOneGitDiff(collection, target, repoRoot) {
|
|
1058
|
-
try {
|
|
1059
|
-
return await runGitDiff(target.args, repoRoot);
|
|
1060
|
-
}
|
|
1061
|
-
catch (error) {
|
|
1062
|
-
// The root was there a moment ago and is not now (or a second collector
|
|
1063
|
-
// pruned it mid-tick). Same named outcome, still not a git failure.
|
|
1064
|
-
if (error instanceof RepoRootMissingError) {
|
|
1065
|
-
collection.skipped.push({
|
|
1066
|
-
kind: "git_diff",
|
|
1067
|
-
label: target.label,
|
|
1068
|
-
reason: REPO_ROOT_MISSING_REASON,
|
|
1069
|
-
});
|
|
1070
|
-
return null;
|
|
1071
|
-
}
|
|
1072
|
-
// `git_diff_failed` is the skip label and stays. It covers git not being
|
|
1073
|
-
// installed, the folder not being a repo, a locked index and a diff that
|
|
1074
|
-
// exceeded the child-process buffer — and the diff is half the evidence
|
|
1075
|
-
// for what someone actually changed, so losing it quietly matters.
|
|
1076
|
-
console.error("[raw-evidence] git diff failed", JSON.stringify({
|
|
1077
|
-
reason: "git_diff_failed",
|
|
1078
|
-
diff_target: target.label,
|
|
1079
|
-
...describeError(error),
|
|
1080
|
-
}));
|
|
1081
|
-
collection.skipped.push({
|
|
1082
|
-
kind: "git_diff",
|
|
1083
|
-
label: target.label,
|
|
1084
|
-
reason: "git_diff_failed",
|
|
1085
|
-
});
|
|
1086
|
-
return null;
|
|
1087
|
-
}
|
|
1088
|
-
}
|
|
1089
|
-
/** A diff that hit its size or time cap is partial evidence, and says so. */
|
|
1090
|
-
function recordGitDiffTruncation(collection, diff) {
|
|
1091
|
-
markCapApplied(collection, "git_diff", diff.truncationCapType);
|
|
1092
|
-
collection.truncated.push({
|
|
1093
|
-
kind: "git_diff",
|
|
1094
|
-
reason: diff.truncationReason,
|
|
1095
|
-
...(diff.truncationCapType === "max_bytes_per_diff"
|
|
1096
|
-
? { max_bytes: MAX_GIT_DIFF_BYTES }
|
|
1097
|
-
: {}),
|
|
1098
|
-
observed_bytes: diff.observedBytes,
|
|
1099
|
-
included_bytes: Buffer.byteLength(diff.stdout, "utf8"),
|
|
1100
|
-
});
|
|
1101
|
-
}
|
|
1102
|
-
async function stageOneGitDiff(collection, target) {
|
|
1103
|
-
const sanitized = maskSecretsInGitDiff(collection, target);
|
|
1104
|
-
const contentHash = sha256(sanitized.bytes);
|
|
1105
|
-
if (collection.skipContentHashes.has(contentHash)) {
|
|
1106
|
-
collection.reused.push({
|
|
1107
|
-
kind: "git_diff",
|
|
1108
|
-
label: target.label,
|
|
1109
|
-
content_hash_sha256: contentHash,
|
|
1110
|
-
codex_session_id: null,
|
|
1111
|
-
});
|
|
1112
|
-
return;
|
|
1113
|
-
}
|
|
1114
|
-
const deferReason = admitToBudget(collection.budget, sanitized.bytes.byteLength);
|
|
1115
|
-
if (deferReason) {
|
|
1116
|
-
markBudgetCapApplied(collection, deferReason);
|
|
1117
|
-
collection.skipped.push({
|
|
1118
|
-
kind: "git_diff",
|
|
1119
|
-
label: target.label,
|
|
1120
|
-
reason: deferReason,
|
|
1121
|
-
});
|
|
1122
|
-
return;
|
|
1123
|
-
}
|
|
1124
|
-
await writeGitDiffToPack(collection, { target, sanitized, contentHash });
|
|
1125
|
-
}
|
|
1126
|
-
/**
|
|
1127
|
-
* Mask secret-like values in a diff and record that it happened. A diff carries
|
|
1128
|
-
* whatever a person pasted into a config file, so this is the branch that most
|
|
1129
|
-
* often fires — and an operator has to see that the bytes went up altered.
|
|
1130
|
-
*/
|
|
1131
|
-
function maskSecretsInGitDiff(collection, target) {
|
|
1132
|
-
const sanitized = sanitizeTextEvidenceForUpload({
|
|
1133
|
-
text: target.diffText,
|
|
1134
|
-
redactedFields: [`git_diff.${target.label}`],
|
|
1135
|
-
});
|
|
1136
|
-
if (sanitized.status === "redacted") {
|
|
1137
|
-
collection.redacted.push({
|
|
1138
|
-
kind: "git_diff",
|
|
1139
|
-
label: target.label,
|
|
1140
|
-
redaction: sanitized.redaction,
|
|
1141
|
-
completenessLabel: sanitized.completenessLabel,
|
|
1142
|
-
});
|
|
1143
|
-
console.error("[raw-evidence] git diff sanitized", JSON.stringify({
|
|
1144
|
-
mode: sanitized.completenessLabel,
|
|
1145
|
-
original_bytes: Buffer.byteLength(target.diffText, "utf8"),
|
|
1146
|
-
uploaded_bytes: sanitized.bytes.byteLength,
|
|
1147
|
-
}));
|
|
1148
|
-
}
|
|
1149
|
-
return sanitized;
|
|
1150
|
-
}
|
|
1151
|
-
/** Put an accepted diff's bytes on disk and add its manifest entry. */
|
|
1152
|
-
async function writeGitDiffToPack(collection, diff) {
|
|
1153
|
-
const label = diff.target.label;
|
|
1154
|
-
const relativePath = path.join("files", `git-${label}.diff`);
|
|
1155
|
-
const diffSourceKey = evidenceSourceKey({
|
|
1156
|
-
kind: "git_diff",
|
|
1157
|
-
sessionId: collection.context.workContextId,
|
|
1158
|
-
label,
|
|
1159
|
-
});
|
|
1160
|
-
const staged = await stageEvidenceBytes(collection, {
|
|
1161
|
-
contentHash: diff.contentHash,
|
|
1162
|
-
bytes: diff.sanitized.bytes,
|
|
1163
|
-
fileName: path.basename(relativePath),
|
|
1164
|
-
kind: "git_diff",
|
|
1165
|
-
sourceKey: diffSourceKey,
|
|
1166
|
-
});
|
|
1167
|
-
collection.entries.push(evidenceEntry({
|
|
1168
|
-
context: collection.context,
|
|
1169
|
-
kind: "git_diff",
|
|
1170
|
-
packId: collection.packId,
|
|
1171
|
-
localPath: staged.local_path,
|
|
1172
|
-
relativePath,
|
|
1173
|
-
stagedInPack: staged.staged_in_pack,
|
|
1174
|
-
sourceKey: diffSourceKey,
|
|
1175
|
-
mediaType: "text/x-diff",
|
|
1176
|
-
redactedSummary: gitDiffSummary(label, {
|
|
1177
|
-
redacted: diff.sanitized.status === "redacted",
|
|
1178
|
-
truncated: diff.target.truncated,
|
|
1179
|
-
}),
|
|
1180
|
-
redaction: diff.sanitized.redaction,
|
|
1181
|
-
bytes: diff.sanitized.bytes,
|
|
1182
|
-
contentAddress: `git-diff/${label}-${diff.contentHash.slice(0, 16)}.diff`,
|
|
1183
|
-
}));
|
|
1184
|
-
}
|
|
1185
|
-
function gitDiffSummary(label, state) {
|
|
1186
|
-
if (state.redacted) {
|
|
1187
|
-
return `Raw git ${label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`;
|
|
1188
|
-
}
|
|
1189
|
-
if (state.truncated) {
|
|
1190
|
-
return `Raw git ${label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`;
|
|
1191
|
-
}
|
|
1192
|
-
return `Raw git ${label} diff preserved locally with env/secret paths excluded.`;
|
|
1193
|
-
}
|
|
1194
|
-
// ---------------------------------------------------------------------------
|
|
1195
|
-
// Walking the session store
|
|
1196
|
-
// ---------------------------------------------------------------------------
|
|
1197
|
-
/**
|
|
1198
|
-
* Newest-first `.jsonl` files under these directories, modified since `cutoffMs`.
|
|
1199
|
-
*
|
|
1200
|
-
* Secret-like directory and file names are never descended into or opened;
|
|
1201
|
-
* symlinked duplicates are collapsed by real path so one transcript reachable
|
|
1202
|
-
* two ways is collected once.
|
|
1203
|
-
*/
|
|
1204
|
-
async function walkJsonlFiles(dir, cutoffMs) {
|
|
1205
|
-
const out = [];
|
|
1206
|
-
const stack = Array.isArray(dir) ? [...dir] : [dir];
|
|
1207
|
-
const seen = new Set();
|
|
1208
|
-
// Counted rather than logged per directory: a wide walk can hit many, and
|
|
1209
|
-
// the useful signal is "N directories in the session store were skipped and
|
|
1210
|
-
// here is the first reason", not N near-identical lines (BLI-3238).
|
|
1211
|
-
let unreadableDirCount = 0;
|
|
1212
|
-
let firstUnreadableDir = null;
|
|
1213
|
-
while (stack.length > 0) {
|
|
1214
|
-
const current = stack.pop();
|
|
1215
|
-
if (!current || isSecretLikePath(current))
|
|
1216
|
-
continue;
|
|
1217
|
-
let entries;
|
|
1218
|
-
try {
|
|
1219
|
-
entries = await fs.readdir(current, { withFileTypes: true });
|
|
1220
|
-
}
|
|
1221
|
-
catch (error) {
|
|
1222
|
-
// A directory that cannot be listed hides every session under it, and
|
|
1223
|
-
// the walk's only visible effect is a smaller file count.
|
|
1224
|
-
unreadableDirCount += 1;
|
|
1225
|
-
firstUnreadableDir ??= describeError(error);
|
|
1226
|
-
continue;
|
|
1227
|
-
}
|
|
1228
|
-
for (const entry of entries) {
|
|
1229
|
-
const full = path.join(current, entry.name);
|
|
1230
|
-
if (isSecretLikePath(full))
|
|
1231
|
-
continue;
|
|
1232
|
-
if (entry.isDirectory()) {
|
|
1233
|
-
stack.push(full);
|
|
1234
|
-
continue;
|
|
1235
|
-
}
|
|
1236
|
-
if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
|
|
1237
|
-
continue;
|
|
1238
|
-
const stat = await fs.stat(full);
|
|
1239
|
-
if (stat.mtimeMs < cutoffMs)
|
|
1240
|
-
continue;
|
|
1241
|
-
const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
|
|
1242
|
-
if (seen.has(dedupeKey))
|
|
1243
|
-
continue;
|
|
1244
|
-
seen.add(dedupeKey);
|
|
1245
|
-
out.push({ file: full, mtimeMs: stat.mtimeMs });
|
|
1246
|
-
}
|
|
1247
|
-
}
|
|
1248
|
-
if (unreadableDirCount > 0) {
|
|
1249
|
-
console.error("[raw-evidence] session-store directories skipped during the walk", JSON.stringify({
|
|
1250
|
-
reason: "session_dir_unreadable",
|
|
1251
|
-
unreadable_dir_count: unreadableDirCount,
|
|
1252
|
-
found_file_count: out.length,
|
|
1253
|
-
...firstUnreadableDir,
|
|
1254
|
-
}));
|
|
1255
|
-
}
|
|
1256
|
-
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
1257
|
-
return out.map((entry) => entry.file);
|
|
1258
138
|
}
|