@bli-cockpit/cli 0.2.28 → 0.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -15
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/raw-evidence-completeness.js +226 -0
- package/dist/adapters/raw-evidence-git-diff.js +90 -0
- package/dist/adapters/raw-evidence-keys.js +92 -0
- package/dist/adapters/raw-evidence-manifest.js +132 -0
- package/dist/adapters/raw-evidence-pack-store.js +136 -0
- package/dist/adapters/raw-evidence-sanitize.js +190 -0
- package/dist/adapters/raw-evidence.js +656 -1257
- package/dist/commands/backfill.js +7 -0
- package/dist/commands/cli-io.js +92 -0
- package/dist/commands/collection-report.js +139 -0
- package/dist/commands/collection-roots.js +153 -0
- package/dist/commands/doctor.js +19 -17
- package/dist/commands/install-receipts.js +193 -0
- package/dist/commands/install-update.js +305 -0
- package/dist/commands/local-auth.js +268 -0
- package/dist/commands/local-discovery.js +100 -0
- package/dist/commands/local-help.js +281 -0
- package/dist/commands/local.js +182 -1872
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sessions.js +162 -0
- package/dist/commands/status.js +230 -0
- package/dist/evidence-upload-client.js +43 -2
- package/dist/raw-evidence-gc.js +1 -1
- package/dist/raw-evidence-staging.js +15 -2
- package/dist/upload-agent-artifacts.js +153 -0
- package/dist/upload-envelope.js +407 -0
- package/dist/upload-evidence-delivery.js +505 -0
- package/dist/upload-http.js +46 -0
- package/dist/upload-session-reports.js +404 -0
- package/dist/upload.js +132 -1264
- package/package.json +2 -2
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { spawn } from "node:child_process";
|
|
1
|
+
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, SourceScanResultSchema, } from "@bli-cockpit/telemetry-core";
|
|
3
2
|
import crypto from "node:crypto";
|
|
4
3
|
import fs from "node:fs/promises";
|
|
5
4
|
import os from "node:os";
|
|
@@ -8,13 +7,18 @@ import { makeSourceAdapterIdentity, } from "./common.js";
|
|
|
8
7
|
import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
|
|
9
8
|
import { defaultCodexSessionDirs, } from "./codex-attribution.js";
|
|
10
9
|
import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-policy.js";
|
|
11
|
-
import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject,
|
|
10
|
+
import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject, } from "../raw-evidence-staging.js";
|
|
11
|
+
import { isSecretLikePath, safeKeySegment, sha256, shortHash, } from "./raw-evidence-keys.js";
|
|
12
|
+
import { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
|
|
13
|
+
import { countEvidenceEntries, makeEvidenceCompleteness, markBudgetCapApplied, markCapApplied, recordScanned, recordSkipCount, recordTruncationCount, } from "./raw-evidence-completeness.js";
|
|
14
|
+
import { evidenceEntry, pointerFromEntry, RAW_EVIDENCE_BUCKET, } from "./raw-evidence-manifest.js";
|
|
15
|
+
import { chmodPrivate, ensurePrivateDir, persistStagingState, promoteStagedPack, stageManifest, } from "./raw-evidence-pack-store.js";
|
|
16
|
+
import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, runGitDiff, } from "./raw-evidence-git-diff.js";
|
|
17
|
+
// Re-exported so every consumer keeps importing from `adapters/raw-evidence`.
|
|
18
|
+
export { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./raw-evidence-manifest.js";
|
|
19
|
+
export { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
|
|
12
20
|
const DEFAULT_SINCE_MINUTES = 24 * 60;
|
|
13
21
|
const DEFAULT_SESSION_LIMIT = 50;
|
|
14
|
-
const MAX_GIT_DIFF_BYTES = 2 * 1024 * 1024;
|
|
15
|
-
const GIT_DIFF_TIMEOUT_MS = 3_000;
|
|
16
|
-
export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
|
|
17
|
-
export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
|
|
18
22
|
// Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
|
|
19
23
|
// transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
|
|
20
24
|
// so 2 GiB leaves room for several files without starving the sync; overflow
|
|
@@ -22,6 +26,9 @@ export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
|
|
|
22
26
|
export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
|
|
23
27
|
export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
|
|
24
28
|
const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
// The pass
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
25
32
|
export async function collectRawEvidencePack(context, options) {
|
|
26
33
|
const startedAt = context.now.toISOString();
|
|
27
34
|
const rawEvidenceRoot = path.join(options.stateDir, "raw-evidence");
|
|
@@ -30,338 +37,383 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
30
37
|
// directory is then renamed to its content-keyed name — or dropped, when an
|
|
31
38
|
// identical pack is already there.
|
|
32
39
|
const stagingDir = path.join(rawEvidenceRoot, `.staging-${process.pid}-${crypto.randomUUID().slice(0, 8)}`);
|
|
33
|
-
const filesDir = path.join(stagingDir, "files");
|
|
34
|
-
const staging = await readRawEvidenceStagingState(options.stateDir);
|
|
35
|
-
const entries = [];
|
|
36
|
-
const skipped = [];
|
|
37
|
-
const truncated = [];
|
|
38
|
-
const failed = [];
|
|
39
|
-
const redacted = [];
|
|
40
|
-
const reused = [];
|
|
41
40
|
const sinceMinutes = options.sinceMinutes ?? DEFAULT_SINCE_MINUTES;
|
|
42
|
-
const
|
|
43
|
-
const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
|
|
44
|
-
const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
|
|
45
|
-
const collection = {
|
|
46
|
-
context,
|
|
47
|
-
filesDir,
|
|
41
|
+
const collection = await openCollection(context, options, {
|
|
48
42
|
rawEvidenceRoot,
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
entries,
|
|
56
|
-
skipped,
|
|
57
|
-
truncated,
|
|
58
|
-
failed,
|
|
59
|
-
redacted,
|
|
60
|
-
reused,
|
|
61
|
-
scanned: new Map(),
|
|
62
|
-
caps: [
|
|
63
|
-
{
|
|
64
|
-
source: "raw_evidence",
|
|
65
|
-
cap_type: "byte_budget",
|
|
66
|
-
limit: byteBudget,
|
|
67
|
-
observed: options.budget?.remainingBytes ?? byteBudget,
|
|
68
|
-
applied: false,
|
|
69
|
-
},
|
|
70
|
-
{
|
|
71
|
-
source: "raw_evidence",
|
|
72
|
-
cap_type: "object_budget",
|
|
73
|
-
limit: objectBudget,
|
|
74
|
-
observed: options.budget?.remainingObjects ?? objectBudget,
|
|
75
|
-
applied: false,
|
|
76
|
-
},
|
|
77
|
-
{
|
|
78
|
-
source: "git_diff",
|
|
79
|
-
cap_type: "max_bytes_per_diff",
|
|
80
|
-
limit: MAX_GIT_DIFF_BYTES,
|
|
81
|
-
applied: false,
|
|
82
|
-
},
|
|
83
|
-
{
|
|
84
|
-
source: "git_diff",
|
|
85
|
-
cap_type: "timeout_ms",
|
|
86
|
-
limit: GIT_DIFF_TIMEOUT_MS,
|
|
87
|
-
applied: false,
|
|
88
|
-
},
|
|
89
|
-
{
|
|
90
|
-
source: "codex_jsonl",
|
|
91
|
-
cap_type: "max_file_bytes",
|
|
92
|
-
limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
93
|
-
applied: false,
|
|
94
|
-
},
|
|
95
|
-
{
|
|
96
|
-
source: "claude_jsonl",
|
|
97
|
-
cap_type: "max_file_bytes",
|
|
98
|
-
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
99
|
-
applied: false,
|
|
100
|
-
},
|
|
101
|
-
{
|
|
102
|
-
source: "claude_jsonl_sidecar",
|
|
103
|
-
cap_type: "max_file_bytes",
|
|
104
|
-
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
105
|
-
applied: false,
|
|
106
|
-
},
|
|
107
|
-
],
|
|
108
|
-
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
109
|
-
budget: options.budget ?? {
|
|
110
|
-
remainingBytes: byteBudget,
|
|
111
|
-
remainingObjects: objectBudget,
|
|
112
|
-
},
|
|
113
|
-
index: { value: 0 },
|
|
43
|
+
stagingDir,
|
|
44
|
+
});
|
|
45
|
+
const scanWindow = {
|
|
46
|
+
startedAt,
|
|
47
|
+
finishedAt: () => context.now.toISOString(),
|
|
48
|
+
sinceMinutes,
|
|
114
49
|
};
|
|
115
50
|
try {
|
|
116
51
|
await ensurePrivateDir(stagingDir);
|
|
117
|
-
await ensurePrivateDir(filesDir);
|
|
118
|
-
recordAttributionCompleteness(collection,
|
|
119
|
-
codex: options.codexAttributionScan,
|
|
120
|
-
claude: options.claudeAttributionScan,
|
|
121
|
-
selectedCodexPaths: new Set(options.codexSessionFiles?.map((file) => file.local_path) ?? []),
|
|
122
|
-
selectedClaudePaths: new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []),
|
|
123
|
-
});
|
|
52
|
+
await ensurePrivateDir(collection.filesDir);
|
|
53
|
+
recordAttributionCompleteness(collection, options);
|
|
124
54
|
if (options.includeCodexJsonl !== false) {
|
|
125
55
|
await collectCodexJsonlFiles(collection, {
|
|
126
56
|
codexSessionFiles: options.codexSessionFiles,
|
|
127
57
|
sessionsDir: options.sessionsDir,
|
|
128
58
|
sessionsDirs: options.sessionsDirs,
|
|
129
59
|
sinceMinutes,
|
|
130
|
-
limit: sessionLimit,
|
|
60
|
+
limit: options.sessionLimit ?? DEFAULT_SESSION_LIMIT,
|
|
131
61
|
});
|
|
132
62
|
}
|
|
133
63
|
if (options.includeClaudeJsonl !== false && options.claudeSessionFiles) {
|
|
134
64
|
await collectClaudeJsonlFiles(collection, options.claudeSessionFiles);
|
|
135
65
|
}
|
|
136
66
|
await collectGitDiffFiles(collection, options.repoRoot);
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
// empty pack behind. This used to leave one empty `work-*` dir per sync.
|
|
142
|
-
await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
|
143
|
-
const packId = contentKeyedRawEvidencePackId({
|
|
144
|
-
workContextId: context.workContextId,
|
|
145
|
-
contentHashes: [],
|
|
67
|
+
if (collection.entries.length === 0) {
|
|
68
|
+
return await finishWithEmptyPack(collection, options, {
|
|
69
|
+
stagingDir,
|
|
70
|
+
window: scanWindow,
|
|
146
71
|
});
|
|
147
|
-
collection.packId = packId;
|
|
148
|
-
const evidenceDir = path.join(rawEvidenceRoot, packId);
|
|
149
|
-
await persistStagingState(options.stateDir, collection);
|
|
150
|
-
const evidenceCompleteness = makeEvidenceCompleteness(collection, {
|
|
151
|
-
startedAt,
|
|
152
|
-
finishedAt: context.now.toISOString(),
|
|
153
|
-
sinceMinutes,
|
|
154
|
-
});
|
|
155
|
-
const facts = {
|
|
156
|
-
pack_id: packId,
|
|
157
|
-
manifest_path: path.join(evidenceDir, "manifest.json"),
|
|
158
|
-
evidence_dir: evidenceDir,
|
|
159
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
160
|
-
file_count: 0,
|
|
161
|
-
byte_size: 0,
|
|
162
|
-
skipped_count: countEvidenceEntries(skipped),
|
|
163
|
-
sanitized_count: redacted.length,
|
|
164
|
-
reused_count: reused.length,
|
|
165
|
-
staged_reused_count: collection.stagedReusedCount,
|
|
166
|
-
staged_new_count: collection.stagedNewCount,
|
|
167
|
-
delivery_held_count: collection.deliveryHeldCount,
|
|
168
|
-
stage_state: "empty",
|
|
169
|
-
deferred_byte_budget_count: deferredByteBudgetCount,
|
|
170
|
-
deferred_object_budget_count: deferredObjectBudgetCount,
|
|
171
|
-
content_kinds: [],
|
|
172
|
-
evidence_completeness: evidenceCompleteness,
|
|
173
|
-
pointers: [],
|
|
174
|
-
upload_files: [],
|
|
175
|
-
reused,
|
|
176
|
-
};
|
|
177
|
-
return {
|
|
178
|
-
facts,
|
|
179
|
-
scan: makeRawEvidenceScan({
|
|
180
|
-
context,
|
|
181
|
-
startedAt,
|
|
182
|
-
status: "partial",
|
|
183
|
-
facts,
|
|
184
|
-
}),
|
|
185
|
-
};
|
|
186
72
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
const packId = contentKeyedRawEvidencePackId({
|
|
191
|
-
workContextId: context.workContextId,
|
|
192
|
-
contentHashes: entries.map((entry) => entry.content_hash_sha256),
|
|
73
|
+
return await finishWithPromotedPack(collection, options, {
|
|
74
|
+
stagingDir,
|
|
75
|
+
window: scanWindow,
|
|
193
76
|
});
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
return await finishWithFailedPack(collection, {
|
|
197
80
|
stagingDir,
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
entries,
|
|
81
|
+
window: scanWindow,
|
|
82
|
+
error,
|
|
201
83
|
});
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
async function openCollection(context, options, places) {
|
|
87
|
+
const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
|
|
88
|
+
const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
|
|
89
|
+
const staging = await readRawEvidenceStagingState(options.stateDir);
|
|
90
|
+
return {
|
|
91
|
+
context,
|
|
92
|
+
filesDir: path.join(places.stagingDir, "files"),
|
|
93
|
+
rawEvidenceRoot: places.rawEvidenceRoot,
|
|
94
|
+
packId: "",
|
|
95
|
+
staging,
|
|
96
|
+
heldSources: collectionHeldSources(staging, context.now, options),
|
|
97
|
+
stagedReusedCount: 0,
|
|
98
|
+
stagedNewCount: 0,
|
|
99
|
+
deliveryHeldCount: 0,
|
|
100
|
+
entries: [],
|
|
101
|
+
skipped: [],
|
|
102
|
+
truncated: [],
|
|
103
|
+
failed: [],
|
|
104
|
+
redacted: [],
|
|
105
|
+
reused: [],
|
|
106
|
+
scanned: new Map(),
|
|
107
|
+
caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
|
|
108
|
+
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
109
|
+
budget: options.budget ?? {
|
|
110
|
+
remainingBytes: byteBudget,
|
|
111
|
+
remainingObjects: objectBudget,
|
|
112
|
+
},
|
|
113
|
+
index: { value: 0 },
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Which sources this pass refuses to even read because their delivery is inside
|
|
118
|
+
* a backoff window — none of them, when a person asked for this pass.
|
|
119
|
+
*
|
|
120
|
+
* The bypass is logged rather than assumed: an operator retry that quietly
|
|
121
|
+
* ignored a hold would be as unreadable as the hold that quietly blocked it.
|
|
122
|
+
*/
|
|
123
|
+
function collectionHeldSources(staging, now, options) {
|
|
124
|
+
const held = heldSourceKeys(staging, now);
|
|
125
|
+
if (deliveryBackoffApplies(options.deliveryMode))
|
|
126
|
+
return held;
|
|
127
|
+
if (held.size > 0) {
|
|
128
|
+
console.error("[raw-evidence] delivery backoff bypassed for operator retry", JSON.stringify({
|
|
129
|
+
reason: DELIVERY_BACKOFF_BYPASS_REASON,
|
|
130
|
+
source_count: held.size,
|
|
131
|
+
}));
|
|
132
|
+
}
|
|
133
|
+
return new Set();
|
|
134
|
+
}
|
|
135
|
+
/** Every cap this pass could hit, declared up front and flipped when applied. */
|
|
136
|
+
function startingCaps(options) {
|
|
137
|
+
return [
|
|
138
|
+
{
|
|
139
|
+
source: "raw_evidence",
|
|
140
|
+
cap_type: "byte_budget",
|
|
141
|
+
limit: options.byteBudget,
|
|
142
|
+
observed: options.budget?.remainingBytes ?? options.byteBudget,
|
|
143
|
+
applied: false,
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
source: "raw_evidence",
|
|
147
|
+
cap_type: "object_budget",
|
|
148
|
+
limit: options.objectBudget,
|
|
149
|
+
observed: options.budget?.remainingObjects ?? options.objectBudget,
|
|
150
|
+
applied: false,
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
source: "git_diff",
|
|
154
|
+
cap_type: "max_bytes_per_diff",
|
|
155
|
+
limit: MAX_GIT_DIFF_BYTES,
|
|
156
|
+
applied: false,
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
source: "git_diff",
|
|
160
|
+
cap_type: "timeout_ms",
|
|
161
|
+
limit: GIT_DIFF_TIMEOUT_MS,
|
|
162
|
+
applied: false,
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
source: "codex_jsonl",
|
|
166
|
+
cap_type: "max_file_bytes",
|
|
167
|
+
limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
168
|
+
applied: false,
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
source: "claude_jsonl",
|
|
172
|
+
cap_type: "max_file_bytes",
|
|
173
|
+
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
174
|
+
applied: false,
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
source: "claude_jsonl_sidecar",
|
|
178
|
+
cap_type: "max_file_bytes",
|
|
179
|
+
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
180
|
+
applied: false,
|
|
181
|
+
},
|
|
182
|
+
];
|
|
183
|
+
}
|
|
184
|
+
// ---------------------------------------------------------------------------
|
|
185
|
+
// The three ways a pass ends
|
|
186
|
+
// ---------------------------------------------------------------------------
|
|
187
|
+
/**
|
|
188
|
+
* Nothing collected: drop the staging directory instead of leaving an empty
|
|
189
|
+
* pack behind. This used to leave one empty `work-*` dir per sync.
|
|
190
|
+
*/
|
|
191
|
+
async function finishWithEmptyPack(collection, options, run) {
|
|
192
|
+
await discardStagingDir(run.stagingDir);
|
|
193
|
+
collection.packId = contentKeyedRawEvidencePackId({
|
|
194
|
+
workContextId: collection.context.workContextId,
|
|
195
|
+
contentHashes: [],
|
|
196
|
+
});
|
|
197
|
+
await persistStagingState(options.stateDir, collection.staging, collection.context.now.toISOString());
|
|
198
|
+
const facts = makeEmptyPackFacts(collection, run.window);
|
|
199
|
+
return {
|
|
200
|
+
facts,
|
|
201
|
+
scan: makeRawEvidenceScan({
|
|
202
|
+
context: collection.context,
|
|
203
|
+
startedAt: run.window.startedAt,
|
|
204
|
+
status: "partial",
|
|
205
|
+
facts,
|
|
206
|
+
}),
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
async function finishWithPromotedPack(collection, options, run) {
|
|
210
|
+
const context = collection.context;
|
|
211
|
+
const entries = collection.entries;
|
|
212
|
+
// The pack is named by what is in it, never by when it was made. Identical
|
|
213
|
+
// content on the next sync resolves to the identical directory, which is
|
|
214
|
+
// the whole fix for BLI-3066.
|
|
215
|
+
const packId = contentKeyedRawEvidencePackId({
|
|
216
|
+
workContextId: context.workContextId,
|
|
217
|
+
contentHashes: entries.map((entry) => entry.content_hash_sha256),
|
|
218
|
+
});
|
|
219
|
+
collection.packId = packId;
|
|
220
|
+
const promotion = await promoteStagedPack({
|
|
221
|
+
rawEvidenceRoot: collection.rawEvidenceRoot,
|
|
222
|
+
stagingDir: run.stagingDir,
|
|
223
|
+
packId,
|
|
224
|
+
workContextId: context.workContextId,
|
|
225
|
+
entries,
|
|
226
|
+
});
|
|
227
|
+
const evidenceDir = promotion.evidenceDir;
|
|
228
|
+
rebaseStagedEntriesOntoPack(collection, evidenceDir);
|
|
229
|
+
const manifestPath = path.join(evidenceDir, "manifest.json");
|
|
230
|
+
const manifestBytes = await stageManifest({
|
|
231
|
+
context,
|
|
232
|
+
packId,
|
|
233
|
+
manifestPath,
|
|
234
|
+
entries,
|
|
235
|
+
skipped: collection.skipped,
|
|
236
|
+
redacted: collection.redacted,
|
|
237
|
+
reused: collection.reused,
|
|
238
|
+
reusePack: promotion.state === "reused",
|
|
239
|
+
});
|
|
240
|
+
entries.push(evidenceEntry({
|
|
241
|
+
context,
|
|
242
|
+
kind: "manifest",
|
|
243
|
+
packId,
|
|
244
|
+
localPath: manifestPath,
|
|
245
|
+
relativePath: "manifest.json",
|
|
246
|
+
mediaType: "application/json",
|
|
247
|
+
redactedSummary: "Local raw evidence pack manifest.",
|
|
248
|
+
bytes: manifestBytes,
|
|
249
|
+
}));
|
|
250
|
+
await persistStagingState(options.stateDir, collection.staging, context.now.toISOString());
|
|
251
|
+
console.error("[raw-evidence] pack staged", JSON.stringify({
|
|
252
|
+
pack_id: packId,
|
|
253
|
+
stage_state: promotion.state,
|
|
254
|
+
reason: stageReasonLabel(promotion.state, promotion.priorPackCount),
|
|
255
|
+
prior_pack_count: promotion.priorPackCount,
|
|
256
|
+
refilled_file_count: promotion.refilledFileCount,
|
|
257
|
+
file_count: entries.length,
|
|
258
|
+
byte_size: totalByteSize(entries),
|
|
259
|
+
staged_new: collection.stagedNewCount,
|
|
260
|
+
staged_reused: collection.stagedReusedCount,
|
|
261
|
+
delivery_held: collection.deliveryHeldCount,
|
|
262
|
+
}));
|
|
263
|
+
const facts = {
|
|
264
|
+
...packFactsShell(collection, {
|
|
218
265
|
packId,
|
|
219
266
|
manifestPath,
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
267
|
+
evidenceDir,
|
|
268
|
+
window: run.window,
|
|
269
|
+
}),
|
|
270
|
+
stage_state: promotion.state,
|
|
271
|
+
file_count: entries.length,
|
|
272
|
+
byte_size: totalByteSize(entries),
|
|
273
|
+
content_kinds: [...new Set(entries.map((entry) => entry.kind))],
|
|
274
|
+
pointers: entries.map(pointerFromEntry),
|
|
275
|
+
upload_files: entries.map(uploadFileFromEntry),
|
|
276
|
+
};
|
|
277
|
+
return {
|
|
278
|
+
facts,
|
|
279
|
+
scan: makeRawEvidenceScan({
|
|
227
280
|
context,
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
localPath: manifestPath,
|
|
231
|
-
relativePath: "manifest.json",
|
|
232
|
-
mediaType: "application/json",
|
|
233
|
-
redactedSummary: "Local raw evidence pack manifest.",
|
|
234
|
-
bytes: manifestBytes,
|
|
235
|
-
});
|
|
236
|
-
entries.push(manifestEntry);
|
|
237
|
-
await persistStagingState(options.stateDir, collection);
|
|
238
|
-
console.error("[raw-evidence] pack staged", JSON.stringify({
|
|
239
|
-
pack_id: packId,
|
|
240
|
-
stage_state: promotion.state,
|
|
241
|
-
reason: promotion.state === "reused"
|
|
242
|
-
? "staged_reused"
|
|
243
|
-
: promotion.state === "restaged_incomplete"
|
|
244
|
-
? "restaged_incomplete"
|
|
245
|
-
: promotion.priorPackCount > 0
|
|
246
|
-
? "restaged_content_changed"
|
|
247
|
-
: "staged_new",
|
|
248
|
-
prior_pack_count: promotion.priorPackCount,
|
|
249
|
-
refilled_file_count: promotion.refilledFileCount,
|
|
250
|
-
file_count: entries.length,
|
|
251
|
-
byte_size: entries.reduce((sum, entry) => sum + entry.byte_size, 0),
|
|
252
|
-
staged_new: collection.stagedNewCount,
|
|
253
|
-
staged_reused: collection.stagedReusedCount,
|
|
254
|
-
delivery_held: collection.deliveryHeldCount,
|
|
255
|
-
}));
|
|
256
|
-
const evidenceCompleteness = makeEvidenceCompleteness(collection, {
|
|
257
|
-
startedAt,
|
|
258
|
-
finishedAt: context.now.toISOString(),
|
|
259
|
-
sinceMinutes,
|
|
260
|
-
});
|
|
261
|
-
const facts = {
|
|
262
|
-
pack_id: packId,
|
|
263
|
-
manifest_path: manifestPath,
|
|
264
|
-
evidence_dir: evidenceDir,
|
|
265
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
266
|
-
staged_reused_count: collection.stagedReusedCount,
|
|
267
|
-
staged_new_count: collection.stagedNewCount,
|
|
268
|
-
delivery_held_count: collection.deliveryHeldCount,
|
|
269
|
-
stage_state: promotion.state,
|
|
270
|
-
file_count: entries.length,
|
|
271
|
-
byte_size: entries.reduce((sum, entry) => sum + entry.byte_size, 0),
|
|
272
|
-
skipped_count: countEvidenceEntries(skipped),
|
|
273
|
-
sanitized_count: redacted.length,
|
|
274
|
-
reused_count: reused.length,
|
|
275
|
-
deferred_byte_budget_count: deferredByteBudgetCount,
|
|
276
|
-
deferred_object_budget_count: deferredObjectBudgetCount,
|
|
277
|
-
content_kinds: [...new Set(entries.map((entry) => entry.kind))],
|
|
278
|
-
evidence_completeness: evidenceCompleteness,
|
|
279
|
-
pointers: entries.map(pointerFromEntry),
|
|
280
|
-
upload_files: entries.map((entry) => ({
|
|
281
|
-
pointer: pointerFromEntry(entry),
|
|
282
|
-
local_path: entry.local_path,
|
|
283
|
-
kind: entry.kind,
|
|
284
|
-
codex_session_id: entry.codex_session_id ?? null,
|
|
285
|
-
...(entry.artifact_metadata
|
|
286
|
-
? { artifact_metadata: entry.artifact_metadata }
|
|
287
|
-
: {}),
|
|
288
|
-
})),
|
|
289
|
-
reused,
|
|
290
|
-
};
|
|
291
|
-
return {
|
|
281
|
+
startedAt: run.window.startedAt,
|
|
282
|
+
status: entries.length > 1 ? "ok" : "partial",
|
|
292
283
|
facts,
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
284
|
+
}),
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* A crashed pass. The staged-object index is deliberately NOT written: this
|
|
289
|
+
* attempt's bookkeeping describes a directory that is about to be deleted.
|
|
290
|
+
*/
|
|
291
|
+
async function finishWithFailedPack(collection, run) {
|
|
292
|
+
collection.failed.push({
|
|
293
|
+
kind: "raw_evidence",
|
|
294
|
+
reason: "collection_failed",
|
|
295
|
+
});
|
|
296
|
+
// Staging is per-attempt scratch: a crashed pass must not leave a partial
|
|
297
|
+
// directory behind to be counted, re-hashed or swept later.
|
|
298
|
+
await discardStagingDir(run.stagingDir);
|
|
299
|
+
collection.packId =
|
|
300
|
+
collection.packId ||
|
|
310
301
|
contentKeyedRawEvidencePackId({
|
|
311
|
-
workContextId: context.workContextId,
|
|
312
|
-
contentHashes: entries.map((entry) => entry.content_hash_sha256),
|
|
302
|
+
workContextId: collection.context.workContextId,
|
|
303
|
+
contentHashes: collection.entries.map((entry) => entry.content_hash_sha256),
|
|
313
304
|
});
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
pack_id: packId,
|
|
330
|
-
manifest_path: path.join(evidenceDir, "manifest.json"),
|
|
331
|
-
evidence_dir: evidenceDir,
|
|
332
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
333
|
-
file_count: 0,
|
|
334
|
-
byte_size: 0,
|
|
335
|
-
skipped_count: countEvidenceEntries(skipped),
|
|
336
|
-
sanitized_count: redacted.length,
|
|
337
|
-
reused_count: reused.length,
|
|
338
|
-
staged_reused_count: collection.stagedReusedCount,
|
|
339
|
-
staged_new_count: collection.stagedNewCount,
|
|
340
|
-
delivery_held_count: collection.deliveryHeldCount,
|
|
341
|
-
stage_state: "empty",
|
|
342
|
-
deferred_byte_budget_count: skipped
|
|
343
|
-
.filter((entry) => entry.reason === "deferred_byte_budget")
|
|
344
|
-
.reduce((sum, entry) => sum + evidenceEntryCount(entry), 0),
|
|
345
|
-
deferred_object_budget_count: skipped
|
|
346
|
-
.filter((entry) => entry.reason === "deferred_object_budget")
|
|
347
|
-
.reduce((sum, entry) => sum + evidenceEntryCount(entry), 0),
|
|
348
|
-
content_kinds: [],
|
|
349
|
-
evidence_completeness: evidenceCompleteness,
|
|
350
|
-
pointers: [],
|
|
351
|
-
upload_files: [],
|
|
352
|
-
reused,
|
|
353
|
-
};
|
|
354
|
-
return {
|
|
305
|
+
console.error("[raw-evidence] pack collection failed", JSON.stringify({
|
|
306
|
+
pack_id: collection.packId,
|
|
307
|
+
reason: "collection_failed",
|
|
308
|
+
detail: run.error instanceof Error ? run.error.name : typeof run.error,
|
|
309
|
+
collected_file_count: collection.entries.length,
|
|
310
|
+
staged_new: collection.stagedNewCount,
|
|
311
|
+
staged_reused: collection.stagedReusedCount,
|
|
312
|
+
}));
|
|
313
|
+
const facts = makeEmptyPackFacts(collection, run.window);
|
|
314
|
+
return {
|
|
315
|
+
facts,
|
|
316
|
+
scan: makeRawEvidenceScan({
|
|
317
|
+
context: collection.context,
|
|
318
|
+
startedAt: run.window.startedAt,
|
|
319
|
+
status: "failed",
|
|
355
320
|
facts,
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
321
|
+
}),
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* A pack with no files of its own: the empty pass and the crashed pass report
|
|
326
|
+
* the same shape, differing only in the scan status and the reason already
|
|
327
|
+
* logged. `evidence_completeness` still carries every gap this pass recorded.
|
|
328
|
+
*/
|
|
329
|
+
function makeEmptyPackFacts(collection, scanWindow) {
|
|
330
|
+
const evidenceDir = path.join(collection.rawEvidenceRoot, collection.packId);
|
|
331
|
+
return {
|
|
332
|
+
...packFactsShell(collection, {
|
|
333
|
+
packId: collection.packId,
|
|
334
|
+
manifestPath: path.join(evidenceDir, "manifest.json"),
|
|
335
|
+
evidenceDir,
|
|
336
|
+
window: scanWindow,
|
|
337
|
+
}),
|
|
338
|
+
stage_state: "empty",
|
|
339
|
+
file_count: 0,
|
|
340
|
+
byte_size: 0,
|
|
341
|
+
content_kinds: [],
|
|
342
|
+
pointers: [],
|
|
343
|
+
upload_files: [],
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
/** The counts every ending reports identically, however the pass ended. */
|
|
347
|
+
function packFactsShell(collection, place) {
|
|
348
|
+
return {
|
|
349
|
+
pack_id: place.packId,
|
|
350
|
+
manifest_path: place.manifestPath,
|
|
351
|
+
evidence_dir: place.evidenceDir,
|
|
352
|
+
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
353
|
+
skipped_count: countEvidenceEntries(collection.skipped),
|
|
354
|
+
sanitized_count: collection.redacted.length,
|
|
355
|
+
reused_count: collection.reused.length,
|
|
356
|
+
staged_reused_count: collection.stagedReusedCount,
|
|
357
|
+
staged_new_count: collection.stagedNewCount,
|
|
358
|
+
delivery_held_count: collection.deliveryHeldCount,
|
|
359
|
+
deferred_byte_budget_count: countDeferred(collection, "deferred_byte_budget"),
|
|
360
|
+
deferred_object_budget_count: countDeferred(collection, "deferred_object_budget"),
|
|
361
|
+
evidence_completeness: makeEvidenceCompleteness(collection, {
|
|
362
|
+
startedAt: place.window.startedAt,
|
|
363
|
+
finishedAt: place.window.finishedAt(),
|
|
364
|
+
sinceMinutes: place.window.sinceMinutes,
|
|
365
|
+
}),
|
|
366
|
+
reused: collection.reused,
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
function countDeferred(collection, reason) {
|
|
370
|
+
return countEvidenceEntries(collection.skipped, (entry) => entry.reason === reason);
|
|
371
|
+
}
|
|
372
|
+
function totalByteSize(entries) {
|
|
373
|
+
return entries.reduce((sum, entry) => sum + entry.byte_size, 0);
|
|
374
|
+
}
|
|
375
|
+
function uploadFileFromEntry(entry) {
|
|
376
|
+
return {
|
|
377
|
+
pointer: pointerFromEntry(entry),
|
|
378
|
+
local_path: entry.local_path,
|
|
379
|
+
kind: entry.kind,
|
|
380
|
+
codex_session_id: entry.codex_session_id ?? null,
|
|
381
|
+
...(entry.artifact_metadata
|
|
382
|
+
? { artifact_metadata: entry.artifact_metadata }
|
|
383
|
+
: {}),
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
function stageReasonLabel(state, priorPackCount) {
|
|
387
|
+
if (state === "reused")
|
|
388
|
+
return "staged_reused";
|
|
389
|
+
if (state === "restaged_incomplete")
|
|
390
|
+
return "restaged_incomplete";
|
|
391
|
+
return priorPackCount > 0 ? "restaged_content_changed" : "staged_new";
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Point every entry this pass staged at its home in the promoted pack, and
|
|
395
|
+
* remember the content hash so the next sync can adopt the copy instead of
|
|
396
|
+
* writing it again. Entries adopted from another pack keep their path.
|
|
397
|
+
*/
|
|
398
|
+
function rebaseStagedEntriesOntoPack(collection, evidenceDir) {
|
|
399
|
+
for (const entry of collection.entries) {
|
|
400
|
+
if (!entry.staged_in_pack)
|
|
401
|
+
continue;
|
|
402
|
+
entry.local_path = path.join(evidenceDir, "files", path.basename(entry.local_path));
|
|
403
|
+
recordStagedObject(collection.staging, entry.content_hash_sha256, {
|
|
404
|
+
pack_id: collection.packId,
|
|
405
|
+
relative_path: `files/${path.basename(entry.local_path)}`,
|
|
406
|
+
byte_size: entry.byte_size,
|
|
407
|
+
source_key: entry.source_key,
|
|
408
|
+
staged_at: collection.context.now.toISOString(),
|
|
409
|
+
});
|
|
363
410
|
}
|
|
364
411
|
}
|
|
412
|
+
async function discardStagingDir(stagingDir) {
|
|
413
|
+
await fs
|
|
414
|
+
.rm(stagingDir, { recursive: true, force: true })
|
|
415
|
+
.catch(() => undefined);
|
|
416
|
+
}
|
|
365
417
|
function makeRawEvidenceScan(options) {
|
|
366
418
|
return SourceScanResultSchema.parse({
|
|
367
419
|
adapter: makeSourceAdapterIdentity("collector_runtime", "raw-evidence-pack"),
|
|
@@ -402,12 +454,20 @@ function makeRawEvidenceScan(options) {
|
|
|
402
454
|
],
|
|
403
455
|
});
|
|
404
456
|
}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
457
|
+
// ---------------------------------------------------------------------------
|
|
458
|
+
// What attribution already knew
|
|
459
|
+
// ---------------------------------------------------------------------------
|
|
460
|
+
/**
|
|
461
|
+
* Attribution ran before collection and already knows which sessions it could
|
|
462
|
+
* not name. Those are gaps in this pass even though no byte was read, so they
|
|
463
|
+
* are recorded first — a session Cockpit never looked at must not be invisible.
|
|
464
|
+
*/
|
|
465
|
+
function recordAttributionCompleteness(collection, options) {
|
|
466
|
+
if (options.codexAttributionScan) {
|
|
467
|
+
recordCodexAttributionCompleteness(collection, options.codexAttributionScan, new Set(options.codexSessionFiles?.map((file) => file.local_path) ?? []));
|
|
408
468
|
}
|
|
409
|
-
if (
|
|
410
|
-
recordClaudeAttributionCompleteness(collection,
|
|
469
|
+
if (options.claudeAttributionScan) {
|
|
470
|
+
recordClaudeAttributionCompleteness(collection, options.claudeAttributionScan, new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []));
|
|
411
471
|
}
|
|
412
472
|
}
|
|
413
473
|
function recordCodexAttributionCompleteness(collection, scan, selectedPaths) {
|
|
@@ -485,39 +545,46 @@ function recordClaudeAttributionCompleteness(collection, scan, selectedPaths) {
|
|
|
485
545
|
}
|
|
486
546
|
function recordAttributionResultSkips(collection, source, results, selectedPaths) {
|
|
487
547
|
for (const result of results) {
|
|
488
|
-
|
|
489
|
-
// synthetic target is also not a skip when another workspace pack owns it.
|
|
490
|
-
if (result.state === "attributed" ||
|
|
491
|
-
selectedPaths.has(result.file_path) ||
|
|
492
|
-
(result.worktree !== null &&
|
|
493
|
-
isLiveRawEvidenceSyncAttribution(result.state, result.reason, true))) {
|
|
548
|
+
if (isAttributionAccountedFor(result, selectedPaths))
|
|
494
549
|
continue;
|
|
495
|
-
}
|
|
496
550
|
const reason = result.state === "skipped"
|
|
497
551
|
? result.reason
|
|
498
552
|
: `attribution_${result.state}:${result.reason}`;
|
|
499
553
|
recordSkipCount(collection, source, reason, 1);
|
|
500
554
|
}
|
|
501
555
|
}
|
|
556
|
+
/**
|
|
557
|
+
* Not every unattributed result is a gap. A selected session is being collected
|
|
558
|
+
* by this pass, and a live-sync-safe synthetic target is owned by another
|
|
559
|
+
* workspace's pack — neither is missing evidence.
|
|
560
|
+
*/
|
|
561
|
+
function isAttributionAccountedFor(result, selectedPaths) {
|
|
562
|
+
if (result.state === "attributed")
|
|
563
|
+
return true;
|
|
564
|
+
if (selectedPaths.has(result.file_path))
|
|
565
|
+
return true;
|
|
566
|
+
return (result.worktree !== null &&
|
|
567
|
+
isLiveRawEvidenceSyncAttribution(result.state, result.reason, true));
|
|
568
|
+
}
|
|
569
|
+
// ---------------------------------------------------------------------------
|
|
570
|
+
// Codex transcripts
|
|
571
|
+
// ---------------------------------------------------------------------------
|
|
502
572
|
async function collectCodexJsonlFiles(collection, options) {
|
|
573
|
+
const attributed = Boolean(options.codexSessionFiles);
|
|
503
574
|
const resolvedCandidates = options.codexSessionFiles
|
|
504
575
|
? options.codexSessionFiles.map((file) => ({
|
|
505
576
|
filePath: file.local_path,
|
|
506
577
|
codexSessionId: file.codex_session_id,
|
|
507
578
|
}))
|
|
508
|
-
: (await
|
|
509
|
-
(options.sessionsDir
|
|
510
|
-
? [options.sessionsDir]
|
|
511
|
-
: defaultCodexSessionDirs(os.homedir())), collection.context.now.getTime() - options.sinceMinutes * 60 * 1000))
|
|
512
|
-
.map((filePath) => ({ filePath, codexSessionId: null }));
|
|
579
|
+
: (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
|
|
513
580
|
collection.caps.push({
|
|
514
581
|
source: "codex_jsonl",
|
|
515
582
|
cap_type: "session_limit",
|
|
516
583
|
limit: options.limit,
|
|
517
584
|
observed: resolvedCandidates.length,
|
|
518
|
-
applied: !
|
|
585
|
+
applied: !attributed && resolvedCandidates.length > options.limit,
|
|
519
586
|
});
|
|
520
|
-
const candidates =
|
|
587
|
+
const candidates = attributed
|
|
521
588
|
? resolvedCandidates
|
|
522
589
|
: resolvedCandidates.slice(0, options.limit);
|
|
523
590
|
for (const candidate of candidates) {
|
|
@@ -534,95 +601,118 @@ async function collectCodexJsonlFiles(collection, options) {
|
|
|
534
601
|
redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
535
602
|
contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
|
|
536
603
|
});
|
|
537
|
-
if (transcriptAccepted)
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
}
|
|
545
|
-
}
|
|
604
|
+
if (!transcriptAccepted)
|
|
605
|
+
continue;
|
|
606
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
607
|
+
filePath: candidate.filePath,
|
|
608
|
+
source: "codex",
|
|
609
|
+
sessionId: codexSessionId,
|
|
610
|
+
kind: "codex_image_attachment",
|
|
611
|
+
contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
|
|
612
|
+
});
|
|
546
613
|
}
|
|
547
614
|
}
|
|
615
|
+
/** The unattributed fallback: every recent Codex transcript on this machine. */
|
|
616
|
+
function walkRecentCodexJsonlFiles(collection, options) {
|
|
617
|
+
const dirs = options.sessionsDirs ??
|
|
618
|
+
(options.sessionsDir
|
|
619
|
+
? [options.sessionsDir]
|
|
620
|
+
: defaultCodexSessionDirs(os.homedir()));
|
|
621
|
+
return walkJsonlFiles(dirs, collection.context.now.getTime() - options.sinceMinutes * 60 * 1000);
|
|
622
|
+
}
|
|
623
|
+
// ---------------------------------------------------------------------------
|
|
624
|
+
// Claude transcripts
|
|
625
|
+
// ---------------------------------------------------------------------------
|
|
548
626
|
async function collectClaudeJsonlFiles(collection, sessions) {
|
|
549
627
|
recordScanned(collection, "claude_jsonl", sessions.reduce((count, session) => count + 1 + session.sidecar_files.length, 0));
|
|
550
628
|
for (const session of sessions) {
|
|
551
|
-
|
|
552
|
-
if (session.main_file_oversized) {
|
|
553
|
-
// D7: the oversized main was attributed via a streamed read but its bytes
|
|
554
|
-
// are never uploaded (server commit assembles in memory). Its sidecars
|
|
555
|
-
// still collect below.
|
|
556
|
-
collection.skipped.push({
|
|
557
|
-
kind: "claude_jsonl",
|
|
558
|
-
label: path.basename(session.local_path),
|
|
559
|
-
reason: "file_too_large",
|
|
560
|
-
});
|
|
561
|
-
}
|
|
562
|
-
else if (session.skip_main) {
|
|
563
|
-
// D9 damped: prior durable copy is still good enough; collect sidecars
|
|
564
|
-
// only. The session reports reused_existing from cursor state, so no
|
|
565
|
-
// skipped entry is recorded here.
|
|
566
|
-
}
|
|
567
|
-
else {
|
|
568
|
-
const mainAccepted = await collectOneEvidenceFile(collection, {
|
|
569
|
-
filePath: session.local_path,
|
|
570
|
-
kind: "claude_jsonl",
|
|
571
|
-
sessionId,
|
|
572
|
-
mediaType: "application/jsonl",
|
|
573
|
-
// Re-check size at collection: a main that grew past the cap between
|
|
574
|
-
// attribution and collection is an honest file_too_large skip, not an
|
|
575
|
-
// upload_failed at the chunk client.
|
|
576
|
-
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
577
|
-
redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
578
|
-
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/${hash16}.jsonl`,
|
|
579
|
-
});
|
|
580
|
-
if (mainAccepted) {
|
|
581
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
582
|
-
filePath: session.local_path,
|
|
583
|
-
source: "claude_code",
|
|
584
|
-
sessionId,
|
|
585
|
-
kind: "claude_image_attachment",
|
|
586
|
-
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
|
|
587
|
-
});
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
if (session.skip_main && !session.main_file_oversized) {
|
|
591
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
592
|
-
filePath: session.local_path,
|
|
593
|
-
source: "claude_code",
|
|
594
|
-
sessionId,
|
|
595
|
-
kind: "claude_image_attachment",
|
|
596
|
-
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
|
|
597
|
-
});
|
|
598
|
-
}
|
|
599
|
-
for (const sidecar of session.sidecar_files) {
|
|
600
|
-
const stem = path.basename(sidecar.local_path).replace(/\.jsonl$/i, "");
|
|
601
|
-
const safeSidecarId = isSecretLikePath(stem)
|
|
602
|
-
? "redacted-file-name"
|
|
603
|
-
: safeKeySegment(stem);
|
|
604
|
-
const sidecarAccepted = await collectOneEvidenceFile(collection, {
|
|
605
|
-
filePath: sidecar.local_path,
|
|
606
|
-
kind: "claude_jsonl_sidecar",
|
|
607
|
-
sessionId,
|
|
608
|
-
mediaType: "application/jsonl",
|
|
609
|
-
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
610
|
-
redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
611
|
-
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
|
|
612
|
-
});
|
|
613
|
-
if (sidecarAccepted) {
|
|
614
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
615
|
-
filePath: sidecar.local_path,
|
|
616
|
-
source: "claude_code",
|
|
617
|
-
sessionId,
|
|
618
|
-
sidecarId: safeSidecarId,
|
|
619
|
-
kind: "claude_image_attachment",
|
|
620
|
-
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
|
|
621
|
-
});
|
|
622
|
-
}
|
|
623
|
-
}
|
|
629
|
+
await collectOneClaudeSession(collection, session);
|
|
624
630
|
}
|
|
625
631
|
}
|
|
632
|
+
async function collectOneClaudeSession(collection, session) {
|
|
633
|
+
const sessionId = session.claude_session_id;
|
|
634
|
+
const mainOutcome = await collectClaudeMainFile(collection, session);
|
|
635
|
+
if (mainOutcome === "collected" || mainOutcome === "damped_reuse") {
|
|
636
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
637
|
+
filePath: session.local_path,
|
|
638
|
+
source: "claude_code",
|
|
639
|
+
sessionId,
|
|
640
|
+
kind: "claude_image_attachment",
|
|
641
|
+
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
for (const sidecar of session.sidecar_files) {
|
|
645
|
+
await collectOneClaudeSidecar(collection, session, sidecar.local_path);
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* What happens to a Claude main file, as a decision table:
|
|
650
|
+
*
|
|
651
|
+
* | condition | main bytes | outcome |
|
|
652
|
+
* | --------------------- | ------------------------- | ------------------- |
|
|
653
|
+
* | `main_file_oversized` | skipped `file_too_large` | `skipped_too_large` |
|
|
654
|
+
* | `skip_main` (D9) | not re-collected | `damped_reuse` |
|
|
655
|
+
* | otherwise | collected, or named skip | `collected` / `not_collected` |
|
|
656
|
+
*
|
|
657
|
+
* Its images are collected for every outcome except `skipped_too_large` and
|
|
658
|
+
* `not_collected` — the caller decides that, this function only reports.
|
|
659
|
+
*
|
|
660
|
+
* D7: an oversized main was attributed via a streamed read but its bytes are
|
|
661
|
+
* never uploaded (server commit assembles in memory). Its sidecars still
|
|
662
|
+
* collect. D9 damped: the prior durable copy is still good enough, and the
|
|
663
|
+
* session reports `reused_existing` from cursor state, so no skip is recorded.
|
|
664
|
+
*/
|
|
665
|
+
async function collectClaudeMainFile(collection, session) {
|
|
666
|
+
if (session.main_file_oversized) {
|
|
667
|
+
collection.skipped.push({
|
|
668
|
+
kind: "claude_jsonl",
|
|
669
|
+
label: path.basename(session.local_path),
|
|
670
|
+
reason: "file_too_large",
|
|
671
|
+
});
|
|
672
|
+
return "skipped_too_large";
|
|
673
|
+
}
|
|
674
|
+
if (session.skip_main)
|
|
675
|
+
return "damped_reuse";
|
|
676
|
+
const accepted = await collectOneEvidenceFile(collection, {
|
|
677
|
+
filePath: session.local_path,
|
|
678
|
+
kind: "claude_jsonl",
|
|
679
|
+
sessionId: session.claude_session_id,
|
|
680
|
+
mediaType: "application/jsonl",
|
|
681
|
+
// Re-check size at collection: a main that grew past the cap between
|
|
682
|
+
// attribution and collection is an honest file_too_large skip, not an
|
|
683
|
+
// upload_failed at the chunk client.
|
|
684
|
+
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
685
|
+
redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
686
|
+
contentAddress: (hash16) => `claude/${safeKeySegment(session.claude_session_id)}/${hash16}.jsonl`,
|
|
687
|
+
});
|
|
688
|
+
return accepted ? "collected" : "not_collected";
|
|
689
|
+
}
|
|
690
|
+
async function collectOneClaudeSidecar(collection, session, sidecarPath) {
|
|
691
|
+
const sessionId = session.claude_session_id;
|
|
692
|
+
const stem = path.basename(sidecarPath).replace(/\.jsonl$/i, "");
|
|
693
|
+
const safeSidecarId = isSecretLikePath(stem)
|
|
694
|
+
? "redacted-file-name"
|
|
695
|
+
: safeKeySegment(stem);
|
|
696
|
+
const sidecarAccepted = await collectOneEvidenceFile(collection, {
|
|
697
|
+
filePath: sidecarPath,
|
|
698
|
+
kind: "claude_jsonl_sidecar",
|
|
699
|
+
sessionId,
|
|
700
|
+
mediaType: "application/jsonl",
|
|
701
|
+
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
702
|
+
redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
703
|
+
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
|
|
704
|
+
});
|
|
705
|
+
if (!sidecarAccepted)
|
|
706
|
+
return;
|
|
707
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
708
|
+
filePath: sidecarPath,
|
|
709
|
+
source: "claude_code",
|
|
710
|
+
sessionId,
|
|
711
|
+
sidecarId: safeSidecarId,
|
|
712
|
+
kind: "claude_image_attachment",
|
|
713
|
+
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
|
|
714
|
+
});
|
|
715
|
+
}
|
|
626
716
|
async function collectAgentImagesFromTranscript(collection, options) {
|
|
627
717
|
const result = await collectAgentImageEvidenceFromJsonlFile({
|
|
628
718
|
filePath: options.filePath,
|
|
@@ -705,35 +795,43 @@ async function collectOneAgentImageFile(collection, options) {
|
|
|
705
795
|
artifactMetadata: metadata,
|
|
706
796
|
}));
|
|
707
797
|
}
|
|
798
|
+
// ---------------------------------------------------------------------------
|
|
799
|
+
// One transcript, end to end
|
|
800
|
+
// ---------------------------------------------------------------------------
|
|
708
801
|
/**
|
|
709
802
|
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
710
|
-
* attributed transcript into the pack.
|
|
711
|
-
*
|
|
712
|
-
*
|
|
803
|
+
* attributed transcript into the pack.
|
|
804
|
+
*
|
|
805
|
+
* Returns whether this file's *content* is accounted for, which is not the same
|
|
806
|
+
* as "was staged": a transcript whose bytes are already durable remotely
|
|
807
|
+
* returns true without staging anything, and its images are still worth
|
|
808
|
+
* collecting. Every false return has pushed a named skip first.
|
|
713
809
|
*/
|
|
714
810
|
async function collectOneEvidenceFile(collection, options) {
|
|
715
811
|
const fileName = path.basename(options.filePath);
|
|
716
812
|
const secretLikeFileName = isSecretLikePath(fileName);
|
|
717
|
-
const evidenceLabel = secretLikeFileName
|
|
718
|
-
? "[REDACTED_FILE_NAME]"
|
|
719
|
-
: fileName;
|
|
813
|
+
const evidenceLabel = secretLikeFileName ? "[REDACTED_FILE_NAME]" : fileName;
|
|
720
814
|
const packedFileName = secretLikeFileName ? "redacted-file.jsonl" : fileName;
|
|
721
815
|
const sourceKey = evidenceSourceKey({
|
|
722
816
|
kind: options.kind,
|
|
723
817
|
sessionId: options.sessionId,
|
|
724
818
|
sourcePath: options.filePath,
|
|
725
819
|
});
|
|
820
|
+
const skip = (reason) => {
|
|
821
|
+
collection.skipped.push({
|
|
822
|
+
kind: options.kind,
|
|
823
|
+
label: evidenceLabel,
|
|
824
|
+
reason,
|
|
825
|
+
});
|
|
826
|
+
return false;
|
|
827
|
+
};
|
|
726
828
|
// Delivery backoff is checked BEFORE the file is read. An object whose commit
|
|
727
829
|
// has failed repeatedly costs nothing at all this cycle — no read, no hash,
|
|
728
830
|
// no copy, no request — and the hold is a named, retryable gap so a held
|
|
729
831
|
// session cannot make the sync look clean (BLI-3066).
|
|
730
832
|
if (collection.heldSources.has(sourceKey)) {
|
|
731
833
|
collection.deliveryHeldCount += 1;
|
|
732
|
-
|
|
733
|
-
kind: options.kind,
|
|
734
|
-
label: evidenceLabel,
|
|
735
|
-
reason: DELIVERY_BACKOFF_HOLDING_REASON,
|
|
736
|
-
});
|
|
834
|
+
skip(DELIVERY_BACKOFF_HOLDING_REASON);
|
|
737
835
|
console.error("[raw-evidence] delivery backoff holding source", JSON.stringify({
|
|
738
836
|
reason: DELIVERY_BACKOFF_HOLDING_REASON,
|
|
739
837
|
kind: options.kind,
|
|
@@ -741,50 +839,14 @@ async function collectOneEvidenceFile(collection, options) {
|
|
|
741
839
|
}));
|
|
742
840
|
return false;
|
|
743
841
|
}
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
}
|
|
749
|
-
catch {
|
|
750
|
-
collection.skipped.push({
|
|
751
|
-
kind: options.kind,
|
|
752
|
-
label: evidenceLabel,
|
|
753
|
-
reason: "file_read_failed",
|
|
754
|
-
});
|
|
755
|
-
return false;
|
|
756
|
-
}
|
|
757
|
-
if (stat.size > options.maxFileBytes) {
|
|
758
|
-
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
759
|
-
collection.skipped.push({
|
|
760
|
-
kind: options.kind,
|
|
761
|
-
label: evidenceLabel,
|
|
762
|
-
reason: "file_too_large",
|
|
763
|
-
});
|
|
764
|
-
return false;
|
|
765
|
-
}
|
|
766
|
-
}
|
|
767
|
-
let raw;
|
|
768
|
-
try {
|
|
769
|
-
raw = await fs.readFile(options.filePath);
|
|
770
|
-
}
|
|
771
|
-
catch {
|
|
772
|
-
collection.skipped.push({
|
|
773
|
-
kind: options.kind,
|
|
774
|
-
label: evidenceLabel,
|
|
775
|
-
reason: "file_read_failed",
|
|
776
|
-
});
|
|
777
|
-
return false;
|
|
778
|
-
}
|
|
779
|
-
if (options.maxFileBytes && raw.byteLength > options.maxFileBytes) {
|
|
842
|
+
const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
|
|
843
|
+
if (read.status === "read_failed")
|
|
844
|
+
return skip("file_read_failed");
|
|
845
|
+
if (read.status === "too_large") {
|
|
780
846
|
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
781
|
-
|
|
782
|
-
kind: options.kind,
|
|
783
|
-
label: evidenceLabel,
|
|
784
|
-
reason: "file_too_large",
|
|
785
|
-
});
|
|
786
|
-
return false;
|
|
847
|
+
return skip("file_too_large");
|
|
787
848
|
}
|
|
849
|
+
const raw = read.bytes;
|
|
788
850
|
const sanitized = sanitizeTextEvidenceForUpload({
|
|
789
851
|
text: raw.toString("utf8"),
|
|
790
852
|
originalBytes: raw,
|
|
@@ -820,12 +882,7 @@ async function collectOneEvidenceFile(collection, options) {
|
|
|
820
882
|
const deferReason = admitToBudget(collection.budget, evidenceBytes.byteLength);
|
|
821
883
|
if (deferReason) {
|
|
822
884
|
markBudgetCapApplied(collection, deferReason);
|
|
823
|
-
|
|
824
|
-
kind: options.kind,
|
|
825
|
-
label: evidenceLabel,
|
|
826
|
-
reason: deferReason,
|
|
827
|
-
});
|
|
828
|
-
return false;
|
|
885
|
+
return skip(deferReason);
|
|
829
886
|
}
|
|
830
887
|
collection.index.value += 1;
|
|
831
888
|
const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(options.filePath)}-${packedFileName}`);
|
|
@@ -855,6 +912,38 @@ async function collectOneEvidenceFile(collection, options) {
|
|
|
855
912
|
}));
|
|
856
913
|
return true;
|
|
857
914
|
}
|
|
915
|
+
/**
|
|
916
|
+
* Size is checked twice on purpose: once by `stat` so an oversized transcript
|
|
917
|
+
* is never buffered at all, and once on the bytes actually read, because a live
|
|
918
|
+
* session can grow between the two.
|
|
919
|
+
*/
|
|
920
|
+
async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
|
|
921
|
+
if (maxFileBytes) {
|
|
922
|
+
let stat;
|
|
923
|
+
try {
|
|
924
|
+
stat = await fs.stat(filePath);
|
|
925
|
+
}
|
|
926
|
+
catch {
|
|
927
|
+
return { status: "read_failed" };
|
|
928
|
+
}
|
|
929
|
+
if (stat.size > maxFileBytes)
|
|
930
|
+
return { status: "too_large" };
|
|
931
|
+
}
|
|
932
|
+
let bytes;
|
|
933
|
+
try {
|
|
934
|
+
bytes = await fs.readFile(filePath);
|
|
935
|
+
}
|
|
936
|
+
catch {
|
|
937
|
+
return { status: "read_failed" };
|
|
938
|
+
}
|
|
939
|
+
if (maxFileBytes && bytes.byteLength > maxFileBytes) {
|
|
940
|
+
return { status: "too_large" };
|
|
941
|
+
}
|
|
942
|
+
return { status: "ok", bytes };
|
|
943
|
+
}
|
|
944
|
+
// ---------------------------------------------------------------------------
|
|
945
|
+
// Staging and budgets
|
|
946
|
+
// ---------------------------------------------------------------------------
|
|
858
947
|
/**
|
|
859
948
|
* Put these bytes on disk once.
|
|
860
949
|
*
|
|
@@ -896,145 +985,15 @@ function admitToBudget(budget, byteLength) {
|
|
|
896
985
|
budget.remainingBytes -= byteLength;
|
|
897
986
|
return null;
|
|
898
987
|
}
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
});
|
|
907
|
-
if (options.secretLikeFileName) {
|
|
908
|
-
const sanitizedText = redactionResult.redacted
|
|
909
|
-
? redactionResult.text
|
|
910
|
-
: options.text;
|
|
911
|
-
const safeText = containsSecretLikeContent(sanitizedText)
|
|
912
|
-
? "[REDACTED_LINE:secret_redaction_failed]\n"
|
|
913
|
-
: sanitizedText;
|
|
914
|
-
const sanitizedBytes = Buffer.from(safeText, "utf8");
|
|
915
|
-
return {
|
|
916
|
-
status: "redacted",
|
|
917
|
-
bytes: sanitizedBytes,
|
|
918
|
-
redaction: withRedactionContentMetadata(redactionResult.metadata ??
|
|
919
|
-
fallbackRedactionMetadata({
|
|
920
|
-
ruleId: "secret_like_file_name",
|
|
921
|
-
originalText: options.text,
|
|
922
|
-
redactedFields: options.redactedFields,
|
|
923
|
-
fullContentRedacted: false,
|
|
924
|
-
}), { originalBytes, sanitizedBytes }),
|
|
925
|
-
completenessLabel: "secret_like_name_masked",
|
|
926
|
-
};
|
|
927
|
-
}
|
|
928
|
-
if (secretLikeContent) {
|
|
929
|
-
const redaction = redactionResult.metadata ??
|
|
930
|
-
fallbackRedactionMetadata({
|
|
931
|
-
ruleId: "secret_like_content_guard",
|
|
932
|
-
originalText: options.text,
|
|
933
|
-
redactedFields: options.redactedFields,
|
|
934
|
-
});
|
|
935
|
-
let maskedText = maskSecretBearingLines(options.text, redaction);
|
|
936
|
-
if (containsSecretLikeContent(maskedText)) {
|
|
937
|
-
maskedText = "[REDACTED_LINE:secret_redaction_failed]\n";
|
|
938
|
-
}
|
|
939
|
-
const sanitizedBytes = Buffer.from(maskedText, "utf8");
|
|
940
|
-
return {
|
|
941
|
-
status: "redacted",
|
|
942
|
-
bytes: sanitizedBytes,
|
|
943
|
-
redaction: withRedactionContentMetadata(redaction, {
|
|
944
|
-
originalBytes,
|
|
945
|
-
sanitizedBytes,
|
|
946
|
-
}),
|
|
947
|
-
completenessLabel: "secret_content_masked",
|
|
948
|
-
};
|
|
949
|
-
}
|
|
950
|
-
if (redactionResult.redacted) {
|
|
951
|
-
const sanitizedBytes = Buffer.from(redactionResult.text, "utf8");
|
|
952
|
-
return {
|
|
953
|
-
status: "redacted",
|
|
954
|
-
bytes: sanitizedBytes,
|
|
955
|
-
redaction: withRedactionContentMetadata(redactionResult.metadata ??
|
|
956
|
-
fallbackRedactionMetadata({
|
|
957
|
-
ruleId: "secret_redaction_failed",
|
|
958
|
-
originalText: options.text,
|
|
959
|
-
redactedFields: options.redactedFields,
|
|
960
|
-
}), { originalBytes, sanitizedBytes }),
|
|
961
|
-
completenessLabel: "secret_content_masked",
|
|
962
|
-
};
|
|
963
|
-
}
|
|
964
|
-
return { status: "clean", bytes: originalBytes };
|
|
965
|
-
}
|
|
966
|
-
catch {
|
|
967
|
-
const sanitizedBytes = Buffer.from("[REDACTED_LINE:redaction_crashed_stubbed]\n", "utf8");
|
|
968
|
-
return {
|
|
969
|
-
status: "redacted",
|
|
970
|
-
bytes: sanitizedBytes,
|
|
971
|
-
redaction: withRedactionContentMetadata(fallbackRedactionMetadata({
|
|
972
|
-
ruleId: "redaction_crashed_stubbed",
|
|
973
|
-
originalText: options.text,
|
|
974
|
-
redactedFields: options.redactedFields,
|
|
975
|
-
}), { originalBytes, sanitizedBytes }),
|
|
976
|
-
completenessLabel: "redaction_crashed_stubbed",
|
|
977
|
-
};
|
|
978
|
-
}
|
|
979
|
-
}
|
|
980
|
-
function maskSecretBearingLines(text, redaction) {
|
|
981
|
-
if (redaction.redacted_ranges.length === 0) {
|
|
982
|
-
return "[REDACTED_LINE:secret_like_content_guard]\n";
|
|
983
|
-
}
|
|
984
|
-
const segments = text.match(/[^\n]*(?:\n|$)/gu)?.filter(Boolean) ?? [];
|
|
985
|
-
let offset = 0;
|
|
986
|
-
return segments
|
|
987
|
-
.map((segment) => {
|
|
988
|
-
const start = offset;
|
|
989
|
-
const end = offset + segment.length;
|
|
990
|
-
offset = end;
|
|
991
|
-
const matched = redaction.redacted_ranges.find((range) => range.start < end && start < range.end);
|
|
992
|
-
if (!matched)
|
|
993
|
-
return segment;
|
|
994
|
-
const lineEnding = segment.endsWith("\r\n")
|
|
995
|
-
? "\r\n"
|
|
996
|
-
: segment.endsWith("\n")
|
|
997
|
-
? "\n"
|
|
998
|
-
: "";
|
|
999
|
-
return `[REDACTED_LINE:${matched.rule_id}]${lineEnding}`;
|
|
1000
|
-
})
|
|
1001
|
-
.join("");
|
|
1002
|
-
}
|
|
1003
|
-
function fallbackRedactionMetadata(options) {
|
|
1004
|
-
const fullContentRedacted = options.fullContentRedacted !== false;
|
|
1005
|
-
return {
|
|
1006
|
-
schema_version: "raw-evidence-redaction.v1",
|
|
1007
|
-
status: "sanitized",
|
|
1008
|
-
mode: "deterministic_text_replacement",
|
|
1009
|
-
applied_by: ["local_collector"],
|
|
1010
|
-
rule_counts: [
|
|
1011
|
-
{
|
|
1012
|
-
rule_id: options.ruleId,
|
|
1013
|
-
match_count: 1,
|
|
1014
|
-
redacted_char_count: fullContentRedacted
|
|
1015
|
-
? options.originalText.length
|
|
1016
|
-
: 0,
|
|
1017
|
-
},
|
|
1018
|
-
],
|
|
1019
|
-
secret_like_match_count: 1,
|
|
1020
|
-
redacted_fields: options.redactedFields,
|
|
1021
|
-
redacted_ranges: fullContentRedacted && options.originalText.length > 0
|
|
1022
|
-
? [
|
|
1023
|
-
{
|
|
1024
|
-
start: 0,
|
|
1025
|
-
end: options.originalText.length,
|
|
1026
|
-
rule_id: options.ruleId,
|
|
1027
|
-
},
|
|
1028
|
-
]
|
|
1029
|
-
: [],
|
|
1030
|
-
};
|
|
1031
|
-
}
|
|
988
|
+
// ---------------------------------------------------------------------------
|
|
989
|
+
// Git diffs
|
|
990
|
+
// ---------------------------------------------------------------------------
|
|
991
|
+
const GIT_DIFF_TARGETS = [
|
|
992
|
+
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
993
|
+
{ label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
|
|
994
|
+
];
|
|
1032
995
|
async function collectGitDiffFiles(collection, repoRoot) {
|
|
1033
|
-
const
|
|
1034
|
-
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
1035
|
-
{ label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
|
|
1036
|
-
];
|
|
1037
|
-
for (const target of diffTargets) {
|
|
996
|
+
for (const target of GIT_DIFF_TARGETS) {
|
|
1038
997
|
recordScanned(collection, "git_diff");
|
|
1039
998
|
let diff;
|
|
1040
999
|
try {
|
|
@@ -1060,345 +1019,107 @@ async function collectGitDiffFiles(collection, repoRoot) {
|
|
|
1060
1019
|
included_bytes: Buffer.byteLength(diff.stdout, "utf8"),
|
|
1061
1020
|
});
|
|
1062
1021
|
}
|
|
1022
|
+
// An empty diff is not a gap: there was simply nothing to record. The
|
|
1023
|
+
// truncation marker above still stands even when zero bytes survived.
|
|
1063
1024
|
if (!diff.stdout.trim())
|
|
1064
1025
|
continue;
|
|
1065
|
-
|
|
1066
|
-
text: diff.stdout,
|
|
1067
|
-
redactedFields: [`git_diff.${target.label}`],
|
|
1068
|
-
});
|
|
1069
|
-
if (sanitized.status === "redacted") {
|
|
1070
|
-
collection.redacted.push({
|
|
1071
|
-
kind: "git_diff",
|
|
1072
|
-
label: target.label,
|
|
1073
|
-
redaction: sanitized.redaction,
|
|
1074
|
-
completenessLabel: sanitized.completenessLabel,
|
|
1075
|
-
});
|
|
1076
|
-
console.error("[raw-evidence] git diff sanitized", JSON.stringify({
|
|
1077
|
-
mode: sanitized.completenessLabel,
|
|
1078
|
-
original_bytes: Buffer.byteLength(diff.stdout, "utf8"),
|
|
1079
|
-
uploaded_bytes: sanitized.bytes.byteLength,
|
|
1080
|
-
}));
|
|
1081
|
-
}
|
|
1082
|
-
const raw = sanitized.bytes;
|
|
1083
|
-
const redaction = sanitized.redaction;
|
|
1084
|
-
const contentHash = sha256(raw);
|
|
1085
|
-
if (collection.skipContentHashes.has(contentHash)) {
|
|
1086
|
-
collection.reused.push({
|
|
1087
|
-
kind: "git_diff",
|
|
1088
|
-
label: target.label,
|
|
1089
|
-
content_hash_sha256: contentHash,
|
|
1090
|
-
codex_session_id: null,
|
|
1091
|
-
});
|
|
1092
|
-
continue;
|
|
1093
|
-
}
|
|
1094
|
-
const deferReason = admitToBudget(collection.budget, raw.byteLength);
|
|
1095
|
-
if (deferReason) {
|
|
1096
|
-
markBudgetCapApplied(collection, deferReason);
|
|
1097
|
-
collection.skipped.push({
|
|
1098
|
-
kind: "git_diff",
|
|
1099
|
-
label: target.label,
|
|
1100
|
-
reason: deferReason,
|
|
1101
|
-
});
|
|
1102
|
-
continue;
|
|
1103
|
-
}
|
|
1104
|
-
const relativePath = path.join("files", `git-${target.label}.diff`);
|
|
1105
|
-
const diffSourceKey = evidenceSourceKey({
|
|
1106
|
-
kind: "git_diff",
|
|
1107
|
-
sessionId: collection.context.workContextId,
|
|
1026
|
+
await stageOneGitDiff(collection, {
|
|
1108
1027
|
label: target.label,
|
|
1028
|
+
diffText: diff.stdout,
|
|
1029
|
+
truncated: diff.truncated,
|
|
1109
1030
|
});
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
async function stageOneGitDiff(collection, target) {
|
|
1034
|
+
const sanitized = sanitizeTextEvidenceForUpload({
|
|
1035
|
+
text: target.diffText,
|
|
1036
|
+
redactedFields: [`git_diff.${target.label}`],
|
|
1037
|
+
});
|
|
1038
|
+
if (sanitized.status === "redacted") {
|
|
1039
|
+
collection.redacted.push({
|
|
1114
1040
|
kind: "git_diff",
|
|
1115
|
-
|
|
1041
|
+
label: target.label,
|
|
1042
|
+
redaction: sanitized.redaction,
|
|
1043
|
+
completenessLabel: sanitized.completenessLabel,
|
|
1116
1044
|
});
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
localPath: staged.local_path,
|
|
1122
|
-
relativePath,
|
|
1123
|
-
stagedInPack: staged.staged_in_pack,
|
|
1124
|
-
sourceKey: diffSourceKey,
|
|
1125
|
-
mediaType: "text/x-diff",
|
|
1126
|
-
redactedSummary: redaction
|
|
1127
|
-
? `Raw git ${target.label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`
|
|
1128
|
-
: diff.truncated
|
|
1129
|
-
? `Raw git ${target.label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`
|
|
1130
|
-
: `Raw git ${target.label} diff preserved locally with env/secret paths excluded.`,
|
|
1131
|
-
redaction,
|
|
1132
|
-
bytes: raw,
|
|
1133
|
-
contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
|
|
1045
|
+
console.error("[raw-evidence] git diff sanitized", JSON.stringify({
|
|
1046
|
+
mode: sanitized.completenessLabel,
|
|
1047
|
+
original_bytes: Buffer.byteLength(target.diffText, "utf8"),
|
|
1048
|
+
uploaded_bytes: sanitized.bytes.byteLength,
|
|
1134
1049
|
}));
|
|
1135
1050
|
}
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
const
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
":(exclude)**/*credential*",
|
|
1146
|
-
":(exclude)**/*private-key*",
|
|
1147
|
-
":(exclude)**/*.pem",
|
|
1148
|
-
":(exclude)**/*.key",
|
|
1149
|
-
];
|
|
1150
|
-
return new Promise((resolve, reject) => {
|
|
1151
|
-
const child = spawn("git", [...args, ...pathspec], {
|
|
1152
|
-
cwd: repoRoot,
|
|
1153
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
1154
|
-
});
|
|
1155
|
-
const stdoutChunks = [];
|
|
1156
|
-
const stderrChunks = [];
|
|
1157
|
-
let observedBytes = 0;
|
|
1158
|
-
let includedBytes = 0;
|
|
1159
|
-
let truncated = false;
|
|
1160
|
-
let timedOut = false;
|
|
1161
|
-
const timeout = setTimeout(() => {
|
|
1162
|
-
timedOut = true;
|
|
1163
|
-
truncated = true;
|
|
1164
|
-
child.kill("SIGTERM");
|
|
1165
|
-
}, GIT_DIFF_TIMEOUT_MS);
|
|
1166
|
-
child.stdout.on("data", (chunk) => {
|
|
1167
|
-
observedBytes += chunk.byteLength;
|
|
1168
|
-
if (includedBytes < MAX_GIT_DIFF_BYTES) {
|
|
1169
|
-
const remaining = MAX_GIT_DIFF_BYTES - includedBytes;
|
|
1170
|
-
const next = chunk.subarray(0, remaining);
|
|
1171
|
-
stdoutChunks.push(next);
|
|
1172
|
-
includedBytes += next.byteLength;
|
|
1173
|
-
}
|
|
1174
|
-
if (observedBytes > MAX_GIT_DIFF_BYTES) {
|
|
1175
|
-
truncated = true;
|
|
1176
|
-
child.kill("SIGTERM");
|
|
1177
|
-
}
|
|
1178
|
-
});
|
|
1179
|
-
child.stderr.on("data", (chunk) => {
|
|
1180
|
-
if (stderrChunks.reduce((sum, item) => sum + item.byteLength, 0) < 4096) {
|
|
1181
|
-
stderrChunks.push(chunk.subarray(0, 4096));
|
|
1182
|
-
}
|
|
1183
|
-
});
|
|
1184
|
-
child.on("error", (error) => {
|
|
1185
|
-
clearTimeout(timeout);
|
|
1186
|
-
reject(error);
|
|
1051
|
+
const raw = sanitized.bytes;
|
|
1052
|
+
const redaction = sanitized.redaction;
|
|
1053
|
+
const contentHash = sha256(raw);
|
|
1054
|
+
if (collection.skipContentHashes.has(contentHash)) {
|
|
1055
|
+
collection.reused.push({
|
|
1056
|
+
kind: "git_diff",
|
|
1057
|
+
label: target.label,
|
|
1058
|
+
content_hash_sha256: contentHash,
|
|
1059
|
+
codex_session_id: null,
|
|
1187
1060
|
});
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
: "max_git_diff_bytes",
|
|
1198
|
-
truncationCapType: timedOut ? "timeout_ms" : "max_bytes_per_diff",
|
|
1199
|
-
});
|
|
1200
|
-
return;
|
|
1201
|
-
}
|
|
1202
|
-
const stderr = Buffer.concat(stderrChunks).toString("utf8").trim();
|
|
1203
|
-
reject(new Error(stderr || `git diff failed with code ${code ?? signal}`));
|
|
1061
|
+
return;
|
|
1062
|
+
}
|
|
1063
|
+
const deferReason = admitToBudget(collection.budget, raw.byteLength);
|
|
1064
|
+
if (deferReason) {
|
|
1065
|
+
markBudgetCapApplied(collection, deferReason);
|
|
1066
|
+
collection.skipped.push({
|
|
1067
|
+
kind: "git_diff",
|
|
1068
|
+
label: target.label,
|
|
1069
|
+
reason: deferReason,
|
|
1204
1070
|
});
|
|
1205
|
-
});
|
|
1206
|
-
}
|
|
1207
|
-
function recordScanned(collection, source, count = 1) {
|
|
1208
|
-
collection.scanned.set(source, (collection.scanned.get(source) ?? 0) + count);
|
|
1209
|
-
}
|
|
1210
|
-
function markBudgetCapApplied(collection, reason) {
|
|
1211
|
-
markCapApplied(collection, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
|
|
1212
|
-
}
|
|
1213
|
-
function markCapApplied(collection, source, capType) {
|
|
1214
|
-
const cap = collection.caps.find((candidate) => candidate.source === source && candidate.cap_type === capType);
|
|
1215
|
-
if (cap)
|
|
1216
|
-
cap.applied = true;
|
|
1217
|
-
}
|
|
1218
|
-
function recordSkipCount(collection, source, reason, count) {
|
|
1219
|
-
if (count <= 0)
|
|
1220
1071
|
return;
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1072
|
+
}
|
|
1073
|
+
const relativePath = path.join("files", `git-${target.label}.diff`);
|
|
1074
|
+
const diffSourceKey = evidenceSourceKey({
|
|
1075
|
+
kind: "git_diff",
|
|
1076
|
+
sessionId: collection.context.workContextId,
|
|
1077
|
+
label: target.label,
|
|
1226
1078
|
});
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
reason,
|
|
1234
|
-
count,
|
|
1235
|
-
...details,
|
|
1079
|
+
const staged = await stageEvidenceBytes(collection, {
|
|
1080
|
+
contentHash,
|
|
1081
|
+
bytes: raw,
|
|
1082
|
+
fileName: path.basename(relativePath),
|
|
1083
|
+
kind: "git_diff",
|
|
1084
|
+
sourceKey: diffSourceKey,
|
|
1236
1085
|
});
|
|
1086
|
+
collection.entries.push(evidenceEntry({
|
|
1087
|
+
context: collection.context,
|
|
1088
|
+
kind: "git_diff",
|
|
1089
|
+
packId: collection.packId,
|
|
1090
|
+
localPath: staged.local_path,
|
|
1091
|
+
relativePath,
|
|
1092
|
+
stagedInPack: staged.staged_in_pack,
|
|
1093
|
+
sourceKey: diffSourceKey,
|
|
1094
|
+
mediaType: "text/x-diff",
|
|
1095
|
+
redactedSummary: gitDiffSummary(target.label, {
|
|
1096
|
+
redacted: Boolean(redaction),
|
|
1097
|
+
truncated: target.truncated,
|
|
1098
|
+
}),
|
|
1099
|
+
redaction,
|
|
1100
|
+
bytes: raw,
|
|
1101
|
+
contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
|
|
1102
|
+
}));
|
|
1237
1103
|
}
|
|
1238
|
-
function
|
|
1239
|
-
|
|
1240
|
-
}
|
|
1241
|
-
function countEvidenceEntries(entries, predicate = () => true) {
|
|
1242
|
-
return entries.reduce((sum, entry) => sum + (predicate(entry) ? evidenceEntryCount(entry) : 0), 0);
|
|
1243
|
-
}
|
|
1244
|
-
function makeEvidenceCompleteness(collection, options) {
|
|
1245
|
-
const sources = new Set(collection.scanned.keys());
|
|
1246
|
-
for (const entry of collection.entries)
|
|
1247
|
-
sources.add(entry.kind);
|
|
1248
|
-
for (const entry of collection.skipped)
|
|
1249
|
-
sources.add(entry.kind);
|
|
1250
|
-
for (const entry of collection.reused)
|
|
1251
|
-
sources.add(entry.kind);
|
|
1252
|
-
for (const entry of collection.truncated)
|
|
1253
|
-
sources.add(entry.kind);
|
|
1254
|
-
for (const entry of collection.failed)
|
|
1255
|
-
sources.add(entry.kind);
|
|
1256
|
-
for (const entry of collection.redacted)
|
|
1257
|
-
sources.add(entry.kind);
|
|
1258
|
-
const sourceCounts = [...sources].sort().map((source) => {
|
|
1259
|
-
const skipped = collection.skipped.filter((entry) => entry.kind === source);
|
|
1260
|
-
return {
|
|
1261
|
-
source,
|
|
1262
|
-
scanned_count: collection.scanned.get(source) ?? 0,
|
|
1263
|
-
included_count: collection.entries.filter((entry) => entry.kind === source).length,
|
|
1264
|
-
skipped_count: countEvidenceEntries(skipped),
|
|
1265
|
-
truncated_count: countEvidenceEntries(collection.truncated, (entry) => entry.kind === source),
|
|
1266
|
-
deferred_count: countEvidenceEntries(skipped, (entry) => entry.reason.startsWith("deferred_")),
|
|
1267
|
-
reused_count: collection.reused.filter((entry) => entry.kind === source).length,
|
|
1268
|
-
failed_count: countEvidenceEntries(collection.failed, (entry) => entry.kind === source),
|
|
1269
|
-
};
|
|
1270
|
-
});
|
|
1271
|
-
const totals = sourceCounts.reduce((sum, count) => ({
|
|
1272
|
-
scanned_count: sum.scanned_count + count.scanned_count,
|
|
1273
|
-
included_count: sum.included_count + count.included_count,
|
|
1274
|
-
skipped_count: sum.skipped_count + count.skipped_count,
|
|
1275
|
-
truncated_count: sum.truncated_count + count.truncated_count,
|
|
1276
|
-
deferred_count: sum.deferred_count + count.deferred_count,
|
|
1277
|
-
reused_count: sum.reused_count + count.reused_count,
|
|
1278
|
-
failed_count: sum.failed_count + count.failed_count,
|
|
1279
|
-
}), {
|
|
1280
|
-
scanned_count: 0,
|
|
1281
|
-
included_count: 0,
|
|
1282
|
-
skipped_count: 0,
|
|
1283
|
-
truncated_count: 0,
|
|
1284
|
-
deferred_count: 0,
|
|
1285
|
-
reused_count: 0,
|
|
1286
|
-
failed_count: 0,
|
|
1287
|
-
});
|
|
1288
|
-
const skipReasonCounts = new Map();
|
|
1289
|
-
for (const skipped of collection.skipped) {
|
|
1290
|
-
const key = `${skipped.kind}:${skipped.reason}`;
|
|
1291
|
-
const existing = skipReasonCounts.get(key);
|
|
1292
|
-
if (existing) {
|
|
1293
|
-
existing.count += evidenceEntryCount(skipped);
|
|
1294
|
-
}
|
|
1295
|
-
else {
|
|
1296
|
-
skipReasonCounts.set(key, {
|
|
1297
|
-
source: skipped.kind,
|
|
1298
|
-
reason: skipped.reason,
|
|
1299
|
-
count: evidenceEntryCount(skipped),
|
|
1300
|
-
});
|
|
1301
|
-
}
|
|
1104
|
+
function gitDiffSummary(label, state) {
|
|
1105
|
+
if (state.redacted) {
|
|
1106
|
+
return `Raw git ${label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`;
|
|
1302
1107
|
}
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
const key = `${failed.kind}:${failed.reason}`;
|
|
1306
|
-
const existing = failureReasonCounts.get(key);
|
|
1307
|
-
if (existing) {
|
|
1308
|
-
existing.count += evidenceEntryCount(failed);
|
|
1309
|
-
}
|
|
1310
|
-
else {
|
|
1311
|
-
failureReasonCounts.set(key, {
|
|
1312
|
-
source: failed.kind,
|
|
1313
|
-
reason: failed.reason,
|
|
1314
|
-
count: evidenceEntryCount(failed),
|
|
1315
|
-
});
|
|
1316
|
-
}
|
|
1108
|
+
if (state.truncated) {
|
|
1109
|
+
return `Raw git ${label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`;
|
|
1317
1110
|
}
|
|
1318
|
-
|
|
1319
|
-
for (const truncated of collection.truncated) {
|
|
1320
|
-
const key = `${truncated.kind}:${truncated.reason}`;
|
|
1321
|
-
const existing = truncationCounts.get(key);
|
|
1322
|
-
if (existing) {
|
|
1323
|
-
existing.count += evidenceEntryCount(truncated);
|
|
1324
|
-
existing.observed_bytes = Math.max(existing.observed_bytes ?? 0, truncated.observed_bytes ?? 0);
|
|
1325
|
-
existing.included_bytes = Math.max(existing.included_bytes ?? 0, truncated.included_bytes ?? 0);
|
|
1326
|
-
}
|
|
1327
|
-
else {
|
|
1328
|
-
truncationCounts.set(key, {
|
|
1329
|
-
source: truncated.kind,
|
|
1330
|
-
reason: truncated.reason,
|
|
1331
|
-
count: evidenceEntryCount(truncated),
|
|
1332
|
-
...(truncated.max_bytes !== undefined
|
|
1333
|
-
? { max_bytes: truncated.max_bytes }
|
|
1334
|
-
: {}),
|
|
1335
|
-
...(truncated.observed_bytes !== undefined
|
|
1336
|
-
? { observed_bytes: truncated.observed_bytes }
|
|
1337
|
-
: {}),
|
|
1338
|
-
...(truncated.included_bytes !== undefined
|
|
1339
|
-
? { included_bytes: truncated.included_bytes }
|
|
1340
|
-
: {}),
|
|
1341
|
-
});
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1344
|
-
const redactionCounts = new Map();
|
|
1345
|
-
for (const redacted of collection.redacted) {
|
|
1346
|
-
const ruleIds = redacted.redaction.rule_counts.map((rule) => rule.rule_id);
|
|
1347
|
-
const key = `${redacted.kind}:${redacted.completenessLabel}:${ruleIds.sort().join(",")}`;
|
|
1348
|
-
const existing = redactionCounts.get(key);
|
|
1349
|
-
if (existing) {
|
|
1350
|
-
existing.count += 1;
|
|
1351
|
-
existing.rule_ids = [...new Set([...existing.rule_ids, ...ruleIds])].sort();
|
|
1352
|
-
}
|
|
1353
|
-
else {
|
|
1354
|
-
redactionCounts.set(key, {
|
|
1355
|
-
source: redacted.kind,
|
|
1356
|
-
status: "sanitized",
|
|
1357
|
-
mode: redacted.completenessLabel,
|
|
1358
|
-
count: 1,
|
|
1359
|
-
rule_ids: [...new Set(ruleIds)].sort(),
|
|
1360
|
-
});
|
|
1361
|
-
}
|
|
1362
|
-
}
|
|
1363
|
-
const hasGaps = totals.skipped_count > 0 ||
|
|
1364
|
-
totals.truncated_count > 0 ||
|
|
1365
|
-
totals.deferred_count > 0 ||
|
|
1366
|
-
totals.failed_count > 0 ||
|
|
1367
|
-
collection.redacted.length > 0 ||
|
|
1368
|
-
collection.caps.some((cap) => cap.applied);
|
|
1369
|
-
const status = totals.failed_count > 0 &&
|
|
1370
|
-
totals.included_count + totals.reused_count === 0
|
|
1371
|
-
? "failed"
|
|
1372
|
-
: totals.included_count + totals.reused_count === 0 && !hasGaps
|
|
1373
|
-
? "empty"
|
|
1374
|
-
: hasGaps
|
|
1375
|
-
? "partial"
|
|
1376
|
-
: "complete";
|
|
1377
|
-
return EvidenceCompletenessPayloadSchema.parse({
|
|
1378
|
-
schema_version: "evidence-completeness.v1",
|
|
1379
|
-
status,
|
|
1380
|
-
generated_at: options.finishedAt,
|
|
1381
|
-
scan_window: {
|
|
1382
|
-
started_at: options.startedAt,
|
|
1383
|
-
finished_at: options.finishedAt,
|
|
1384
|
-
since_minutes: options.sinceMinutes,
|
|
1385
|
-
},
|
|
1386
|
-
source_counts: sourceCounts,
|
|
1387
|
-
totals,
|
|
1388
|
-
caps: collection.caps,
|
|
1389
|
-
skip_reasons: [...skipReasonCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
|
|
1390
|
-
failure_reasons: [...failureReasonCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
|
|
1391
|
-
truncation_markers: [...truncationCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
|
|
1392
|
-
redaction_markers: [...redactionCounts.values()].sort((a, b) => `${a.source}:${a.mode}`.localeCompare(`${b.source}:${b.mode}`)),
|
|
1393
|
-
notes: totals.failed_count > 0
|
|
1394
|
-
? ["Evidence collection failed; downstream analysis should not infer confidence."]
|
|
1395
|
-
: collection.redacted.length > 0
|
|
1396
|
-
? ["Evidence was sanitized before upload; downstream analysis should not treat it as raw-complete."]
|
|
1397
|
-
: hasGaps
|
|
1398
|
-
? ["Evidence is incomplete; downstream analysis should lower confidence."]
|
|
1399
|
-
: [],
|
|
1400
|
-
});
|
|
1111
|
+
return `Raw git ${label} diff preserved locally with env/secret paths excluded.`;
|
|
1401
1112
|
}
|
|
1113
|
+
// ---------------------------------------------------------------------------
|
|
1114
|
+
// Walking the session store
|
|
1115
|
+
// ---------------------------------------------------------------------------
|
|
1116
|
+
/**
|
|
1117
|
+
* Newest-first `.jsonl` files under these directories, modified since `cutoffMs`.
|
|
1118
|
+
*
|
|
1119
|
+
* Secret-like directory and file names are never descended into or opened;
|
|
1120
|
+
* symlinked duplicates are collapsed by real path so one transcript reachable
|
|
1121
|
+
* two ways is collected once.
|
|
1122
|
+
*/
|
|
1402
1123
|
async function walkJsonlFiles(dir, cutoffMs) {
|
|
1403
1124
|
const out = [];
|
|
1404
1125
|
const stack = Array.isArray(dir) ? [...dir] : [dir];
|
|
@@ -1425,337 +1146,15 @@ async function walkJsonlFiles(dir, cutoffMs) {
|
|
|
1425
1146
|
if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
|
|
1426
1147
|
continue;
|
|
1427
1148
|
const stat = await fs.stat(full);
|
|
1428
|
-
if (stat.mtimeMs
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
}
|
|
1149
|
+
if (stat.mtimeMs < cutoffMs)
|
|
1150
|
+
continue;
|
|
1151
|
+
const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
|
|
1152
|
+
if (seen.has(dedupeKey))
|
|
1153
|
+
continue;
|
|
1154
|
+
seen.add(dedupeKey);
|
|
1155
|
+
out.push({ file: full, mtimeMs: stat.mtimeMs });
|
|
1435
1156
|
}
|
|
1436
1157
|
}
|
|
1437
1158
|
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
1438
1159
|
return out.map((entry) => entry.file);
|
|
1439
|
-
}
|
|
1440
|
-
function makeManifest(options) {
|
|
1441
|
-
return {
|
|
1442
|
-
schema: "bli.local_raw_evidence_pack.v1",
|
|
1443
|
-
pack_id: options.packId,
|
|
1444
|
-
created_at: options.context.now.toISOString(),
|
|
1445
|
-
work_context_id: options.context.workContextId,
|
|
1446
|
-
session_id: options.context.sessionId,
|
|
1447
|
-
operator_id: options.context.operatorId,
|
|
1448
|
-
operator_label: options.context.operatorLabel,
|
|
1449
|
-
repo_label: options.context.repoLabel ?? path.basename(options.context.repoRoot),
|
|
1450
|
-
worktree_label: options.context.worktreeLabel,
|
|
1451
|
-
active_ticket_id: options.context.activeTicketId ?? null,
|
|
1452
|
-
repo_basename: path.basename(options.context.repoRoot),
|
|
1453
|
-
branch: options.context.branch,
|
|
1454
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
1455
|
-
raw_policy: {
|
|
1456
|
-
raw_prompts: "preserved_private_durable_remote",
|
|
1457
|
-
raw_responses: "preserved_private_durable_remote",
|
|
1458
|
-
transcripts: "preserved_private_durable_remote",
|
|
1459
|
-
claude_transcripts: "preserved_private_durable_remote",
|
|
1460
|
-
tool_payloads: "preserved_private_durable_remote",
|
|
1461
|
-
git_diffs: "preserved_private_durable_remote_env_secret_paths_excluded",
|
|
1462
|
-
agent_image_attachments: "preserved_private_durable_remote_explicit_agent_session_attachment_only",
|
|
1463
|
-
env_files: "never_read",
|
|
1464
|
-
stdout: "manifest_only_no_raw_content",
|
|
1465
|
-
},
|
|
1466
|
-
files: options.entries.map(redactManifestEntry),
|
|
1467
|
-
skipped: options.skipped,
|
|
1468
|
-
redacted: options.redacted,
|
|
1469
|
-
reused: options.reused,
|
|
1470
|
-
};
|
|
1471
|
-
}
|
|
1472
|
-
function evidenceEntry(options) {
|
|
1473
|
-
const digest = sha256(options.bytes);
|
|
1474
|
-
const objectKey = remoteObjectKey({
|
|
1475
|
-
context: options.context,
|
|
1476
|
-
packId: options.packId,
|
|
1477
|
-
relativePath: options.relativePath,
|
|
1478
|
-
contentAddress: options.contentAddress,
|
|
1479
|
-
});
|
|
1480
|
-
return {
|
|
1481
|
-
kind: options.kind,
|
|
1482
|
-
local_path: options.localPath,
|
|
1483
|
-
relative_path: options.relativePath,
|
|
1484
|
-
object_key: objectKey,
|
|
1485
|
-
content_hash_sha256: digest,
|
|
1486
|
-
byte_size: options.bytes.byteLength,
|
|
1487
|
-
media_type: options.mediaType,
|
|
1488
|
-
redacted_summary: options.redactedSummary,
|
|
1489
|
-
...(options.redaction ? { redaction: options.redaction } : {}),
|
|
1490
|
-
codex_session_id: options.codexSessionId ?? null,
|
|
1491
|
-
staged_in_pack: options.stagedInPack !== false,
|
|
1492
|
-
source_key: options.sourceKey ?? null,
|
|
1493
|
-
...(options.artifactMetadata
|
|
1494
|
-
? {
|
|
1495
|
-
artifact_metadata: {
|
|
1496
|
-
...options.artifactMetadata,
|
|
1497
|
-
raw_evidence_pointer_id: objectKey,
|
|
1498
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
1499
|
-
object_key: objectKey,
|
|
1500
|
-
content_hash_sha256: digest,
|
|
1501
|
-
byte_size: options.bytes.byteLength,
|
|
1502
|
-
},
|
|
1503
|
-
}
|
|
1504
|
-
: {}),
|
|
1505
|
-
};
|
|
1506
|
-
}
|
|
1507
|
-
// `local_path` is redacted out, and so are the staging bookkeeping fields:
|
|
1508
|
-
// where a copy happens to live on this disk is not part of the pack's identity,
|
|
1509
|
-
// and putting it in the manifest would make byte-identical content produce
|
|
1510
|
-
// different manifests.
|
|
1511
|
-
function redactManifestEntry(entry) {
|
|
1512
|
-
return {
|
|
1513
|
-
kind: entry.kind,
|
|
1514
|
-
relative_path: entry.relative_path,
|
|
1515
|
-
object_key: entry.object_key,
|
|
1516
|
-
content_hash_sha256: entry.content_hash_sha256,
|
|
1517
|
-
byte_size: entry.byte_size,
|
|
1518
|
-
media_type: entry.media_type,
|
|
1519
|
-
redacted_summary: entry.redacted_summary,
|
|
1520
|
-
...(entry.redaction ? { redaction: entry.redaction } : {}),
|
|
1521
|
-
...(entry.artifact_metadata
|
|
1522
|
-
? { artifact_metadata: entry.artifact_metadata }
|
|
1523
|
-
: {}),
|
|
1524
|
-
};
|
|
1525
|
-
}
|
|
1526
|
-
function pointerFromEntry(entry) {
|
|
1527
|
-
return {
|
|
1528
|
-
raw_evidence_pointer_id: entry.object_key,
|
|
1529
|
-
privacy_classification: "remote_durable_raw_evidence",
|
|
1530
|
-
retention_policy: {
|
|
1531
|
-
mode: RAW_EVIDENCE_RETENTION_MODE,
|
|
1532
|
-
privacy_classification: "remote_durable_raw_evidence",
|
|
1533
|
-
},
|
|
1534
|
-
storage_scope: "remote_object",
|
|
1535
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
1536
|
-
object_key: entry.object_key,
|
|
1537
|
-
content_hash_sha256: entry.content_hash_sha256,
|
|
1538
|
-
byte_size: entry.byte_size,
|
|
1539
|
-
media_type: entry.media_type,
|
|
1540
|
-
redacted_summary: entry.redacted_summary,
|
|
1541
|
-
...(entry.redaction ? { redaction: entry.redaction } : {}),
|
|
1542
|
-
};
|
|
1543
|
-
}
|
|
1544
|
-
/**
|
|
1545
|
-
* Raw evidence keys start with human-readable context, then end in immutable
|
|
1546
|
-
* content addresses or pack-relative manifest paths. The local cursor reuses
|
|
1547
|
-
* prior content hashes across syncs; the readable date/session folders are for
|
|
1548
|
-
* operator debugging and incident response.
|
|
1549
|
-
*/
|
|
1550
|
-
function remoteObjectKey(options) {
|
|
1551
|
-
const namespace = readableEvidenceNamespace(options.context);
|
|
1552
|
-
if (options.contentAddress) {
|
|
1553
|
-
return posixPath([
|
|
1554
|
-
...namespace,
|
|
1555
|
-
options.contentAddress,
|
|
1556
|
-
]);
|
|
1557
|
-
}
|
|
1558
|
-
return posixPath([
|
|
1559
|
-
...namespace,
|
|
1560
|
-
options.packId,
|
|
1561
|
-
options.relativePath,
|
|
1562
|
-
]);
|
|
1563
|
-
}
|
|
1564
|
-
function readableEvidenceNamespace(context) {
|
|
1565
|
-
return [
|
|
1566
|
-
"operators",
|
|
1567
|
-
operatorSlug(context),
|
|
1568
|
-
"repos",
|
|
1569
|
-
readableKeySegment(context.repoLabel ?? path.basename(context.repoRoot), "repo"),
|
|
1570
|
-
"worktrees",
|
|
1571
|
-
readableKeySegment(context.worktreeLabel ?? path.basename(context.repoRoot), "worktree"),
|
|
1572
|
-
"tickets",
|
|
1573
|
-
readableKeySegment(context.activeTicketId ?? "unbound", "unbound", {
|
|
1574
|
-
lowercase: false,
|
|
1575
|
-
}),
|
|
1576
|
-
"dates",
|
|
1577
|
-
context.now.toISOString().slice(0, 10),
|
|
1578
|
-
"sessions",
|
|
1579
|
-
readableKeySegment(context.sessionId, "session"),
|
|
1580
|
-
"ids",
|
|
1581
|
-
safeKeySegment(context.operatorId),
|
|
1582
|
-
safeKeySegment(context.workContextId),
|
|
1583
|
-
];
|
|
1584
|
-
}
|
|
1585
|
-
function operatorSlug(context) {
|
|
1586
|
-
const labelBeforeDomain = (context.operatorLabel ?? context.operatorId)
|
|
1587
|
-
.split("@", 1)[0]
|
|
1588
|
-
.trim();
|
|
1589
|
-
const readable = readableKeySegment(labelBeforeDomain, "operator");
|
|
1590
|
-
return `${readable}-${shortHash(context.operatorId).slice(0, 6)}`;
|
|
1591
|
-
}
|
|
1592
|
-
function readableKeySegment(value, fallback, options = {}) {
|
|
1593
|
-
const base = options.lowercase === false ? value : value.toLowerCase();
|
|
1594
|
-
const slug = base
|
|
1595
|
-
.trim()
|
|
1596
|
-
.replace(/[^A-Za-z0-9._-]+/g, "-")
|
|
1597
|
-
.replace(/^-+|-+$/g, "")
|
|
1598
|
-
.replace(/-{2,}/g, "-")
|
|
1599
|
-
.slice(0, 80);
|
|
1600
|
-
return slug || fallback;
|
|
1601
|
-
}
|
|
1602
|
-
function posixPath(parts) {
|
|
1603
|
-
return parts.join("/").replace(/\\/g, "/").replace(/\/+/g, "/");
|
|
1604
|
-
}
|
|
1605
|
-
/**
|
|
1606
|
-
* Turn the staging directory into the content-keyed pack directory.
|
|
1607
|
-
*
|
|
1608
|
-
* Three outcomes, all of them named:
|
|
1609
|
-
* - the pack does not exist yet: rename staging into place (`new`);
|
|
1610
|
-
* - it exists and holds every file this pass staged: adopt it and delete the
|
|
1611
|
-
* staging copy (`reused`) — nothing is copied twice;
|
|
1612
|
-
* - it exists but is missing files: fill the gaps from staging
|
|
1613
|
-
* (`restaged_incomplete`), because a half-written pack must not be trusted.
|
|
1614
|
-
*
|
|
1615
|
-
* Never deletes an existing pack directory wholesale: another pack's entries
|
|
1616
|
-
* can point into it through the staged-object index.
|
|
1617
|
-
*/
|
|
1618
|
-
async function promoteStagedPack(options) {
|
|
1619
|
-
const evidenceDir = path.join(options.rawEvidenceRoot, options.packId);
|
|
1620
|
-
const priorPackCount = await countPriorPacks(options.rawEvidenceRoot, options.workContextId, options.packId);
|
|
1621
|
-
const existing = await fs.stat(evidenceDir).catch(() => null);
|
|
1622
|
-
if (!existing?.isDirectory()) {
|
|
1623
|
-
try {
|
|
1624
|
-
await fs.rename(options.stagingDir, evidenceDir);
|
|
1625
|
-
return {
|
|
1626
|
-
state: "new",
|
|
1627
|
-
evidenceDir,
|
|
1628
|
-
priorPackCount,
|
|
1629
|
-
refilledFileCount: 0,
|
|
1630
|
-
};
|
|
1631
|
-
}
|
|
1632
|
-
catch {
|
|
1633
|
-
// A concurrent sync can win the race to the same content-keyed name.
|
|
1634
|
-
// Losing it is fine: the winner staged the identical bytes.
|
|
1635
|
-
}
|
|
1636
|
-
}
|
|
1637
|
-
let refilledFileCount = 0;
|
|
1638
|
-
await ensurePrivateDir(path.join(evidenceDir, "files"));
|
|
1639
|
-
for (const entry of options.entries) {
|
|
1640
|
-
if (!entry.staged_in_pack)
|
|
1641
|
-
continue;
|
|
1642
|
-
const fileName = path.basename(entry.local_path);
|
|
1643
|
-
const target = path.join(evidenceDir, "files", fileName);
|
|
1644
|
-
const info = await fs.stat(target).catch(() => null);
|
|
1645
|
-
if (info?.isFile() && info.size === entry.byte_size)
|
|
1646
|
-
continue;
|
|
1647
|
-
await fs.copyFile(entry.local_path, target).catch(() => undefined);
|
|
1648
|
-
await chmodPrivate(target, 0o600);
|
|
1649
|
-
refilledFileCount += 1;
|
|
1650
|
-
}
|
|
1651
|
-
await fs.rm(options.stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
|
1652
|
-
return {
|
|
1653
|
-
state: refilledFileCount > 0 ? "restaged_incomplete" : "reused",
|
|
1654
|
-
evidenceDir,
|
|
1655
|
-
priorPackCount,
|
|
1656
|
-
refilledFileCount,
|
|
1657
|
-
};
|
|
1658
|
-
}
|
|
1659
|
-
async function countPriorPacks(rawEvidenceRoot, workContextId, packId) {
|
|
1660
|
-
const entries = await fs
|
|
1661
|
-
.readdir(rawEvidenceRoot, { withFileTypes: true })
|
|
1662
|
-
.catch(() => []);
|
|
1663
|
-
return entries.filter((entry) => entry.isDirectory() &&
|
|
1664
|
-
entry.name !== packId &&
|
|
1665
|
-
entry.name.startsWith(`${workContextId}-`)).length;
|
|
1666
|
-
}
|
|
1667
|
-
/**
|
|
1668
|
-
* Write the manifest, or keep the one already in a reused pack.
|
|
1669
|
-
*
|
|
1670
|
-
* Byte-stability matters here: the manifest's object key is the only key that
|
|
1671
|
-
* embeds the pack id, so rewriting it with a fresh `created_at` every sync
|
|
1672
|
-
* would push different bytes at the same content-addressed key forever. A
|
|
1673
|
-
* reused pack whose manifest already describes exactly these files keeps it.
|
|
1674
|
-
*/
|
|
1675
|
-
async function stageManifest(options) {
|
|
1676
|
-
if (options.reusePack) {
|
|
1677
|
-
const existing = await fs.readFile(options.manifestPath).catch(() => null);
|
|
1678
|
-
if (existing && manifestDescribesEntries(existing, options.entries)) {
|
|
1679
|
-
return existing;
|
|
1680
|
-
}
|
|
1681
|
-
}
|
|
1682
|
-
const manifestBytes = Buffer.from(`${JSON.stringify(makeManifest({
|
|
1683
|
-
context: options.context,
|
|
1684
|
-
packId: options.packId,
|
|
1685
|
-
entries: options.entries,
|
|
1686
|
-
skipped: options.skipped,
|
|
1687
|
-
redacted: options.redacted,
|
|
1688
|
-
reused: options.reused,
|
|
1689
|
-
}), null, 2)}\n`, "utf8");
|
|
1690
|
-
await fs.writeFile(options.manifestPath, manifestBytes, { mode: 0o600 });
|
|
1691
|
-
await chmodPrivate(options.manifestPath, 0o600);
|
|
1692
|
-
return manifestBytes;
|
|
1693
|
-
}
|
|
1694
|
-
function manifestDescribesEntries(manifestBytes, entries) {
|
|
1695
|
-
try {
|
|
1696
|
-
const parsed = JSON.parse(manifestBytes.toString("utf8"));
|
|
1697
|
-
const recorded = (parsed.files ?? [])
|
|
1698
|
-
.map((file) => typeof file.content_hash_sha256 === "string"
|
|
1699
|
-
? file.content_hash_sha256
|
|
1700
|
-
: "")
|
|
1701
|
-
.filter(Boolean)
|
|
1702
|
-
.sort();
|
|
1703
|
-
const expected = entries
|
|
1704
|
-
.map((entry) => entry.content_hash_sha256)
|
|
1705
|
-
.sort();
|
|
1706
|
-
return (recorded.length === expected.length &&
|
|
1707
|
-
recorded.every((hash, index) => hash === expected[index]));
|
|
1708
|
-
}
|
|
1709
|
-
catch {
|
|
1710
|
-
return false;
|
|
1711
|
-
}
|
|
1712
|
-
}
|
|
1713
|
-
async function persistStagingState(stateDir, collection) {
|
|
1714
|
-
collection.staging.updated_at = collection.context.now.toISOString();
|
|
1715
|
-
await writeRawEvidenceStagingState(stateDir, collection.staging).catch((error) => {
|
|
1716
|
-
// Losing this file costs reuse and attempt counts, not evidence, so it
|
|
1717
|
-
// must never fail a collection — but it must never be silent either.
|
|
1718
|
-
console.error("[raw-evidence] staging state write failed", JSON.stringify({
|
|
1719
|
-
reason: "staging_state_write_failed",
|
|
1720
|
-
detail: error instanceof Error ? error.name : typeof error,
|
|
1721
|
-
staged_count: Object.keys(collection.staging.staged).length,
|
|
1722
|
-
}));
|
|
1723
|
-
});
|
|
1724
|
-
}
|
|
1725
|
-
function shortHash(value) {
|
|
1726
|
-
return crypto.createHash("sha256").update(value, "utf8").digest("hex").slice(0, 12);
|
|
1727
|
-
}
|
|
1728
|
-
/**
|
|
1729
|
-
* Object keys must satisfy the server's key pattern; ids derived from file
|
|
1730
|
-
* content fall back to a hash rather than failing the whole upload batch.
|
|
1731
|
-
*/
|
|
1732
|
-
function safeKeySegment(value) {
|
|
1733
|
-
return /^[A-Za-z0-9._-]{1,80}$/.test(value) ? value : shortHash(value);
|
|
1734
|
-
}
|
|
1735
|
-
function sha256(value) {
|
|
1736
|
-
return crypto.createHash("sha256").update(value).digest("hex");
|
|
1737
|
-
}
|
|
1738
|
-
function withRedactionContentMetadata(metadata, options) {
|
|
1739
|
-
if (!metadata) {
|
|
1740
|
-
throw new Error("redacted evidence is missing redaction metadata");
|
|
1741
|
-
}
|
|
1742
|
-
return {
|
|
1743
|
-
...metadata,
|
|
1744
|
-
original_content_hash_sha256: sha256(options.originalBytes),
|
|
1745
|
-
sanitized_content_hash_sha256: sha256(options.sanitizedBytes),
|
|
1746
|
-
original_byte_size: options.originalBytes.byteLength,
|
|
1747
|
-
sanitized_byte_size: options.sanitizedBytes.byteLength,
|
|
1748
|
-
};
|
|
1749
|
-
}
|
|
1750
|
-
async function ensurePrivateDir(dir) {
|
|
1751
|
-
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
1752
|
-
await chmodPrivate(dir, 0o700);
|
|
1753
|
-
}
|
|
1754
|
-
async function chmodPrivate(target, mode) {
|
|
1755
|
-
if (process.platform === "win32")
|
|
1756
|
-
return;
|
|
1757
|
-
await fs.chmod(target, mode).catch(() => undefined);
|
|
1758
|
-
}
|
|
1759
|
-
function isSecretLikePath(value) {
|
|
1760
|
-
return SECRET_FILE_SEGMENT_PATTERN.test(value);
|
|
1761
1160
|
}
|