@bli-cockpit/cli 0.2.27 → 0.2.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/raw-evidence-completeness.js +226 -0
- package/dist/adapters/raw-evidence-git-diff.js +90 -0
- package/dist/adapters/raw-evidence-keys.js +92 -0
- package/dist/adapters/raw-evidence-manifest.js +132 -0
- package/dist/adapters/raw-evidence-pack-store.js +136 -0
- package/dist/adapters/raw-evidence-sanitize.js +190 -0
- package/dist/adapters/raw-evidence.js +740 -1044
- package/dist/commands/backfill.js +7 -0
- package/dist/commands/cli-io.js +92 -0
- package/dist/commands/collection-report.js +135 -0
- package/dist/commands/collection-roots.js +153 -0
- package/dist/commands/install-receipts.js +193 -0
- package/dist/commands/install-update.js +305 -0
- package/dist/commands/local-auth.js +268 -0
- package/dist/commands/local-discovery.js +100 -0
- package/dist/commands/local-help.js +281 -0
- package/dist/commands/local.js +183 -1842
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sessions.js +162 -0
- package/dist/commands/status.js +191 -0
- package/dist/evidence-upload-client.js +43 -2
- package/dist/local-state.js +12 -1
- package/dist/raw-evidence-gc.js +178 -0
- package/dist/raw-evidence-staging.js +322 -0
- package/dist/upload-agent-artifacts.js +153 -0
- package/dist/upload-envelope.js +407 -0
- package/dist/upload-evidence-delivery.js +505 -0
- package/dist/upload-http.js +46 -0
- package/dist/upload-session-reports.js +404 -0
- package/dist/upload.js +153 -1159
- package/package.json +2 -2
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { spawn } from "node:child_process";
|
|
1
|
+
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, SourceScanResultSchema, } from "@bli-cockpit/telemetry-core";
|
|
3
2
|
import crypto from "node:crypto";
|
|
4
3
|
import fs from "node:fs/promises";
|
|
5
4
|
import os from "node:os";
|
|
@@ -8,12 +7,18 @@ import { makeSourceAdapterIdentity, } from "./common.js";
|
|
|
8
7
|
import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
|
|
9
8
|
import { defaultCodexSessionDirs, } from "./codex-attribution.js";
|
|
10
9
|
import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-policy.js";
|
|
10
|
+
import { contentKeyedRawEvidencePackId, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, evidenceSourceKey, heldSourceKeys, readRawEvidenceStagingState, recordStagedObject, resolveStagedObject, } from "../raw-evidence-staging.js";
|
|
11
|
+
import { isSecretLikePath, safeKeySegment, sha256, shortHash, } from "./raw-evidence-keys.js";
|
|
12
|
+
import { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
|
|
13
|
+
import { countEvidenceEntries, makeEvidenceCompleteness, markBudgetCapApplied, markCapApplied, recordScanned, recordSkipCount, recordTruncationCount, } from "./raw-evidence-completeness.js";
|
|
14
|
+
import { evidenceEntry, pointerFromEntry, RAW_EVIDENCE_BUCKET, } from "./raw-evidence-manifest.js";
|
|
15
|
+
import { chmodPrivate, ensurePrivateDir, persistStagingState, promoteStagedPack, stageManifest, } from "./raw-evidence-pack-store.js";
|
|
16
|
+
import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, runGitDiff, } from "./raw-evidence-git-diff.js";
|
|
17
|
+
// Re-exported so every consumer keeps importing from `adapters/raw-evidence`.
|
|
18
|
+
export { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./raw-evidence-manifest.js";
|
|
19
|
+
export { sanitizeTextEvidenceForUpload } from "./raw-evidence-sanitize.js";
|
|
11
20
|
const DEFAULT_SINCE_MINUTES = 24 * 60;
|
|
12
21
|
const DEFAULT_SESSION_LIMIT = 50;
|
|
13
|
-
const MAX_GIT_DIFF_BYTES = 2 * 1024 * 1024;
|
|
14
|
-
const GIT_DIFF_TIMEOUT_MS = 3_000;
|
|
15
|
-
export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
|
|
16
|
-
export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
|
|
17
22
|
// Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
|
|
18
23
|
// transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
|
|
19
24
|
// so 2 GiB leaves room for several files without starving the sync; overflow
|
|
@@ -21,250 +26,394 @@ export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
|
|
|
21
26
|
export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
|
|
22
27
|
export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
|
|
23
28
|
const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
// The pass
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
24
32
|
export async function collectRawEvidencePack(context, options) {
|
|
25
33
|
const startedAt = context.now.toISOString();
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
const
|
|
32
|
-
const failed = [];
|
|
33
|
-
const redacted = [];
|
|
34
|
-
const reused = [];
|
|
34
|
+
const rawEvidenceRoot = path.join(options.stateDir, "raw-evidence");
|
|
35
|
+
// Staging first, promotion second (BLI-3066). The pack id cannot be known
|
|
36
|
+
// until the content is, so bytes land in a private staging directory and the
|
|
37
|
+
// directory is then renamed to its content-keyed name — or dropped, when an
|
|
38
|
+
// identical pack is already there.
|
|
39
|
+
const stagingDir = path.join(rawEvidenceRoot, `.staging-${process.pid}-${crypto.randomUUID().slice(0, 8)}`);
|
|
35
40
|
const sinceMinutes = options.sinceMinutes ?? DEFAULT_SINCE_MINUTES;
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
skipped,
|
|
45
|
-
truncated,
|
|
46
|
-
failed,
|
|
47
|
-
redacted,
|
|
48
|
-
reused,
|
|
49
|
-
scanned: new Map(),
|
|
50
|
-
caps: [
|
|
51
|
-
{
|
|
52
|
-
source: "raw_evidence",
|
|
53
|
-
cap_type: "byte_budget",
|
|
54
|
-
limit: byteBudget,
|
|
55
|
-
observed: options.budget?.remainingBytes ?? byteBudget,
|
|
56
|
-
applied: false,
|
|
57
|
-
},
|
|
58
|
-
{
|
|
59
|
-
source: "raw_evidence",
|
|
60
|
-
cap_type: "object_budget",
|
|
61
|
-
limit: objectBudget,
|
|
62
|
-
observed: options.budget?.remainingObjects ?? objectBudget,
|
|
63
|
-
applied: false,
|
|
64
|
-
},
|
|
65
|
-
{
|
|
66
|
-
source: "git_diff",
|
|
67
|
-
cap_type: "max_bytes_per_diff",
|
|
68
|
-
limit: MAX_GIT_DIFF_BYTES,
|
|
69
|
-
applied: false,
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
source: "git_diff",
|
|
73
|
-
cap_type: "timeout_ms",
|
|
74
|
-
limit: GIT_DIFF_TIMEOUT_MS,
|
|
75
|
-
applied: false,
|
|
76
|
-
},
|
|
77
|
-
{
|
|
78
|
-
source: "codex_jsonl",
|
|
79
|
-
cap_type: "max_file_bytes",
|
|
80
|
-
limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
81
|
-
applied: false,
|
|
82
|
-
},
|
|
83
|
-
{
|
|
84
|
-
source: "claude_jsonl",
|
|
85
|
-
cap_type: "max_file_bytes",
|
|
86
|
-
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
87
|
-
applied: false,
|
|
88
|
-
},
|
|
89
|
-
{
|
|
90
|
-
source: "claude_jsonl_sidecar",
|
|
91
|
-
cap_type: "max_file_bytes",
|
|
92
|
-
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
93
|
-
applied: false,
|
|
94
|
-
},
|
|
95
|
-
],
|
|
96
|
-
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
97
|
-
budget: options.budget ?? {
|
|
98
|
-
remainingBytes: byteBudget,
|
|
99
|
-
remainingObjects: objectBudget,
|
|
100
|
-
},
|
|
101
|
-
index: { value: 0 },
|
|
41
|
+
const collection = await openCollection(context, options, {
|
|
42
|
+
rawEvidenceRoot,
|
|
43
|
+
stagingDir,
|
|
44
|
+
});
|
|
45
|
+
const scanWindow = {
|
|
46
|
+
startedAt,
|
|
47
|
+
finishedAt: () => context.now.toISOString(),
|
|
48
|
+
sinceMinutes,
|
|
102
49
|
};
|
|
103
50
|
try {
|
|
104
|
-
await ensurePrivateDir(
|
|
105
|
-
await ensurePrivateDir(filesDir);
|
|
106
|
-
recordAttributionCompleteness(collection,
|
|
107
|
-
codex: options.codexAttributionScan,
|
|
108
|
-
claude: options.claudeAttributionScan,
|
|
109
|
-
selectedCodexPaths: new Set(options.codexSessionFiles?.map((file) => file.local_path) ?? []),
|
|
110
|
-
selectedClaudePaths: new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []),
|
|
111
|
-
});
|
|
51
|
+
await ensurePrivateDir(stagingDir);
|
|
52
|
+
await ensurePrivateDir(collection.filesDir);
|
|
53
|
+
recordAttributionCompleteness(collection, options);
|
|
112
54
|
if (options.includeCodexJsonl !== false) {
|
|
113
55
|
await collectCodexJsonlFiles(collection, {
|
|
114
56
|
codexSessionFiles: options.codexSessionFiles,
|
|
115
57
|
sessionsDir: options.sessionsDir,
|
|
116
58
|
sessionsDirs: options.sessionsDirs,
|
|
117
59
|
sinceMinutes,
|
|
118
|
-
limit: sessionLimit,
|
|
60
|
+
limit: options.sessionLimit ?? DEFAULT_SESSION_LIMIT,
|
|
119
61
|
});
|
|
120
62
|
}
|
|
121
63
|
if (options.includeClaudeJsonl !== false && options.claudeSessionFiles) {
|
|
122
64
|
await collectClaudeJsonlFiles(collection, options.claudeSessionFiles);
|
|
123
65
|
}
|
|
124
66
|
await collectGitDiffFiles(collection, options.repoRoot);
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
startedAt,
|
|
130
|
-
finishedAt: context.now.toISOString(),
|
|
131
|
-
sinceMinutes,
|
|
67
|
+
if (collection.entries.length === 0) {
|
|
68
|
+
return await finishWithEmptyPack(collection, options, {
|
|
69
|
+
stagingDir,
|
|
70
|
+
window: scanWindow,
|
|
132
71
|
});
|
|
133
|
-
const facts = {
|
|
134
|
-
pack_id: packId,
|
|
135
|
-
manifest_path: path.join(evidenceDir, "manifest.json"),
|
|
136
|
-
evidence_dir: evidenceDir,
|
|
137
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
138
|
-
file_count: 0,
|
|
139
|
-
byte_size: 0,
|
|
140
|
-
skipped_count: countEvidenceEntries(skipped),
|
|
141
|
-
sanitized_count: redacted.length,
|
|
142
|
-
reused_count: reused.length,
|
|
143
|
-
deferred_byte_budget_count: deferredByteBudgetCount,
|
|
144
|
-
deferred_object_budget_count: deferredObjectBudgetCount,
|
|
145
|
-
content_kinds: [],
|
|
146
|
-
evidence_completeness: evidenceCompleteness,
|
|
147
|
-
pointers: [],
|
|
148
|
-
upload_files: [],
|
|
149
|
-
reused,
|
|
150
|
-
};
|
|
151
|
-
return {
|
|
152
|
-
facts,
|
|
153
|
-
scan: makeRawEvidenceScan({
|
|
154
|
-
context,
|
|
155
|
-
startedAt,
|
|
156
|
-
status: "partial",
|
|
157
|
-
facts,
|
|
158
|
-
}),
|
|
159
|
-
};
|
|
160
72
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
entries,
|
|
165
|
-
skipped,
|
|
166
|
-
redacted,
|
|
167
|
-
reused,
|
|
73
|
+
return await finishWithPromotedPack(collection, options, {
|
|
74
|
+
stagingDir,
|
|
75
|
+
window: scanWindow,
|
|
168
76
|
});
|
|
169
|
-
const manifestPath = path.join(evidenceDir, "manifest.json");
|
|
170
|
-
const manifestBytes = Buffer.from(`${JSON.stringify(manifestWithoutSelf, null, 2)}\n`, "utf8");
|
|
171
|
-
await fs.writeFile(manifestPath, manifestBytes, { mode: 0o600 });
|
|
172
|
-
await chmodPrivate(manifestPath, 0o600);
|
|
173
|
-
const manifestEntry = evidenceEntry({
|
|
174
|
-
context,
|
|
175
|
-
kind: "manifest",
|
|
176
|
-
packId,
|
|
177
|
-
localPath: manifestPath,
|
|
178
|
-
relativePath: "manifest.json",
|
|
179
|
-
mediaType: "application/json",
|
|
180
|
-
redactedSummary: "Local raw evidence pack manifest.",
|
|
181
|
-
bytes: manifestBytes,
|
|
182
|
-
});
|
|
183
|
-
entries.push(manifestEntry);
|
|
184
|
-
const evidenceCompleteness = makeEvidenceCompleteness(collection, {
|
|
185
|
-
startedAt,
|
|
186
|
-
finishedAt: context.now.toISOString(),
|
|
187
|
-
sinceMinutes,
|
|
188
|
-
});
|
|
189
|
-
const facts = {
|
|
190
|
-
pack_id: packId,
|
|
191
|
-
manifest_path: manifestPath,
|
|
192
|
-
evidence_dir: evidenceDir,
|
|
193
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
194
|
-
file_count: entries.length,
|
|
195
|
-
byte_size: entries.reduce((sum, entry) => sum + entry.byte_size, 0),
|
|
196
|
-
skipped_count: countEvidenceEntries(skipped),
|
|
197
|
-
sanitized_count: redacted.length,
|
|
198
|
-
reused_count: reused.length,
|
|
199
|
-
deferred_byte_budget_count: deferredByteBudgetCount,
|
|
200
|
-
deferred_object_budget_count: deferredObjectBudgetCount,
|
|
201
|
-
content_kinds: [...new Set(entries.map((entry) => entry.kind))],
|
|
202
|
-
evidence_completeness: evidenceCompleteness,
|
|
203
|
-
pointers: entries.map(pointerFromEntry),
|
|
204
|
-
upload_files: entries.map((entry) => ({
|
|
205
|
-
pointer: pointerFromEntry(entry),
|
|
206
|
-
local_path: entry.local_path,
|
|
207
|
-
kind: entry.kind,
|
|
208
|
-
codex_session_id: entry.codex_session_id ?? null,
|
|
209
|
-
...(entry.artifact_metadata
|
|
210
|
-
? { artifact_metadata: entry.artifact_metadata }
|
|
211
|
-
: {}),
|
|
212
|
-
})),
|
|
213
|
-
reused,
|
|
214
|
-
};
|
|
215
|
-
return {
|
|
216
|
-
facts,
|
|
217
|
-
scan: makeRawEvidenceScan({
|
|
218
|
-
context,
|
|
219
|
-
startedAt,
|
|
220
|
-
status: entries.length > 1 ? "ok" : "partial",
|
|
221
|
-
facts,
|
|
222
|
-
}),
|
|
223
|
-
};
|
|
224
77
|
}
|
|
225
|
-
catch {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
78
|
+
catch (error) {
|
|
79
|
+
return await finishWithFailedPack(collection, {
|
|
80
|
+
stagingDir,
|
|
81
|
+
window: scanWindow,
|
|
82
|
+
error,
|
|
229
83
|
});
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
async function openCollection(context, options, places) {
|
|
87
|
+
const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
|
|
88
|
+
const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
|
|
89
|
+
const staging = await readRawEvidenceStagingState(options.stateDir);
|
|
90
|
+
return {
|
|
91
|
+
context,
|
|
92
|
+
filesDir: path.join(places.stagingDir, "files"),
|
|
93
|
+
rawEvidenceRoot: places.rawEvidenceRoot,
|
|
94
|
+
packId: "",
|
|
95
|
+
staging,
|
|
96
|
+
heldSources: collectionHeldSources(staging, context.now, options),
|
|
97
|
+
stagedReusedCount: 0,
|
|
98
|
+
stagedNewCount: 0,
|
|
99
|
+
deliveryHeldCount: 0,
|
|
100
|
+
entries: [],
|
|
101
|
+
skipped: [],
|
|
102
|
+
truncated: [],
|
|
103
|
+
failed: [],
|
|
104
|
+
redacted: [],
|
|
105
|
+
reused: [],
|
|
106
|
+
scanned: new Map(),
|
|
107
|
+
caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
|
|
108
|
+
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
109
|
+
budget: options.budget ?? {
|
|
110
|
+
remainingBytes: byteBudget,
|
|
111
|
+
remainingObjects: objectBudget,
|
|
112
|
+
},
|
|
113
|
+
index: { value: 0 },
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Which sources this pass refuses to even read because their delivery is inside
|
|
118
|
+
* a backoff window — none of them, when a person asked for this pass.
|
|
119
|
+
*
|
|
120
|
+
* The bypass is logged rather than assumed: an operator retry that quietly
|
|
121
|
+
* ignored a hold would be as unreadable as the hold that quietly blocked it.
|
|
122
|
+
*/
|
|
123
|
+
function collectionHeldSources(staging, now, options) {
|
|
124
|
+
const held = heldSourceKeys(staging, now);
|
|
125
|
+
if (deliveryBackoffApplies(options.deliveryMode))
|
|
126
|
+
return held;
|
|
127
|
+
if (held.size > 0) {
|
|
128
|
+
console.error("[raw-evidence] delivery backoff bypassed for operator retry", JSON.stringify({
|
|
129
|
+
reason: DELIVERY_BACKOFF_BYPASS_REASON,
|
|
130
|
+
source_count: held.size,
|
|
131
|
+
}));
|
|
132
|
+
}
|
|
133
|
+
return new Set();
|
|
134
|
+
}
|
|
135
|
+
/** Every cap this pass could hit, declared up front and flipped when applied. */
|
|
136
|
+
function startingCaps(options) {
|
|
137
|
+
return [
|
|
138
|
+
{
|
|
139
|
+
source: "raw_evidence",
|
|
140
|
+
cap_type: "byte_budget",
|
|
141
|
+
limit: options.byteBudget,
|
|
142
|
+
observed: options.budget?.remainingBytes ?? options.byteBudget,
|
|
143
|
+
applied: false,
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
source: "raw_evidence",
|
|
147
|
+
cap_type: "object_budget",
|
|
148
|
+
limit: options.objectBudget,
|
|
149
|
+
observed: options.budget?.remainingObjects ?? options.objectBudget,
|
|
150
|
+
applied: false,
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
source: "git_diff",
|
|
154
|
+
cap_type: "max_bytes_per_diff",
|
|
155
|
+
limit: MAX_GIT_DIFF_BYTES,
|
|
156
|
+
applied: false,
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
source: "git_diff",
|
|
160
|
+
cap_type: "timeout_ms",
|
|
161
|
+
limit: GIT_DIFF_TIMEOUT_MS,
|
|
162
|
+
applied: false,
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
source: "codex_jsonl",
|
|
166
|
+
cap_type: "max_file_bytes",
|
|
167
|
+
limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
168
|
+
applied: false,
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
source: "claude_jsonl",
|
|
172
|
+
cap_type: "max_file_bytes",
|
|
173
|
+
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
174
|
+
applied: false,
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
source: "claude_jsonl_sidecar",
|
|
178
|
+
cap_type: "max_file_bytes",
|
|
179
|
+
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
180
|
+
applied: false,
|
|
181
|
+
},
|
|
182
|
+
];
|
|
183
|
+
}
|
|
184
|
+
// ---------------------------------------------------------------------------
|
|
185
|
+
// The three ways a pass ends
|
|
186
|
+
// ---------------------------------------------------------------------------
|
|
187
|
+
/**
|
|
188
|
+
* Nothing collected: drop the staging directory instead of leaving an empty
|
|
189
|
+
* pack behind. This used to leave one empty `work-*` dir per sync.
|
|
190
|
+
*/
|
|
191
|
+
async function finishWithEmptyPack(collection, options, run) {
|
|
192
|
+
await discardStagingDir(run.stagingDir);
|
|
193
|
+
collection.packId = contentKeyedRawEvidencePackId({
|
|
194
|
+
workContextId: collection.context.workContextId,
|
|
195
|
+
contentHashes: [],
|
|
196
|
+
});
|
|
197
|
+
await persistStagingState(options.stateDir, collection.staging, collection.context.now.toISOString());
|
|
198
|
+
const facts = makeEmptyPackFacts(collection, run.window);
|
|
199
|
+
return {
|
|
200
|
+
facts,
|
|
201
|
+
scan: makeRawEvidenceScan({
|
|
202
|
+
context: collection.context,
|
|
203
|
+
startedAt: run.window.startedAt,
|
|
204
|
+
status: "partial",
|
|
205
|
+
facts,
|
|
206
|
+
}),
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
async function finishWithPromotedPack(collection, options, run) {
|
|
210
|
+
const context = collection.context;
|
|
211
|
+
const entries = collection.entries;
|
|
212
|
+
// The pack is named by what is in it, never by when it was made. Identical
|
|
213
|
+
// content on the next sync resolves to the identical directory, which is
|
|
214
|
+
// the whole fix for BLI-3066.
|
|
215
|
+
const packId = contentKeyedRawEvidencePackId({
|
|
216
|
+
workContextId: context.workContextId,
|
|
217
|
+
contentHashes: entries.map((entry) => entry.content_hash_sha256),
|
|
218
|
+
});
|
|
219
|
+
collection.packId = packId;
|
|
220
|
+
const promotion = await promoteStagedPack({
|
|
221
|
+
rawEvidenceRoot: collection.rawEvidenceRoot,
|
|
222
|
+
stagingDir: run.stagingDir,
|
|
223
|
+
packId,
|
|
224
|
+
workContextId: context.workContextId,
|
|
225
|
+
entries,
|
|
226
|
+
});
|
|
227
|
+
const evidenceDir = promotion.evidenceDir;
|
|
228
|
+
rebaseStagedEntriesOntoPack(collection, evidenceDir);
|
|
229
|
+
const manifestPath = path.join(evidenceDir, "manifest.json");
|
|
230
|
+
const manifestBytes = await stageManifest({
|
|
231
|
+
context,
|
|
232
|
+
packId,
|
|
233
|
+
manifestPath,
|
|
234
|
+
entries,
|
|
235
|
+
skipped: collection.skipped,
|
|
236
|
+
redacted: collection.redacted,
|
|
237
|
+
reused: collection.reused,
|
|
238
|
+
reusePack: promotion.state === "reused",
|
|
239
|
+
});
|
|
240
|
+
entries.push(evidenceEntry({
|
|
241
|
+
context,
|
|
242
|
+
kind: "manifest",
|
|
243
|
+
packId,
|
|
244
|
+
localPath: manifestPath,
|
|
245
|
+
relativePath: "manifest.json",
|
|
246
|
+
mediaType: "application/json",
|
|
247
|
+
redactedSummary: "Local raw evidence pack manifest.",
|
|
248
|
+
bytes: manifestBytes,
|
|
249
|
+
}));
|
|
250
|
+
await persistStagingState(options.stateDir, collection.staging, context.now.toISOString());
|
|
251
|
+
console.error("[raw-evidence] pack staged", JSON.stringify({
|
|
252
|
+
pack_id: packId,
|
|
253
|
+
stage_state: promotion.state,
|
|
254
|
+
reason: stageReasonLabel(promotion.state, promotion.priorPackCount),
|
|
255
|
+
prior_pack_count: promotion.priorPackCount,
|
|
256
|
+
refilled_file_count: promotion.refilledFileCount,
|
|
257
|
+
file_count: entries.length,
|
|
258
|
+
byte_size: totalByteSize(entries),
|
|
259
|
+
staged_new: collection.stagedNewCount,
|
|
260
|
+
staged_reused: collection.stagedReusedCount,
|
|
261
|
+
delivery_held: collection.deliveryHeldCount,
|
|
262
|
+
}));
|
|
263
|
+
const facts = {
|
|
264
|
+
...packFactsShell(collection, {
|
|
265
|
+
packId,
|
|
266
|
+
manifestPath,
|
|
267
|
+
evidenceDir,
|
|
268
|
+
window: run.window,
|
|
269
|
+
}),
|
|
270
|
+
stage_state: promotion.state,
|
|
271
|
+
file_count: entries.length,
|
|
272
|
+
byte_size: totalByteSize(entries),
|
|
273
|
+
content_kinds: [...new Set(entries.map((entry) => entry.kind))],
|
|
274
|
+
pointers: entries.map(pointerFromEntry),
|
|
275
|
+
upload_files: entries.map(uploadFileFromEntry),
|
|
276
|
+
};
|
|
277
|
+
return {
|
|
278
|
+
facts,
|
|
279
|
+
scan: makeRawEvidenceScan({
|
|
280
|
+
context,
|
|
281
|
+
startedAt: run.window.startedAt,
|
|
282
|
+
status: entries.length > 1 ? "ok" : "partial",
|
|
283
|
+
facts,
|
|
284
|
+
}),
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* A crashed pass. The staged-object index is deliberately NOT written: this
|
|
289
|
+
* attempt's bookkeeping describes a directory that is about to be deleted.
|
|
290
|
+
*/
|
|
291
|
+
async function finishWithFailedPack(collection, run) {
|
|
292
|
+
collection.failed.push({
|
|
293
|
+
kind: "raw_evidence",
|
|
294
|
+
reason: "collection_failed",
|
|
295
|
+
});
|
|
296
|
+
// Staging is per-attempt scratch: a crashed pass must not leave a partial
|
|
297
|
+
// directory behind to be counted, re-hashed or swept later.
|
|
298
|
+
await discardStagingDir(run.stagingDir);
|
|
299
|
+
collection.packId =
|
|
300
|
+
collection.packId ||
|
|
301
|
+
contentKeyedRawEvidencePackId({
|
|
302
|
+
workContextId: collection.context.workContextId,
|
|
303
|
+
contentHashes: collection.entries.map((entry) => entry.content_hash_sha256),
|
|
304
|
+
});
|
|
305
|
+
console.error("[raw-evidence] pack collection failed", JSON.stringify({
|
|
306
|
+
pack_id: collection.packId,
|
|
307
|
+
reason: "collection_failed",
|
|
308
|
+
detail: run.error instanceof Error ? run.error.name : typeof run.error,
|
|
309
|
+
collected_file_count: collection.entries.length,
|
|
310
|
+
staged_new: collection.stagedNewCount,
|
|
311
|
+
staged_reused: collection.stagedReusedCount,
|
|
312
|
+
}));
|
|
313
|
+
const facts = makeEmptyPackFacts(collection, run.window);
|
|
314
|
+
return {
|
|
315
|
+
facts,
|
|
316
|
+
scan: makeRawEvidenceScan({
|
|
317
|
+
context: collection.context,
|
|
318
|
+
startedAt: run.window.startedAt,
|
|
319
|
+
status: "failed",
|
|
258
320
|
facts,
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
321
|
+
}),
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* A pack with no files of its own: the empty pass and the crashed pass report
|
|
326
|
+
* the same shape, differing only in the scan status and the reason already
|
|
327
|
+
* logged. `evidence_completeness` still carries every gap this pass recorded.
|
|
328
|
+
*/
|
|
329
|
+
function makeEmptyPackFacts(collection, scanWindow) {
|
|
330
|
+
const evidenceDir = path.join(collection.rawEvidenceRoot, collection.packId);
|
|
331
|
+
return {
|
|
332
|
+
...packFactsShell(collection, {
|
|
333
|
+
packId: collection.packId,
|
|
334
|
+
manifestPath: path.join(evidenceDir, "manifest.json"),
|
|
335
|
+
evidenceDir,
|
|
336
|
+
window: scanWindow,
|
|
337
|
+
}),
|
|
338
|
+
stage_state: "empty",
|
|
339
|
+
file_count: 0,
|
|
340
|
+
byte_size: 0,
|
|
341
|
+
content_kinds: [],
|
|
342
|
+
pointers: [],
|
|
343
|
+
upload_files: [],
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
/** The counts every ending reports identically, however the pass ended. */
|
|
347
|
+
function packFactsShell(collection, place) {
|
|
348
|
+
return {
|
|
349
|
+
pack_id: place.packId,
|
|
350
|
+
manifest_path: place.manifestPath,
|
|
351
|
+
evidence_dir: place.evidenceDir,
|
|
352
|
+
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
353
|
+
skipped_count: countEvidenceEntries(collection.skipped),
|
|
354
|
+
sanitized_count: collection.redacted.length,
|
|
355
|
+
reused_count: collection.reused.length,
|
|
356
|
+
staged_reused_count: collection.stagedReusedCount,
|
|
357
|
+
staged_new_count: collection.stagedNewCount,
|
|
358
|
+
delivery_held_count: collection.deliveryHeldCount,
|
|
359
|
+
deferred_byte_budget_count: countDeferred(collection, "deferred_byte_budget"),
|
|
360
|
+
deferred_object_budget_count: countDeferred(collection, "deferred_object_budget"),
|
|
361
|
+
evidence_completeness: makeEvidenceCompleteness(collection, {
|
|
362
|
+
startedAt: place.window.startedAt,
|
|
363
|
+
finishedAt: place.window.finishedAt(),
|
|
364
|
+
sinceMinutes: place.window.sinceMinutes,
|
|
365
|
+
}),
|
|
366
|
+
reused: collection.reused,
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
function countDeferred(collection, reason) {
|
|
370
|
+
return countEvidenceEntries(collection.skipped, (entry) => entry.reason === reason);
|
|
371
|
+
}
|
|
372
|
+
function totalByteSize(entries) {
|
|
373
|
+
return entries.reduce((sum, entry) => sum + entry.byte_size, 0);
|
|
374
|
+
}
|
|
375
|
+
function uploadFileFromEntry(entry) {
|
|
376
|
+
return {
|
|
377
|
+
pointer: pointerFromEntry(entry),
|
|
378
|
+
local_path: entry.local_path,
|
|
379
|
+
kind: entry.kind,
|
|
380
|
+
codex_session_id: entry.codex_session_id ?? null,
|
|
381
|
+
...(entry.artifact_metadata
|
|
382
|
+
? { artifact_metadata: entry.artifact_metadata }
|
|
383
|
+
: {}),
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
function stageReasonLabel(state, priorPackCount) {
|
|
387
|
+
if (state === "reused")
|
|
388
|
+
return "staged_reused";
|
|
389
|
+
if (state === "restaged_incomplete")
|
|
390
|
+
return "restaged_incomplete";
|
|
391
|
+
return priorPackCount > 0 ? "restaged_content_changed" : "staged_new";
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Point every entry this pass staged at its home in the promoted pack, and
|
|
395
|
+
* remember the content hash so the next sync can adopt the copy instead of
|
|
396
|
+
* writing it again. Entries adopted from another pack keep their path.
|
|
397
|
+
*/
|
|
398
|
+
function rebaseStagedEntriesOntoPack(collection, evidenceDir) {
|
|
399
|
+
for (const entry of collection.entries) {
|
|
400
|
+
if (!entry.staged_in_pack)
|
|
401
|
+
continue;
|
|
402
|
+
entry.local_path = path.join(evidenceDir, "files", path.basename(entry.local_path));
|
|
403
|
+
recordStagedObject(collection.staging, entry.content_hash_sha256, {
|
|
404
|
+
pack_id: collection.packId,
|
|
405
|
+
relative_path: `files/${path.basename(entry.local_path)}`,
|
|
406
|
+
byte_size: entry.byte_size,
|
|
407
|
+
source_key: entry.source_key,
|
|
408
|
+
staged_at: collection.context.now.toISOString(),
|
|
409
|
+
});
|
|
266
410
|
}
|
|
267
411
|
}
|
|
412
|
+
async function discardStagingDir(stagingDir) {
|
|
413
|
+
await fs
|
|
414
|
+
.rm(stagingDir, { recursive: true, force: true })
|
|
415
|
+
.catch(() => undefined);
|
|
416
|
+
}
|
|
268
417
|
function makeRawEvidenceScan(options) {
|
|
269
418
|
return SourceScanResultSchema.parse({
|
|
270
419
|
adapter: makeSourceAdapterIdentity("collector_runtime", "raw-evidence-pack"),
|
|
@@ -293,6 +442,10 @@ function makeRawEvidenceScan(options) {
|
|
|
293
442
|
`bytes:${options.facts.byte_size}`,
|
|
294
443
|
`skipped:${options.facts.skipped_count}`,
|
|
295
444
|
`reused:${options.facts.reused_count}`,
|
|
445
|
+
`stage_state:${options.facts.stage_state}`,
|
|
446
|
+
`staged_new:${options.facts.staged_new_count}`,
|
|
447
|
+
`staged_reused:${options.facts.staged_reused_count}`,
|
|
448
|
+
`delivery_held:${options.facts.delivery_held_count}`,
|
|
296
449
|
`completeness:${options.facts.evidence_completeness.status}`,
|
|
297
450
|
`truncated:${options.facts.evidence_completeness.totals.truncated_count}`,
|
|
298
451
|
`deferred:${options.facts.evidence_completeness.totals.deferred_count}`,
|
|
@@ -301,12 +454,20 @@ function makeRawEvidenceScan(options) {
|
|
|
301
454
|
],
|
|
302
455
|
});
|
|
303
456
|
}
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
457
|
+
// ---------------------------------------------------------------------------
|
|
458
|
+
// What attribution already knew
|
|
459
|
+
// ---------------------------------------------------------------------------
|
|
460
|
+
/**
|
|
461
|
+
* Attribution ran before collection and already knows which sessions it could
|
|
462
|
+
* not name. Those are gaps in this pass even though no byte was read, so they
|
|
463
|
+
* are recorded first — a session Cockpit never looked at must not be invisible.
|
|
464
|
+
*/
|
|
465
|
+
function recordAttributionCompleteness(collection, options) {
|
|
466
|
+
if (options.codexAttributionScan) {
|
|
467
|
+
recordCodexAttributionCompleteness(collection, options.codexAttributionScan, new Set(options.codexSessionFiles?.map((file) => file.local_path) ?? []));
|
|
307
468
|
}
|
|
308
|
-
if (
|
|
309
|
-
recordClaudeAttributionCompleteness(collection,
|
|
469
|
+
if (options.claudeAttributionScan) {
|
|
470
|
+
recordClaudeAttributionCompleteness(collection, options.claudeAttributionScan, new Set(options.claudeSessionFiles?.map((file) => file.local_path) ?? []));
|
|
310
471
|
}
|
|
311
472
|
}
|
|
312
473
|
function recordCodexAttributionCompleteness(collection, scan, selectedPaths) {
|
|
@@ -384,39 +545,46 @@ function recordClaudeAttributionCompleteness(collection, scan, selectedPaths) {
|
|
|
384
545
|
}
|
|
385
546
|
function recordAttributionResultSkips(collection, source, results, selectedPaths) {
|
|
386
547
|
for (const result of results) {
|
|
387
|
-
|
|
388
|
-
// synthetic target is also not a skip when another workspace pack owns it.
|
|
389
|
-
if (result.state === "attributed" ||
|
|
390
|
-
selectedPaths.has(result.file_path) ||
|
|
391
|
-
(result.worktree !== null &&
|
|
392
|
-
isLiveRawEvidenceSyncAttribution(result.state, result.reason, true))) {
|
|
548
|
+
if (isAttributionAccountedFor(result, selectedPaths))
|
|
393
549
|
continue;
|
|
394
|
-
}
|
|
395
550
|
const reason = result.state === "skipped"
|
|
396
551
|
? result.reason
|
|
397
552
|
: `attribution_${result.state}:${result.reason}`;
|
|
398
553
|
recordSkipCount(collection, source, reason, 1);
|
|
399
554
|
}
|
|
400
555
|
}
|
|
556
|
+
/**
|
|
557
|
+
* Not every unattributed result is a gap. A selected session is being collected
|
|
558
|
+
* by this pass, and a live-sync-safe synthetic target is owned by another
|
|
559
|
+
* workspace's pack — neither is missing evidence.
|
|
560
|
+
*/
|
|
561
|
+
function isAttributionAccountedFor(result, selectedPaths) {
|
|
562
|
+
if (result.state === "attributed")
|
|
563
|
+
return true;
|
|
564
|
+
if (selectedPaths.has(result.file_path))
|
|
565
|
+
return true;
|
|
566
|
+
return (result.worktree !== null &&
|
|
567
|
+
isLiveRawEvidenceSyncAttribution(result.state, result.reason, true));
|
|
568
|
+
}
|
|
569
|
+
// ---------------------------------------------------------------------------
|
|
570
|
+
// Codex transcripts
|
|
571
|
+
// ---------------------------------------------------------------------------
|
|
401
572
|
async function collectCodexJsonlFiles(collection, options) {
|
|
573
|
+
const attributed = Boolean(options.codexSessionFiles);
|
|
402
574
|
const resolvedCandidates = options.codexSessionFiles
|
|
403
575
|
? options.codexSessionFiles.map((file) => ({
|
|
404
576
|
filePath: file.local_path,
|
|
405
577
|
codexSessionId: file.codex_session_id,
|
|
406
578
|
}))
|
|
407
|
-
: (await
|
|
408
|
-
(options.sessionsDir
|
|
409
|
-
? [options.sessionsDir]
|
|
410
|
-
: defaultCodexSessionDirs(os.homedir())), collection.context.now.getTime() - options.sinceMinutes * 60 * 1000))
|
|
411
|
-
.map((filePath) => ({ filePath, codexSessionId: null }));
|
|
579
|
+
: (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
|
|
412
580
|
collection.caps.push({
|
|
413
581
|
source: "codex_jsonl",
|
|
414
582
|
cap_type: "session_limit",
|
|
415
583
|
limit: options.limit,
|
|
416
584
|
observed: resolvedCandidates.length,
|
|
417
|
-
applied: !
|
|
585
|
+
applied: !attributed && resolvedCandidates.length > options.limit,
|
|
418
586
|
});
|
|
419
|
-
const candidates =
|
|
587
|
+
const candidates = attributed
|
|
420
588
|
? resolvedCandidates
|
|
421
589
|
: resolvedCandidates.slice(0, options.limit);
|
|
422
590
|
for (const candidate of candidates) {
|
|
@@ -433,95 +601,118 @@ async function collectCodexJsonlFiles(collection, options) {
|
|
|
433
601
|
redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
434
602
|
contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
|
|
435
603
|
});
|
|
436
|
-
if (transcriptAccepted)
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
}
|
|
444
|
-
}
|
|
604
|
+
if (!transcriptAccepted)
|
|
605
|
+
continue;
|
|
606
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
607
|
+
filePath: candidate.filePath,
|
|
608
|
+
source: "codex",
|
|
609
|
+
sessionId: codexSessionId,
|
|
610
|
+
kind: "codex_image_attachment",
|
|
611
|
+
contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
|
|
612
|
+
});
|
|
445
613
|
}
|
|
446
614
|
}
|
|
615
|
+
/** The unattributed fallback: every recent Codex transcript on this machine. */
|
|
616
|
+
function walkRecentCodexJsonlFiles(collection, options) {
|
|
617
|
+
const dirs = options.sessionsDirs ??
|
|
618
|
+
(options.sessionsDir
|
|
619
|
+
? [options.sessionsDir]
|
|
620
|
+
: defaultCodexSessionDirs(os.homedir()));
|
|
621
|
+
return walkJsonlFiles(dirs, collection.context.now.getTime() - options.sinceMinutes * 60 * 1000);
|
|
622
|
+
}
|
|
623
|
+
// ---------------------------------------------------------------------------
|
|
624
|
+
// Claude transcripts
|
|
625
|
+
// ---------------------------------------------------------------------------
|
|
447
626
|
async function collectClaudeJsonlFiles(collection, sessions) {
|
|
448
627
|
recordScanned(collection, "claude_jsonl", sessions.reduce((count, session) => count + 1 + session.sidecar_files.length, 0));
|
|
449
628
|
for (const session of sessions) {
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
const mainAccepted = await collectOneEvidenceFile(collection, {
|
|
468
|
-
filePath: session.local_path,
|
|
469
|
-
kind: "claude_jsonl",
|
|
470
|
-
sessionId,
|
|
471
|
-
mediaType: "application/jsonl",
|
|
472
|
-
// Re-check size at collection: a main that grew past the cap between
|
|
473
|
-
// attribution and collection is an honest file_too_large skip, not an
|
|
474
|
-
// upload_failed at the chunk client.
|
|
475
|
-
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
476
|
-
redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
477
|
-
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/${hash16}.jsonl`,
|
|
478
|
-
});
|
|
479
|
-
if (mainAccepted) {
|
|
480
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
481
|
-
filePath: session.local_path,
|
|
482
|
-
source: "claude_code",
|
|
483
|
-
sessionId,
|
|
484
|
-
kind: "claude_image_attachment",
|
|
485
|
-
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
|
|
486
|
-
});
|
|
487
|
-
}
|
|
488
|
-
}
|
|
489
|
-
if (session.skip_main && !session.main_file_oversized) {
|
|
490
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
491
|
-
filePath: session.local_path,
|
|
492
|
-
source: "claude_code",
|
|
493
|
-
sessionId,
|
|
494
|
-
kind: "claude_image_attachment",
|
|
495
|
-
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
|
|
496
|
-
});
|
|
497
|
-
}
|
|
498
|
-
for (const sidecar of session.sidecar_files) {
|
|
499
|
-
const stem = path.basename(sidecar.local_path).replace(/\.jsonl$/i, "");
|
|
500
|
-
const safeSidecarId = isSecretLikePath(stem)
|
|
501
|
-
? "redacted-file-name"
|
|
502
|
-
: safeKeySegment(stem);
|
|
503
|
-
const sidecarAccepted = await collectOneEvidenceFile(collection, {
|
|
504
|
-
filePath: sidecar.local_path,
|
|
505
|
-
kind: "claude_jsonl_sidecar",
|
|
506
|
-
sessionId,
|
|
507
|
-
mediaType: "application/jsonl",
|
|
508
|
-
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
509
|
-
redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
510
|
-
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
|
|
511
|
-
});
|
|
512
|
-
if (sidecarAccepted) {
|
|
513
|
-
await collectAgentImagesFromTranscript(collection, {
|
|
514
|
-
filePath: sidecar.local_path,
|
|
515
|
-
source: "claude_code",
|
|
516
|
-
sessionId,
|
|
517
|
-
sidecarId: safeSidecarId,
|
|
518
|
-
kind: "claude_image_attachment",
|
|
519
|
-
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
|
|
520
|
-
});
|
|
521
|
-
}
|
|
522
|
-
}
|
|
629
|
+
await collectOneClaudeSession(collection, session);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
async function collectOneClaudeSession(collection, session) {
|
|
633
|
+
const sessionId = session.claude_session_id;
|
|
634
|
+
const mainOutcome = await collectClaudeMainFile(collection, session);
|
|
635
|
+
if (mainOutcome === "collected" || mainOutcome === "damped_reuse") {
|
|
636
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
637
|
+
filePath: session.local_path,
|
|
638
|
+
source: "claude_code",
|
|
639
|
+
sessionId,
|
|
640
|
+
kind: "claude_image_attachment",
|
|
641
|
+
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
for (const sidecar of session.sidecar_files) {
|
|
645
|
+
await collectOneClaudeSidecar(collection, session, sidecar.local_path);
|
|
523
646
|
}
|
|
524
647
|
}
|
|
648
|
+
/**
|
|
649
|
+
* What happens to a Claude main file, as a decision table:
|
|
650
|
+
*
|
|
651
|
+
* | condition | main bytes | outcome |
|
|
652
|
+
* | --------------------- | ------------------------- | ------------------- |
|
|
653
|
+
* | `main_file_oversized` | skipped `file_too_large` | `skipped_too_large` |
|
|
654
|
+
* | `skip_main` (D9) | not re-collected | `damped_reuse` |
|
|
655
|
+
* | otherwise | collected, or named skip | `collected` / `not_collected` |
|
|
656
|
+
*
|
|
657
|
+
* Its images are collected for every outcome except `skipped_too_large` and
|
|
658
|
+
* `not_collected` — the caller decides that, this function only reports.
|
|
659
|
+
*
|
|
660
|
+
* D7: an oversized main was attributed via a streamed read but its bytes are
|
|
661
|
+
* never uploaded (server commit assembles in memory). Its sidecars still
|
|
662
|
+
* collect. D9 damped: the prior durable copy is still good enough, and the
|
|
663
|
+
* session reports `reused_existing` from cursor state, so no skip is recorded.
|
|
664
|
+
*/
|
|
665
|
+
async function collectClaudeMainFile(collection, session) {
|
|
666
|
+
if (session.main_file_oversized) {
|
|
667
|
+
collection.skipped.push({
|
|
668
|
+
kind: "claude_jsonl",
|
|
669
|
+
label: path.basename(session.local_path),
|
|
670
|
+
reason: "file_too_large",
|
|
671
|
+
});
|
|
672
|
+
return "skipped_too_large";
|
|
673
|
+
}
|
|
674
|
+
if (session.skip_main)
|
|
675
|
+
return "damped_reuse";
|
|
676
|
+
const accepted = await collectOneEvidenceFile(collection, {
|
|
677
|
+
filePath: session.local_path,
|
|
678
|
+
kind: "claude_jsonl",
|
|
679
|
+
sessionId: session.claude_session_id,
|
|
680
|
+
mediaType: "application/jsonl",
|
|
681
|
+
// Re-check size at collection: a main that grew past the cap between
|
|
682
|
+
// attribution and collection is an honest file_too_large skip, not an
|
|
683
|
+
// upload_failed at the chunk client.
|
|
684
|
+
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
685
|
+
redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
686
|
+
contentAddress: (hash16) => `claude/${safeKeySegment(session.claude_session_id)}/${hash16}.jsonl`,
|
|
687
|
+
});
|
|
688
|
+
return accepted ? "collected" : "not_collected";
|
|
689
|
+
}
|
|
690
|
+
async function collectOneClaudeSidecar(collection, session, sidecarPath) {
|
|
691
|
+
const sessionId = session.claude_session_id;
|
|
692
|
+
const stem = path.basename(sidecarPath).replace(/\.jsonl$/i, "");
|
|
693
|
+
const safeSidecarId = isSecretLikePath(stem)
|
|
694
|
+
? "redacted-file-name"
|
|
695
|
+
: safeKeySegment(stem);
|
|
696
|
+
const sidecarAccepted = await collectOneEvidenceFile(collection, {
|
|
697
|
+
filePath: sidecarPath,
|
|
698
|
+
kind: "claude_jsonl_sidecar",
|
|
699
|
+
sessionId,
|
|
700
|
+
mediaType: "application/jsonl",
|
|
701
|
+
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
702
|
+
redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
703
|
+
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
|
|
704
|
+
});
|
|
705
|
+
if (!sidecarAccepted)
|
|
706
|
+
return;
|
|
707
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
708
|
+
filePath: sidecarPath,
|
|
709
|
+
source: "claude_code",
|
|
710
|
+
sessionId,
|
|
711
|
+
sidecarId: safeSidecarId,
|
|
712
|
+
kind: "claude_image_attachment",
|
|
713
|
+
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
|
|
714
|
+
});
|
|
715
|
+
}
|
|
525
716
|
async function collectAgentImagesFromTranscript(collection, options) {
|
|
526
717
|
const result = await collectAgentImageEvidenceFromJsonlFile({
|
|
527
718
|
filePath: options.filePath,
|
|
@@ -576,15 +767,26 @@ async function collectOneAgentImageFile(collection, options) {
|
|
|
576
767
|
}
|
|
577
768
|
collection.index.value += 1;
|
|
578
769
|
const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-agent-image-${contentHash.slice(0, 16)}.${options.image.extension}`);
|
|
579
|
-
const
|
|
580
|
-
|
|
581
|
-
|
|
770
|
+
const imageSourceKey = evidenceSourceKey({
|
|
771
|
+
kind: options.kind,
|
|
772
|
+
sessionId: options.sessionId,
|
|
773
|
+
label: options.image.label,
|
|
774
|
+
});
|
|
775
|
+
const staged = await stageEvidenceBytes(collection, {
|
|
776
|
+
contentHash,
|
|
777
|
+
bytes: raw,
|
|
778
|
+
fileName: path.basename(relativePath),
|
|
779
|
+
kind: options.kind,
|
|
780
|
+
sourceKey: imageSourceKey,
|
|
781
|
+
});
|
|
582
782
|
collection.entries.push(evidenceEntry({
|
|
583
783
|
context: collection.context,
|
|
584
784
|
kind: options.kind,
|
|
585
785
|
packId: collection.packId,
|
|
586
|
-
localPath:
|
|
786
|
+
localPath: staged.local_path,
|
|
587
787
|
relativePath,
|
|
788
|
+
stagedInPack: staged.staged_in_pack,
|
|
789
|
+
sourceKey: imageSourceKey,
|
|
588
790
|
mediaType: metadata.media_type,
|
|
589
791
|
redactedSummary: "Raw image explicitly attached to an agent session, preserved in private durable storage.",
|
|
590
792
|
bytes: raw,
|
|
@@ -593,63 +795,58 @@ async function collectOneAgentImageFile(collection, options) {
|
|
|
593
795
|
artifactMetadata: metadata,
|
|
594
796
|
}));
|
|
595
797
|
}
|
|
798
|
+
// ---------------------------------------------------------------------------
|
|
799
|
+
// One transcript, end to end
|
|
800
|
+
// ---------------------------------------------------------------------------
|
|
596
801
|
/**
|
|
597
802
|
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
598
|
-
* attributed transcript into the pack.
|
|
599
|
-
*
|
|
600
|
-
*
|
|
803
|
+
* attributed transcript into the pack.
|
|
804
|
+
*
|
|
805
|
+
* Returns whether this file's *content* is accounted for, which is not the same
|
|
806
|
+
* as "was staged": a transcript whose bytes are already durable remotely
|
|
807
|
+
* returns true without staging anything, and its images are still worth
|
|
808
|
+
* collecting. Every false return has pushed a named skip first.
|
|
601
809
|
*/
|
|
602
810
|
async function collectOneEvidenceFile(collection, options) {
|
|
603
811
|
const fileName = path.basename(options.filePath);
|
|
604
812
|
const secretLikeFileName = isSecretLikePath(fileName);
|
|
605
|
-
const evidenceLabel = secretLikeFileName
|
|
606
|
-
? "[REDACTED_FILE_NAME]"
|
|
607
|
-
: fileName;
|
|
813
|
+
const evidenceLabel = secretLikeFileName ? "[REDACTED_FILE_NAME]" : fileName;
|
|
608
814
|
const packedFileName = secretLikeFileName ? "redacted-file.jsonl" : fileName;
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
collection.skipped.push({
|
|
616
|
-
kind: options.kind,
|
|
617
|
-
label: evidenceLabel,
|
|
618
|
-
reason: "file_read_failed",
|
|
619
|
-
});
|
|
620
|
-
return false;
|
|
621
|
-
}
|
|
622
|
-
if (stat.size > options.maxFileBytes) {
|
|
623
|
-
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
624
|
-
collection.skipped.push({
|
|
625
|
-
kind: options.kind,
|
|
626
|
-
label: evidenceLabel,
|
|
627
|
-
reason: "file_too_large",
|
|
628
|
-
});
|
|
629
|
-
return false;
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
let raw;
|
|
633
|
-
try {
|
|
634
|
-
raw = await fs.readFile(options.filePath);
|
|
635
|
-
}
|
|
636
|
-
catch {
|
|
815
|
+
const sourceKey = evidenceSourceKey({
|
|
816
|
+
kind: options.kind,
|
|
817
|
+
sessionId: options.sessionId,
|
|
818
|
+
sourcePath: options.filePath,
|
|
819
|
+
});
|
|
820
|
+
const skip = (reason) => {
|
|
637
821
|
collection.skipped.push({
|
|
638
822
|
kind: options.kind,
|
|
639
823
|
label: evidenceLabel,
|
|
640
|
-
reason
|
|
824
|
+
reason,
|
|
641
825
|
});
|
|
642
826
|
return false;
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
827
|
+
};
|
|
828
|
+
// Delivery backoff is checked BEFORE the file is read. An object whose commit
|
|
829
|
+
// has failed repeatedly costs nothing at all this cycle — no read, no hash,
|
|
830
|
+
// no copy, no request — and the hold is a named, retryable gap so a held
|
|
831
|
+
// session cannot make the sync look clean (BLI-3066).
|
|
832
|
+
if (collection.heldSources.has(sourceKey)) {
|
|
833
|
+
collection.deliveryHeldCount += 1;
|
|
834
|
+
skip(DELIVERY_BACKOFF_HOLDING_REASON);
|
|
835
|
+
console.error("[raw-evidence] delivery backoff holding source", JSON.stringify({
|
|
836
|
+
reason: DELIVERY_BACKOFF_HOLDING_REASON,
|
|
647
837
|
kind: options.kind,
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
});
|
|
838
|
+
source_key: sourceKey,
|
|
839
|
+
}));
|
|
651
840
|
return false;
|
|
652
841
|
}
|
|
842
|
+
const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
|
|
843
|
+
if (read.status === "read_failed")
|
|
844
|
+
return skip("file_read_failed");
|
|
845
|
+
if (read.status === "too_large") {
|
|
846
|
+
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
847
|
+
return skip("file_too_large");
|
|
848
|
+
}
|
|
849
|
+
const raw = read.bytes;
|
|
653
850
|
const sanitized = sanitizeTextEvidenceForUpload({
|
|
654
851
|
text: raw.toString("utf8"),
|
|
655
852
|
originalBytes: raw,
|
|
@@ -685,23 +882,22 @@ async function collectOneEvidenceFile(collection, options) {
|
|
|
685
882
|
const deferReason = admitToBudget(collection.budget, evidenceBytes.byteLength);
|
|
686
883
|
if (deferReason) {
|
|
687
884
|
markBudgetCapApplied(collection, deferReason);
|
|
688
|
-
|
|
689
|
-
kind: options.kind,
|
|
690
|
-
label: evidenceLabel,
|
|
691
|
-
reason: deferReason,
|
|
692
|
-
});
|
|
693
|
-
return false;
|
|
885
|
+
return skip(deferReason);
|
|
694
886
|
}
|
|
695
887
|
collection.index.value += 1;
|
|
696
888
|
const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(options.filePath)}-${packedFileName}`);
|
|
697
|
-
const
|
|
698
|
-
|
|
699
|
-
|
|
889
|
+
const staged = await stageEvidenceBytes(collection, {
|
|
890
|
+
contentHash,
|
|
891
|
+
bytes: evidenceBytes,
|
|
892
|
+
fileName: path.basename(relativePath),
|
|
893
|
+
kind: options.kind,
|
|
894
|
+
sourceKey,
|
|
895
|
+
});
|
|
700
896
|
collection.entries.push(evidenceEntry({
|
|
701
897
|
context: collection.context,
|
|
702
898
|
kind: options.kind,
|
|
703
899
|
packId: collection.packId,
|
|
704
|
-
localPath:
|
|
900
|
+
localPath: staged.local_path,
|
|
705
901
|
relativePath,
|
|
706
902
|
mediaType: options.mediaType,
|
|
707
903
|
redactedSummary: redaction
|
|
@@ -711,9 +907,70 @@ async function collectOneEvidenceFile(collection, options) {
|
|
|
711
907
|
bytes: evidenceBytes,
|
|
712
908
|
codexSessionId: options.sessionId,
|
|
713
909
|
contentAddress: options.contentAddress(contentHash.slice(0, 16)),
|
|
910
|
+
stagedInPack: staged.staged_in_pack,
|
|
911
|
+
sourceKey,
|
|
714
912
|
}));
|
|
715
913
|
return true;
|
|
716
914
|
}
|
|
915
|
+
/**
|
|
916
|
+
* Size is checked twice on purpose: once by `stat` so an oversized transcript
|
|
917
|
+
* is never buffered at all, and once on the bytes actually read, because a live
|
|
918
|
+
* session can grow between the two.
|
|
919
|
+
*/
|
|
920
|
+
async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
|
|
921
|
+
if (maxFileBytes) {
|
|
922
|
+
let stat;
|
|
923
|
+
try {
|
|
924
|
+
stat = await fs.stat(filePath);
|
|
925
|
+
}
|
|
926
|
+
catch {
|
|
927
|
+
return { status: "read_failed" };
|
|
928
|
+
}
|
|
929
|
+
if (stat.size > maxFileBytes)
|
|
930
|
+
return { status: "too_large" };
|
|
931
|
+
}
|
|
932
|
+
let bytes;
|
|
933
|
+
try {
|
|
934
|
+
bytes = await fs.readFile(filePath);
|
|
935
|
+
}
|
|
936
|
+
catch {
|
|
937
|
+
return { status: "read_failed" };
|
|
938
|
+
}
|
|
939
|
+
if (maxFileBytes && bytes.byteLength > maxFileBytes) {
|
|
940
|
+
return { status: "too_large" };
|
|
941
|
+
}
|
|
942
|
+
return { status: "ok", bytes };
|
|
943
|
+
}
|
|
944
|
+
// ---------------------------------------------------------------------------
|
|
945
|
+
// Staging and budgets
|
|
946
|
+
// ---------------------------------------------------------------------------
|
|
947
|
+
/**
|
|
948
|
+
* Put these bytes on disk once.
|
|
949
|
+
*
|
|
950
|
+
* If an earlier sync already staged this exact content and the copy is still
|
|
951
|
+
* there, that copy is used — the uploader only needs a readable path, and it
|
|
952
|
+
* does not care which pack directory holds it. This is the branch that stops
|
|
953
|
+
* one 334 MB rollout from becoming 559 copies while its commit keeps failing.
|
|
954
|
+
*/
|
|
955
|
+
async function stageEvidenceBytes(collection, options) {
|
|
956
|
+
const existing = await resolveStagedObject(collection.rawEvidenceRoot, collection.staging, options.contentHash);
|
|
957
|
+
if (existing) {
|
|
958
|
+
collection.stagedReusedCount += 1;
|
|
959
|
+
console.error("[raw-evidence] staged copy reused", JSON.stringify({
|
|
960
|
+
reason: "staged_reused",
|
|
961
|
+
kind: options.kind,
|
|
962
|
+
content_hash_prefix: options.contentHash.slice(0, 16),
|
|
963
|
+
byte_size: existing.entry.byte_size,
|
|
964
|
+
pack_id: existing.entry.pack_id,
|
|
965
|
+
}));
|
|
966
|
+
return { local_path: existing.local_path, staged_in_pack: false };
|
|
967
|
+
}
|
|
968
|
+
const destination = path.join(collection.filesDir, options.fileName);
|
|
969
|
+
await fs.writeFile(destination, options.bytes, { mode: 0o600 });
|
|
970
|
+
await chmodPrivate(destination, 0o600);
|
|
971
|
+
collection.stagedNewCount += 1;
|
|
972
|
+
return { local_path: destination, staged_in_pack: true };
|
|
973
|
+
}
|
|
717
974
|
/**
|
|
718
975
|
* Decrements the per-sync budget when a file fits, or returns a deferred-skip
|
|
719
976
|
* reason when it does not. The object budget bounds request count; the byte
|
|
@@ -728,145 +985,15 @@ function admitToBudget(budget, byteLength) {
|
|
|
728
985
|
budget.remainingBytes -= byteLength;
|
|
729
986
|
return null;
|
|
730
987
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
});
|
|
739
|
-
if (options.secretLikeFileName) {
|
|
740
|
-
const sanitizedText = redactionResult.redacted
|
|
741
|
-
? redactionResult.text
|
|
742
|
-
: options.text;
|
|
743
|
-
const safeText = containsSecretLikeContent(sanitizedText)
|
|
744
|
-
? "[REDACTED_LINE:secret_redaction_failed]\n"
|
|
745
|
-
: sanitizedText;
|
|
746
|
-
const sanitizedBytes = Buffer.from(safeText, "utf8");
|
|
747
|
-
return {
|
|
748
|
-
status: "redacted",
|
|
749
|
-
bytes: sanitizedBytes,
|
|
750
|
-
redaction: withRedactionContentMetadata(redactionResult.metadata ??
|
|
751
|
-
fallbackRedactionMetadata({
|
|
752
|
-
ruleId: "secret_like_file_name",
|
|
753
|
-
originalText: options.text,
|
|
754
|
-
redactedFields: options.redactedFields,
|
|
755
|
-
fullContentRedacted: false,
|
|
756
|
-
}), { originalBytes, sanitizedBytes }),
|
|
757
|
-
completenessLabel: "secret_like_name_masked",
|
|
758
|
-
};
|
|
759
|
-
}
|
|
760
|
-
if (secretLikeContent) {
|
|
761
|
-
const redaction = redactionResult.metadata ??
|
|
762
|
-
fallbackRedactionMetadata({
|
|
763
|
-
ruleId: "secret_like_content_guard",
|
|
764
|
-
originalText: options.text,
|
|
765
|
-
redactedFields: options.redactedFields,
|
|
766
|
-
});
|
|
767
|
-
let maskedText = maskSecretBearingLines(options.text, redaction);
|
|
768
|
-
if (containsSecretLikeContent(maskedText)) {
|
|
769
|
-
maskedText = "[REDACTED_LINE:secret_redaction_failed]\n";
|
|
770
|
-
}
|
|
771
|
-
const sanitizedBytes = Buffer.from(maskedText, "utf8");
|
|
772
|
-
return {
|
|
773
|
-
status: "redacted",
|
|
774
|
-
bytes: sanitizedBytes,
|
|
775
|
-
redaction: withRedactionContentMetadata(redaction, {
|
|
776
|
-
originalBytes,
|
|
777
|
-
sanitizedBytes,
|
|
778
|
-
}),
|
|
779
|
-
completenessLabel: "secret_content_masked",
|
|
780
|
-
};
|
|
781
|
-
}
|
|
782
|
-
if (redactionResult.redacted) {
|
|
783
|
-
const sanitizedBytes = Buffer.from(redactionResult.text, "utf8");
|
|
784
|
-
return {
|
|
785
|
-
status: "redacted",
|
|
786
|
-
bytes: sanitizedBytes,
|
|
787
|
-
redaction: withRedactionContentMetadata(redactionResult.metadata ??
|
|
788
|
-
fallbackRedactionMetadata({
|
|
789
|
-
ruleId: "secret_redaction_failed",
|
|
790
|
-
originalText: options.text,
|
|
791
|
-
redactedFields: options.redactedFields,
|
|
792
|
-
}), { originalBytes, sanitizedBytes }),
|
|
793
|
-
completenessLabel: "secret_content_masked",
|
|
794
|
-
};
|
|
795
|
-
}
|
|
796
|
-
return { status: "clean", bytes: originalBytes };
|
|
797
|
-
}
|
|
798
|
-
catch {
|
|
799
|
-
const sanitizedBytes = Buffer.from("[REDACTED_LINE:redaction_crashed_stubbed]\n", "utf8");
|
|
800
|
-
return {
|
|
801
|
-
status: "redacted",
|
|
802
|
-
bytes: sanitizedBytes,
|
|
803
|
-
redaction: withRedactionContentMetadata(fallbackRedactionMetadata({
|
|
804
|
-
ruleId: "redaction_crashed_stubbed",
|
|
805
|
-
originalText: options.text,
|
|
806
|
-
redactedFields: options.redactedFields,
|
|
807
|
-
}), { originalBytes, sanitizedBytes }),
|
|
808
|
-
completenessLabel: "redaction_crashed_stubbed",
|
|
809
|
-
};
|
|
810
|
-
}
|
|
811
|
-
}
|
|
812
|
-
function maskSecretBearingLines(text, redaction) {
|
|
813
|
-
if (redaction.redacted_ranges.length === 0) {
|
|
814
|
-
return "[REDACTED_LINE:secret_like_content_guard]\n";
|
|
815
|
-
}
|
|
816
|
-
const segments = text.match(/[^\n]*(?:\n|$)/gu)?.filter(Boolean) ?? [];
|
|
817
|
-
let offset = 0;
|
|
818
|
-
return segments
|
|
819
|
-
.map((segment) => {
|
|
820
|
-
const start = offset;
|
|
821
|
-
const end = offset + segment.length;
|
|
822
|
-
offset = end;
|
|
823
|
-
const matched = redaction.redacted_ranges.find((range) => range.start < end && start < range.end);
|
|
824
|
-
if (!matched)
|
|
825
|
-
return segment;
|
|
826
|
-
const lineEnding = segment.endsWith("\r\n")
|
|
827
|
-
? "\r\n"
|
|
828
|
-
: segment.endsWith("\n")
|
|
829
|
-
? "\n"
|
|
830
|
-
: "";
|
|
831
|
-
return `[REDACTED_LINE:${matched.rule_id}]${lineEnding}`;
|
|
832
|
-
})
|
|
833
|
-
.join("");
|
|
834
|
-
}
|
|
835
|
-
function fallbackRedactionMetadata(options) {
|
|
836
|
-
const fullContentRedacted = options.fullContentRedacted !== false;
|
|
837
|
-
return {
|
|
838
|
-
schema_version: "raw-evidence-redaction.v1",
|
|
839
|
-
status: "sanitized",
|
|
840
|
-
mode: "deterministic_text_replacement",
|
|
841
|
-
applied_by: ["local_collector"],
|
|
842
|
-
rule_counts: [
|
|
843
|
-
{
|
|
844
|
-
rule_id: options.ruleId,
|
|
845
|
-
match_count: 1,
|
|
846
|
-
redacted_char_count: fullContentRedacted
|
|
847
|
-
? options.originalText.length
|
|
848
|
-
: 0,
|
|
849
|
-
},
|
|
850
|
-
],
|
|
851
|
-
secret_like_match_count: 1,
|
|
852
|
-
redacted_fields: options.redactedFields,
|
|
853
|
-
redacted_ranges: fullContentRedacted && options.originalText.length > 0
|
|
854
|
-
? [
|
|
855
|
-
{
|
|
856
|
-
start: 0,
|
|
857
|
-
end: options.originalText.length,
|
|
858
|
-
rule_id: options.ruleId,
|
|
859
|
-
},
|
|
860
|
-
]
|
|
861
|
-
: [],
|
|
862
|
-
};
|
|
863
|
-
}
|
|
988
|
+
// ---------------------------------------------------------------------------
|
|
989
|
+
// Git diffs
|
|
990
|
+
// ---------------------------------------------------------------------------
|
|
991
|
+
const GIT_DIFF_TARGETS = [
|
|
992
|
+
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
993
|
+
{ label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
|
|
994
|
+
];
|
|
864
995
|
async function collectGitDiffFiles(collection, repoRoot) {
|
|
865
|
-
const
|
|
866
|
-
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
867
|
-
{ label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
|
|
868
|
-
];
|
|
869
|
-
for (const target of diffTargets) {
|
|
996
|
+
for (const target of GIT_DIFF_TARGETS) {
|
|
870
997
|
recordScanned(collection, "git_diff");
|
|
871
998
|
let diff;
|
|
872
999
|
try {
|
|
@@ -892,334 +1019,107 @@ async function collectGitDiffFiles(collection, repoRoot) {
|
|
|
892
1019
|
included_bytes: Buffer.byteLength(diff.stdout, "utf8"),
|
|
893
1020
|
});
|
|
894
1021
|
}
|
|
1022
|
+
// An empty diff is not a gap: there was simply nothing to record. The
|
|
1023
|
+
// truncation marker above still stands even when zero bytes survived.
|
|
895
1024
|
if (!diff.stdout.trim())
|
|
896
1025
|
continue;
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
1026
|
+
await stageOneGitDiff(collection, {
|
|
1027
|
+
label: target.label,
|
|
1028
|
+
diffText: diff.stdout,
|
|
1029
|
+
truncated: diff.truncated,
|
|
900
1030
|
});
|
|
901
|
-
if (sanitized.status === "redacted") {
|
|
902
|
-
collection.redacted.push({
|
|
903
|
-
kind: "git_diff",
|
|
904
|
-
label: target.label,
|
|
905
|
-
redaction: sanitized.redaction,
|
|
906
|
-
completenessLabel: sanitized.completenessLabel,
|
|
907
|
-
});
|
|
908
|
-
console.error("[raw-evidence] git diff sanitized", JSON.stringify({
|
|
909
|
-
mode: sanitized.completenessLabel,
|
|
910
|
-
original_bytes: Buffer.byteLength(diff.stdout, "utf8"),
|
|
911
|
-
uploaded_bytes: sanitized.bytes.byteLength,
|
|
912
|
-
}));
|
|
913
|
-
}
|
|
914
|
-
const raw = sanitized.bytes;
|
|
915
|
-
const redaction = sanitized.redaction;
|
|
916
|
-
const contentHash = sha256(raw);
|
|
917
|
-
if (collection.skipContentHashes.has(contentHash)) {
|
|
918
|
-
collection.reused.push({
|
|
919
|
-
kind: "git_diff",
|
|
920
|
-
label: target.label,
|
|
921
|
-
content_hash_sha256: contentHash,
|
|
922
|
-
codex_session_id: null,
|
|
923
|
-
});
|
|
924
|
-
continue;
|
|
925
|
-
}
|
|
926
|
-
const deferReason = admitToBudget(collection.budget, raw.byteLength);
|
|
927
|
-
if (deferReason) {
|
|
928
|
-
markBudgetCapApplied(collection, deferReason);
|
|
929
|
-
collection.skipped.push({
|
|
930
|
-
kind: "git_diff",
|
|
931
|
-
label: target.label,
|
|
932
|
-
reason: deferReason,
|
|
933
|
-
});
|
|
934
|
-
continue;
|
|
935
|
-
}
|
|
936
|
-
const relativePath = path.join("files", `git-${target.label}.diff`);
|
|
937
|
-
const destination = path.join(collection.filesDir, path.basename(relativePath));
|
|
938
|
-
await fs.writeFile(destination, raw, { mode: 0o600 });
|
|
939
|
-
await chmodPrivate(destination, 0o600);
|
|
940
|
-
collection.entries.push(evidenceEntry({
|
|
941
|
-
context: collection.context,
|
|
942
|
-
kind: "git_diff",
|
|
943
|
-
packId: collection.packId,
|
|
944
|
-
localPath: destination,
|
|
945
|
-
relativePath,
|
|
946
|
-
mediaType: "text/x-diff",
|
|
947
|
-
redactedSummary: redaction
|
|
948
|
-
? `Raw git ${target.label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`
|
|
949
|
-
: diff.truncated
|
|
950
|
-
? `Raw git ${target.label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`
|
|
951
|
-
: `Raw git ${target.label} diff preserved locally with env/secret paths excluded.`,
|
|
952
|
-
redaction,
|
|
953
|
-
bytes: raw,
|
|
954
|
-
contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
|
|
955
|
-
}));
|
|
956
1031
|
}
|
|
957
1032
|
}
|
|
958
|
-
async function
|
|
959
|
-
const
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
":(exclude)**/*.key",
|
|
970
|
-
];
|
|
971
|
-
return new Promise((resolve, reject) => {
|
|
972
|
-
const child = spawn("git", [...args, ...pathspec], {
|
|
973
|
-
cwd: repoRoot,
|
|
974
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
975
|
-
});
|
|
976
|
-
const stdoutChunks = [];
|
|
977
|
-
const stderrChunks = [];
|
|
978
|
-
let observedBytes = 0;
|
|
979
|
-
let includedBytes = 0;
|
|
980
|
-
let truncated = false;
|
|
981
|
-
let timedOut = false;
|
|
982
|
-
const timeout = setTimeout(() => {
|
|
983
|
-
timedOut = true;
|
|
984
|
-
truncated = true;
|
|
985
|
-
child.kill("SIGTERM");
|
|
986
|
-
}, GIT_DIFF_TIMEOUT_MS);
|
|
987
|
-
child.stdout.on("data", (chunk) => {
|
|
988
|
-
observedBytes += chunk.byteLength;
|
|
989
|
-
if (includedBytes < MAX_GIT_DIFF_BYTES) {
|
|
990
|
-
const remaining = MAX_GIT_DIFF_BYTES - includedBytes;
|
|
991
|
-
const next = chunk.subarray(0, remaining);
|
|
992
|
-
stdoutChunks.push(next);
|
|
993
|
-
includedBytes += next.byteLength;
|
|
994
|
-
}
|
|
995
|
-
if (observedBytes > MAX_GIT_DIFF_BYTES) {
|
|
996
|
-
truncated = true;
|
|
997
|
-
child.kill("SIGTERM");
|
|
998
|
-
}
|
|
999
|
-
});
|
|
1000
|
-
child.stderr.on("data", (chunk) => {
|
|
1001
|
-
if (stderrChunks.reduce((sum, item) => sum + item.byteLength, 0) < 4096) {
|
|
1002
|
-
stderrChunks.push(chunk.subarray(0, 4096));
|
|
1003
|
-
}
|
|
1033
|
+
async function stageOneGitDiff(collection, target) {
|
|
1034
|
+
const sanitized = sanitizeTextEvidenceForUpload({
|
|
1035
|
+
text: target.diffText,
|
|
1036
|
+
redactedFields: [`git_diff.${target.label}`],
|
|
1037
|
+
});
|
|
1038
|
+
if (sanitized.status === "redacted") {
|
|
1039
|
+
collection.redacted.push({
|
|
1040
|
+
kind: "git_diff",
|
|
1041
|
+
label: target.label,
|
|
1042
|
+
redaction: sanitized.redaction,
|
|
1043
|
+
completenessLabel: sanitized.completenessLabel,
|
|
1004
1044
|
});
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1045
|
+
console.error("[raw-evidence] git diff sanitized", JSON.stringify({
|
|
1046
|
+
mode: sanitized.completenessLabel,
|
|
1047
|
+
original_bytes: Buffer.byteLength(target.diffText, "utf8"),
|
|
1048
|
+
uploaded_bytes: sanitized.bytes.byteLength,
|
|
1049
|
+
}));
|
|
1050
|
+
}
|
|
1051
|
+
const raw = sanitized.bytes;
|
|
1052
|
+
const redaction = sanitized.redaction;
|
|
1053
|
+
const contentHash = sha256(raw);
|
|
1054
|
+
if (collection.skipContentHashes.has(contentHash)) {
|
|
1055
|
+
collection.reused.push({
|
|
1056
|
+
kind: "git_diff",
|
|
1057
|
+
label: target.label,
|
|
1058
|
+
content_hash_sha256: contentHash,
|
|
1059
|
+
codex_session_id: null,
|
|
1008
1060
|
});
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
: "max_git_diff_bytes",
|
|
1019
|
-
truncationCapType: timedOut ? "timeout_ms" : "max_bytes_per_diff",
|
|
1020
|
-
});
|
|
1021
|
-
return;
|
|
1022
|
-
}
|
|
1023
|
-
const stderr = Buffer.concat(stderrChunks).toString("utf8").trim();
|
|
1024
|
-
reject(new Error(stderr || `git diff failed with code ${code ?? signal}`));
|
|
1061
|
+
return;
|
|
1062
|
+
}
|
|
1063
|
+
const deferReason = admitToBudget(collection.budget, raw.byteLength);
|
|
1064
|
+
if (deferReason) {
|
|
1065
|
+
markBudgetCapApplied(collection, deferReason);
|
|
1066
|
+
collection.skipped.push({
|
|
1067
|
+
kind: "git_diff",
|
|
1068
|
+
label: target.label,
|
|
1069
|
+
reason: deferReason,
|
|
1025
1070
|
});
|
|
1026
|
-
});
|
|
1027
|
-
}
|
|
1028
|
-
function recordScanned(collection, source, count = 1) {
|
|
1029
|
-
collection.scanned.set(source, (collection.scanned.get(source) ?? 0) + count);
|
|
1030
|
-
}
|
|
1031
|
-
function markBudgetCapApplied(collection, reason) {
|
|
1032
|
-
markCapApplied(collection, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
|
|
1033
|
-
}
|
|
1034
|
-
function markCapApplied(collection, source, capType) {
|
|
1035
|
-
const cap = collection.caps.find((candidate) => candidate.source === source && candidate.cap_type === capType);
|
|
1036
|
-
if (cap)
|
|
1037
|
-
cap.applied = true;
|
|
1038
|
-
}
|
|
1039
|
-
function recordSkipCount(collection, source, reason, count) {
|
|
1040
|
-
if (count <= 0)
|
|
1041
1071
|
return;
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1072
|
+
}
|
|
1073
|
+
const relativePath = path.join("files", `git-${target.label}.diff`);
|
|
1074
|
+
const diffSourceKey = evidenceSourceKey({
|
|
1075
|
+
kind: "git_diff",
|
|
1076
|
+
sessionId: collection.context.workContextId,
|
|
1077
|
+
label: target.label,
|
|
1047
1078
|
});
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
reason,
|
|
1055
|
-
count,
|
|
1056
|
-
...details,
|
|
1079
|
+
const staged = await stageEvidenceBytes(collection, {
|
|
1080
|
+
contentHash,
|
|
1081
|
+
bytes: raw,
|
|
1082
|
+
fileName: path.basename(relativePath),
|
|
1083
|
+
kind: "git_diff",
|
|
1084
|
+
sourceKey: diffSourceKey,
|
|
1057
1085
|
});
|
|
1086
|
+
collection.entries.push(evidenceEntry({
|
|
1087
|
+
context: collection.context,
|
|
1088
|
+
kind: "git_diff",
|
|
1089
|
+
packId: collection.packId,
|
|
1090
|
+
localPath: staged.local_path,
|
|
1091
|
+
relativePath,
|
|
1092
|
+
stagedInPack: staged.staged_in_pack,
|
|
1093
|
+
sourceKey: diffSourceKey,
|
|
1094
|
+
mediaType: "text/x-diff",
|
|
1095
|
+
redactedSummary: gitDiffSummary(target.label, {
|
|
1096
|
+
redacted: Boolean(redaction),
|
|
1097
|
+
truncated: target.truncated,
|
|
1098
|
+
}),
|
|
1099
|
+
redaction,
|
|
1100
|
+
bytes: raw,
|
|
1101
|
+
contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
|
|
1102
|
+
}));
|
|
1058
1103
|
}
|
|
1059
|
-
function
|
|
1060
|
-
|
|
1061
|
-
}
|
|
1062
|
-
function countEvidenceEntries(entries, predicate = () => true) {
|
|
1063
|
-
return entries.reduce((sum, entry) => sum + (predicate(entry) ? evidenceEntryCount(entry) : 0), 0);
|
|
1064
|
-
}
|
|
1065
|
-
function makeEvidenceCompleteness(collection, options) {
|
|
1066
|
-
const sources = new Set(collection.scanned.keys());
|
|
1067
|
-
for (const entry of collection.entries)
|
|
1068
|
-
sources.add(entry.kind);
|
|
1069
|
-
for (const entry of collection.skipped)
|
|
1070
|
-
sources.add(entry.kind);
|
|
1071
|
-
for (const entry of collection.reused)
|
|
1072
|
-
sources.add(entry.kind);
|
|
1073
|
-
for (const entry of collection.truncated)
|
|
1074
|
-
sources.add(entry.kind);
|
|
1075
|
-
for (const entry of collection.failed)
|
|
1076
|
-
sources.add(entry.kind);
|
|
1077
|
-
for (const entry of collection.redacted)
|
|
1078
|
-
sources.add(entry.kind);
|
|
1079
|
-
const sourceCounts = [...sources].sort().map((source) => {
|
|
1080
|
-
const skipped = collection.skipped.filter((entry) => entry.kind === source);
|
|
1081
|
-
return {
|
|
1082
|
-
source,
|
|
1083
|
-
scanned_count: collection.scanned.get(source) ?? 0,
|
|
1084
|
-
included_count: collection.entries.filter((entry) => entry.kind === source).length,
|
|
1085
|
-
skipped_count: countEvidenceEntries(skipped),
|
|
1086
|
-
truncated_count: countEvidenceEntries(collection.truncated, (entry) => entry.kind === source),
|
|
1087
|
-
deferred_count: countEvidenceEntries(skipped, (entry) => entry.reason.startsWith("deferred_")),
|
|
1088
|
-
reused_count: collection.reused.filter((entry) => entry.kind === source).length,
|
|
1089
|
-
failed_count: countEvidenceEntries(collection.failed, (entry) => entry.kind === source),
|
|
1090
|
-
};
|
|
1091
|
-
});
|
|
1092
|
-
const totals = sourceCounts.reduce((sum, count) => ({
|
|
1093
|
-
scanned_count: sum.scanned_count + count.scanned_count,
|
|
1094
|
-
included_count: sum.included_count + count.included_count,
|
|
1095
|
-
skipped_count: sum.skipped_count + count.skipped_count,
|
|
1096
|
-
truncated_count: sum.truncated_count + count.truncated_count,
|
|
1097
|
-
deferred_count: sum.deferred_count + count.deferred_count,
|
|
1098
|
-
reused_count: sum.reused_count + count.reused_count,
|
|
1099
|
-
failed_count: sum.failed_count + count.failed_count,
|
|
1100
|
-
}), {
|
|
1101
|
-
scanned_count: 0,
|
|
1102
|
-
included_count: 0,
|
|
1103
|
-
skipped_count: 0,
|
|
1104
|
-
truncated_count: 0,
|
|
1105
|
-
deferred_count: 0,
|
|
1106
|
-
reused_count: 0,
|
|
1107
|
-
failed_count: 0,
|
|
1108
|
-
});
|
|
1109
|
-
const skipReasonCounts = new Map();
|
|
1110
|
-
for (const skipped of collection.skipped) {
|
|
1111
|
-
const key = `${skipped.kind}:${skipped.reason}`;
|
|
1112
|
-
const existing = skipReasonCounts.get(key);
|
|
1113
|
-
if (existing) {
|
|
1114
|
-
existing.count += evidenceEntryCount(skipped);
|
|
1115
|
-
}
|
|
1116
|
-
else {
|
|
1117
|
-
skipReasonCounts.set(key, {
|
|
1118
|
-
source: skipped.kind,
|
|
1119
|
-
reason: skipped.reason,
|
|
1120
|
-
count: evidenceEntryCount(skipped),
|
|
1121
|
-
});
|
|
1122
|
-
}
|
|
1123
|
-
}
|
|
1124
|
-
const failureReasonCounts = new Map();
|
|
1125
|
-
for (const failed of collection.failed) {
|
|
1126
|
-
const key = `${failed.kind}:${failed.reason}`;
|
|
1127
|
-
const existing = failureReasonCounts.get(key);
|
|
1128
|
-
if (existing) {
|
|
1129
|
-
existing.count += evidenceEntryCount(failed);
|
|
1130
|
-
}
|
|
1131
|
-
else {
|
|
1132
|
-
failureReasonCounts.set(key, {
|
|
1133
|
-
source: failed.kind,
|
|
1134
|
-
reason: failed.reason,
|
|
1135
|
-
count: evidenceEntryCount(failed),
|
|
1136
|
-
});
|
|
1137
|
-
}
|
|
1138
|
-
}
|
|
1139
|
-
const truncationCounts = new Map();
|
|
1140
|
-
for (const truncated of collection.truncated) {
|
|
1141
|
-
const key = `${truncated.kind}:${truncated.reason}`;
|
|
1142
|
-
const existing = truncationCounts.get(key);
|
|
1143
|
-
if (existing) {
|
|
1144
|
-
existing.count += evidenceEntryCount(truncated);
|
|
1145
|
-
existing.observed_bytes = Math.max(existing.observed_bytes ?? 0, truncated.observed_bytes ?? 0);
|
|
1146
|
-
existing.included_bytes = Math.max(existing.included_bytes ?? 0, truncated.included_bytes ?? 0);
|
|
1147
|
-
}
|
|
1148
|
-
else {
|
|
1149
|
-
truncationCounts.set(key, {
|
|
1150
|
-
source: truncated.kind,
|
|
1151
|
-
reason: truncated.reason,
|
|
1152
|
-
count: evidenceEntryCount(truncated),
|
|
1153
|
-
...(truncated.max_bytes !== undefined
|
|
1154
|
-
? { max_bytes: truncated.max_bytes }
|
|
1155
|
-
: {}),
|
|
1156
|
-
...(truncated.observed_bytes !== undefined
|
|
1157
|
-
? { observed_bytes: truncated.observed_bytes }
|
|
1158
|
-
: {}),
|
|
1159
|
-
...(truncated.included_bytes !== undefined
|
|
1160
|
-
? { included_bytes: truncated.included_bytes }
|
|
1161
|
-
: {}),
|
|
1162
|
-
});
|
|
1163
|
-
}
|
|
1104
|
+
function gitDiffSummary(label, state) {
|
|
1105
|
+
if (state.redacted) {
|
|
1106
|
+
return `Raw git ${label} diff preserved locally with env/secret paths excluded and secret-like values deterministically redacted.`;
|
|
1164
1107
|
}
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
const ruleIds = redacted.redaction.rule_counts.map((rule) => rule.rule_id);
|
|
1168
|
-
const key = `${redacted.kind}:${redacted.completenessLabel}:${ruleIds.sort().join(",")}`;
|
|
1169
|
-
const existing = redactionCounts.get(key);
|
|
1170
|
-
if (existing) {
|
|
1171
|
-
existing.count += 1;
|
|
1172
|
-
existing.rule_ids = [...new Set([...existing.rule_ids, ...ruleIds])].sort();
|
|
1173
|
-
}
|
|
1174
|
-
else {
|
|
1175
|
-
redactionCounts.set(key, {
|
|
1176
|
-
source: redacted.kind,
|
|
1177
|
-
status: "sanitized",
|
|
1178
|
-
mode: redacted.completenessLabel,
|
|
1179
|
-
count: 1,
|
|
1180
|
-
rule_ids: [...new Set(ruleIds)].sort(),
|
|
1181
|
-
});
|
|
1182
|
-
}
|
|
1108
|
+
if (state.truncated) {
|
|
1109
|
+
return `Raw git ${label} diff truncated to the capture cap and preserved locally with env/secret paths excluded.`;
|
|
1183
1110
|
}
|
|
1184
|
-
|
|
1185
|
-
totals.truncated_count > 0 ||
|
|
1186
|
-
totals.deferred_count > 0 ||
|
|
1187
|
-
totals.failed_count > 0 ||
|
|
1188
|
-
collection.redacted.length > 0 ||
|
|
1189
|
-
collection.caps.some((cap) => cap.applied);
|
|
1190
|
-
const status = totals.failed_count > 0 &&
|
|
1191
|
-
totals.included_count + totals.reused_count === 0
|
|
1192
|
-
? "failed"
|
|
1193
|
-
: totals.included_count + totals.reused_count === 0 && !hasGaps
|
|
1194
|
-
? "empty"
|
|
1195
|
-
: hasGaps
|
|
1196
|
-
? "partial"
|
|
1197
|
-
: "complete";
|
|
1198
|
-
return EvidenceCompletenessPayloadSchema.parse({
|
|
1199
|
-
schema_version: "evidence-completeness.v1",
|
|
1200
|
-
status,
|
|
1201
|
-
generated_at: options.finishedAt,
|
|
1202
|
-
scan_window: {
|
|
1203
|
-
started_at: options.startedAt,
|
|
1204
|
-
finished_at: options.finishedAt,
|
|
1205
|
-
since_minutes: options.sinceMinutes,
|
|
1206
|
-
},
|
|
1207
|
-
source_counts: sourceCounts,
|
|
1208
|
-
totals,
|
|
1209
|
-
caps: collection.caps,
|
|
1210
|
-
skip_reasons: [...skipReasonCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
|
|
1211
|
-
failure_reasons: [...failureReasonCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
|
|
1212
|
-
truncation_markers: [...truncationCounts.values()].sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`)),
|
|
1213
|
-
redaction_markers: [...redactionCounts.values()].sort((a, b) => `${a.source}:${a.mode}`.localeCompare(`${b.source}:${b.mode}`)),
|
|
1214
|
-
notes: totals.failed_count > 0
|
|
1215
|
-
? ["Evidence collection failed; downstream analysis should not infer confidence."]
|
|
1216
|
-
: collection.redacted.length > 0
|
|
1217
|
-
? ["Evidence was sanitized before upload; downstream analysis should not treat it as raw-complete."]
|
|
1218
|
-
: hasGaps
|
|
1219
|
-
? ["Evidence is incomplete; downstream analysis should lower confidence."]
|
|
1220
|
-
: [],
|
|
1221
|
-
});
|
|
1111
|
+
return `Raw git ${label} diff preserved locally with env/secret paths excluded.`;
|
|
1222
1112
|
}
|
|
1113
|
+
// ---------------------------------------------------------------------------
|
|
1114
|
+
// Walking the session store
|
|
1115
|
+
// ---------------------------------------------------------------------------
|
|
1116
|
+
/**
|
|
1117
|
+
* Newest-first `.jsonl` files under these directories, modified since `cutoffMs`.
|
|
1118
|
+
*
|
|
1119
|
+
* Secret-like directory and file names are never descended into or opened;
|
|
1120
|
+
* symlinked duplicates are collapsed by real path so one transcript reachable
|
|
1121
|
+
* two ways is collected once.
|
|
1122
|
+
*/
|
|
1223
1123
|
async function walkJsonlFiles(dir, cutoffMs) {
|
|
1224
1124
|
const out = [];
|
|
1225
1125
|
const stack = Array.isArray(dir) ? [...dir] : [dir];
|
|
@@ -1246,219 +1146,15 @@ async function walkJsonlFiles(dir, cutoffMs) {
|
|
|
1246
1146
|
if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
|
|
1247
1147
|
continue;
|
|
1248
1148
|
const stat = await fs.stat(full);
|
|
1249
|
-
if (stat.mtimeMs
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
}
|
|
1149
|
+
if (stat.mtimeMs < cutoffMs)
|
|
1150
|
+
continue;
|
|
1151
|
+
const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
|
|
1152
|
+
if (seen.has(dedupeKey))
|
|
1153
|
+
continue;
|
|
1154
|
+
seen.add(dedupeKey);
|
|
1155
|
+
out.push({ file: full, mtimeMs: stat.mtimeMs });
|
|
1256
1156
|
}
|
|
1257
1157
|
}
|
|
1258
1158
|
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
1259
1159
|
return out.map((entry) => entry.file);
|
|
1260
|
-
}
|
|
1261
|
-
function makeManifest(options) {
|
|
1262
|
-
return {
|
|
1263
|
-
schema: "bli.local_raw_evidence_pack.v1",
|
|
1264
|
-
pack_id: options.packId,
|
|
1265
|
-
created_at: options.context.now.toISOString(),
|
|
1266
|
-
work_context_id: options.context.workContextId,
|
|
1267
|
-
session_id: options.context.sessionId,
|
|
1268
|
-
operator_id: options.context.operatorId,
|
|
1269
|
-
operator_label: options.context.operatorLabel,
|
|
1270
|
-
repo_label: options.context.repoLabel ?? path.basename(options.context.repoRoot),
|
|
1271
|
-
worktree_label: options.context.worktreeLabel,
|
|
1272
|
-
active_ticket_id: options.context.activeTicketId ?? null,
|
|
1273
|
-
repo_basename: path.basename(options.context.repoRoot),
|
|
1274
|
-
branch: options.context.branch,
|
|
1275
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
1276
|
-
raw_policy: {
|
|
1277
|
-
raw_prompts: "preserved_private_durable_remote",
|
|
1278
|
-
raw_responses: "preserved_private_durable_remote",
|
|
1279
|
-
transcripts: "preserved_private_durable_remote",
|
|
1280
|
-
claude_transcripts: "preserved_private_durable_remote",
|
|
1281
|
-
tool_payloads: "preserved_private_durable_remote",
|
|
1282
|
-
git_diffs: "preserved_private_durable_remote_env_secret_paths_excluded",
|
|
1283
|
-
agent_image_attachments: "preserved_private_durable_remote_explicit_agent_session_attachment_only",
|
|
1284
|
-
env_files: "never_read",
|
|
1285
|
-
stdout: "manifest_only_no_raw_content",
|
|
1286
|
-
},
|
|
1287
|
-
files: options.entries.map(redactManifestEntry),
|
|
1288
|
-
skipped: options.skipped,
|
|
1289
|
-
redacted: options.redacted,
|
|
1290
|
-
reused: options.reused,
|
|
1291
|
-
};
|
|
1292
|
-
}
|
|
1293
|
-
function evidenceEntry(options) {
|
|
1294
|
-
const digest = sha256(options.bytes);
|
|
1295
|
-
const objectKey = remoteObjectKey({
|
|
1296
|
-
context: options.context,
|
|
1297
|
-
packId: options.packId,
|
|
1298
|
-
relativePath: options.relativePath,
|
|
1299
|
-
contentAddress: options.contentAddress,
|
|
1300
|
-
});
|
|
1301
|
-
return {
|
|
1302
|
-
kind: options.kind,
|
|
1303
|
-
local_path: options.localPath,
|
|
1304
|
-
relative_path: options.relativePath,
|
|
1305
|
-
object_key: objectKey,
|
|
1306
|
-
content_hash_sha256: digest,
|
|
1307
|
-
byte_size: options.bytes.byteLength,
|
|
1308
|
-
media_type: options.mediaType,
|
|
1309
|
-
redacted_summary: options.redactedSummary,
|
|
1310
|
-
...(options.redaction ? { redaction: options.redaction } : {}),
|
|
1311
|
-
codex_session_id: options.codexSessionId ?? null,
|
|
1312
|
-
...(options.artifactMetadata
|
|
1313
|
-
? {
|
|
1314
|
-
artifact_metadata: {
|
|
1315
|
-
...options.artifactMetadata,
|
|
1316
|
-
raw_evidence_pointer_id: objectKey,
|
|
1317
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
1318
|
-
object_key: objectKey,
|
|
1319
|
-
content_hash_sha256: digest,
|
|
1320
|
-
byte_size: options.bytes.byteLength,
|
|
1321
|
-
},
|
|
1322
|
-
}
|
|
1323
|
-
: {}),
|
|
1324
|
-
};
|
|
1325
|
-
}
|
|
1326
|
-
function redactManifestEntry(entry) {
|
|
1327
|
-
return {
|
|
1328
|
-
kind: entry.kind,
|
|
1329
|
-
relative_path: entry.relative_path,
|
|
1330
|
-
object_key: entry.object_key,
|
|
1331
|
-
content_hash_sha256: entry.content_hash_sha256,
|
|
1332
|
-
byte_size: entry.byte_size,
|
|
1333
|
-
media_type: entry.media_type,
|
|
1334
|
-
redacted_summary: entry.redacted_summary,
|
|
1335
|
-
...(entry.redaction ? { redaction: entry.redaction } : {}),
|
|
1336
|
-
...(entry.artifact_metadata
|
|
1337
|
-
? { artifact_metadata: entry.artifact_metadata }
|
|
1338
|
-
: {}),
|
|
1339
|
-
};
|
|
1340
|
-
}
|
|
1341
|
-
function pointerFromEntry(entry) {
|
|
1342
|
-
return {
|
|
1343
|
-
raw_evidence_pointer_id: entry.object_key,
|
|
1344
|
-
privacy_classification: "remote_durable_raw_evidence",
|
|
1345
|
-
retention_policy: {
|
|
1346
|
-
mode: RAW_EVIDENCE_RETENTION_MODE,
|
|
1347
|
-
privacy_classification: "remote_durable_raw_evidence",
|
|
1348
|
-
},
|
|
1349
|
-
storage_scope: "remote_object",
|
|
1350
|
-
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
1351
|
-
object_key: entry.object_key,
|
|
1352
|
-
content_hash_sha256: entry.content_hash_sha256,
|
|
1353
|
-
byte_size: entry.byte_size,
|
|
1354
|
-
media_type: entry.media_type,
|
|
1355
|
-
redacted_summary: entry.redacted_summary,
|
|
1356
|
-
...(entry.redaction ? { redaction: entry.redaction } : {}),
|
|
1357
|
-
};
|
|
1358
|
-
}
|
|
1359
|
-
/**
|
|
1360
|
-
* Raw evidence keys start with human-readable context, then end in immutable
|
|
1361
|
-
* content addresses or pack-relative manifest paths. The local cursor reuses
|
|
1362
|
-
* prior content hashes across syncs; the readable date/session folders are for
|
|
1363
|
-
* operator debugging and incident response.
|
|
1364
|
-
*/
|
|
1365
|
-
function remoteObjectKey(options) {
|
|
1366
|
-
const namespace = readableEvidenceNamespace(options.context);
|
|
1367
|
-
if (options.contentAddress) {
|
|
1368
|
-
return posixPath([
|
|
1369
|
-
...namespace,
|
|
1370
|
-
options.contentAddress,
|
|
1371
|
-
]);
|
|
1372
|
-
}
|
|
1373
|
-
return posixPath([
|
|
1374
|
-
...namespace,
|
|
1375
|
-
options.packId,
|
|
1376
|
-
options.relativePath,
|
|
1377
|
-
]);
|
|
1378
|
-
}
|
|
1379
|
-
function readableEvidenceNamespace(context) {
|
|
1380
|
-
return [
|
|
1381
|
-
"operators",
|
|
1382
|
-
operatorSlug(context),
|
|
1383
|
-
"repos",
|
|
1384
|
-
readableKeySegment(context.repoLabel ?? path.basename(context.repoRoot), "repo"),
|
|
1385
|
-
"worktrees",
|
|
1386
|
-
readableKeySegment(context.worktreeLabel ?? path.basename(context.repoRoot), "worktree"),
|
|
1387
|
-
"tickets",
|
|
1388
|
-
readableKeySegment(context.activeTicketId ?? "unbound", "unbound", {
|
|
1389
|
-
lowercase: false,
|
|
1390
|
-
}),
|
|
1391
|
-
"dates",
|
|
1392
|
-
context.now.toISOString().slice(0, 10),
|
|
1393
|
-
"sessions",
|
|
1394
|
-
readableKeySegment(context.sessionId, "session"),
|
|
1395
|
-
"ids",
|
|
1396
|
-
safeKeySegment(context.operatorId),
|
|
1397
|
-
safeKeySegment(context.workContextId),
|
|
1398
|
-
];
|
|
1399
|
-
}
|
|
1400
|
-
function operatorSlug(context) {
|
|
1401
|
-
const labelBeforeDomain = (context.operatorLabel ?? context.operatorId)
|
|
1402
|
-
.split("@", 1)[0]
|
|
1403
|
-
.trim();
|
|
1404
|
-
const readable = readableKeySegment(labelBeforeDomain, "operator");
|
|
1405
|
-
return `${readable}-${shortHash(context.operatorId).slice(0, 6)}`;
|
|
1406
|
-
}
|
|
1407
|
-
function readableKeySegment(value, fallback, options = {}) {
|
|
1408
|
-
const base = options.lowercase === false ? value : value.toLowerCase();
|
|
1409
|
-
const slug = base
|
|
1410
|
-
.trim()
|
|
1411
|
-
.replace(/[^A-Za-z0-9._-]+/g, "-")
|
|
1412
|
-
.replace(/^-+|-+$/g, "")
|
|
1413
|
-
.replace(/-{2,}/g, "-")
|
|
1414
|
-
.slice(0, 80);
|
|
1415
|
-
return slug || fallback;
|
|
1416
|
-
}
|
|
1417
|
-
function posixPath(parts) {
|
|
1418
|
-
return parts.join("/").replace(/\\/g, "/").replace(/\/+/g, "/");
|
|
1419
|
-
}
|
|
1420
|
-
function rawEvidencePackId(context) {
|
|
1421
|
-
const material = [
|
|
1422
|
-
context.workContextId,
|
|
1423
|
-
context.sessionId,
|
|
1424
|
-
context.now.toISOString(),
|
|
1425
|
-
].join(":");
|
|
1426
|
-
return `${context.workContextId}-${shortHash(material)}`;
|
|
1427
|
-
}
|
|
1428
|
-
function shortHash(value) {
|
|
1429
|
-
return crypto.createHash("sha256").update(value, "utf8").digest("hex").slice(0, 12);
|
|
1430
|
-
}
|
|
1431
|
-
/**
|
|
1432
|
-
* Object keys must satisfy the server's key pattern; ids derived from file
|
|
1433
|
-
* content fall back to a hash rather than failing the whole upload batch.
|
|
1434
|
-
*/
|
|
1435
|
-
function safeKeySegment(value) {
|
|
1436
|
-
return /^[A-Za-z0-9._-]{1,80}$/.test(value) ? value : shortHash(value);
|
|
1437
|
-
}
|
|
1438
|
-
function sha256(value) {
|
|
1439
|
-
return crypto.createHash("sha256").update(value).digest("hex");
|
|
1440
|
-
}
|
|
1441
|
-
function withRedactionContentMetadata(metadata, options) {
|
|
1442
|
-
if (!metadata) {
|
|
1443
|
-
throw new Error("redacted evidence is missing redaction metadata");
|
|
1444
|
-
}
|
|
1445
|
-
return {
|
|
1446
|
-
...metadata,
|
|
1447
|
-
original_content_hash_sha256: sha256(options.originalBytes),
|
|
1448
|
-
sanitized_content_hash_sha256: sha256(options.sanitizedBytes),
|
|
1449
|
-
original_byte_size: options.originalBytes.byteLength,
|
|
1450
|
-
sanitized_byte_size: options.sanitizedBytes.byteLength,
|
|
1451
|
-
};
|
|
1452
|
-
}
|
|
1453
|
-
async function ensurePrivateDir(dir) {
|
|
1454
|
-
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
1455
|
-
await chmodPrivate(dir, 0o700);
|
|
1456
|
-
}
|
|
1457
|
-
async function chmodPrivate(target, mode) {
|
|
1458
|
-
if (process.platform === "win32")
|
|
1459
|
-
return;
|
|
1460
|
-
await fs.chmod(target, mode).catch(() => undefined);
|
|
1461
|
-
}
|
|
1462
|
-
function isSecretLikePath(value) {
|
|
1463
|
-
return SECRET_FILE_SEGMENT_PATTERN.test(value);
|
|
1464
1160
|
}
|