@bli-cockpit/cli 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/attribution-core.js +172 -0
- package/dist/adapters/claude-attribution.js +535 -0
- package/dist/adapters/codex-attribution.js +16 -134
- package/dist/adapters/common.js +4 -1
- package/dist/adapters/local-sources.js +21 -2
- package/dist/adapters/raw-evidence.js +205 -90
- package/dist/commands/local.js +619 -90
- package/dist/cursors/raw-evidence-cursor.js +65 -14
- package/dist/local-state.js +2 -2
- package/dist/repo-identity.js +50 -4
- package/dist/sync-lock.js +113 -0
- package/dist/upload.js +8 -0
- package/package.json +2 -2
|
@@ -1,29 +1,24 @@
|
|
|
1
|
-
import { containsSecretLikeContent } from "@bli-cockpit/telemetry-core";
|
|
2
|
-
import crypto from "node:crypto";
|
|
1
|
+
import { SECRET_FILE_SEGMENT_PATTERN, containsSecretLikeContent, } from "@bli-cockpit/telemetry-core";
|
|
3
2
|
import fs from "node:fs/promises";
|
|
4
3
|
import path from "node:path";
|
|
5
4
|
import { normalizeGitOrigin } from "../repo-identity.js";
|
|
5
|
+
import { sanitizeSessionId, scoreSignalsAgainstWorktrees, sessionIdFromFileName, sha256, shortHash, } from "./attribution-core.js";
|
|
6
6
|
/**
|
|
7
7
|
* Deterministic Codex session JSONL -> repo/worktree attribution.
|
|
8
8
|
*
|
|
9
9
|
* Only metadata-bearing records are inspected (session_meta, turn_context):
|
|
10
10
|
* cwd, workspace roots, and git branch/commit/origin. Prompt, response,
|
|
11
11
|
* reasoning, and tool payload fields are never extracted, printed, or
|
|
12
|
-
* summarized.
|
|
13
|
-
*
|
|
12
|
+
* summarized. Scoring, the deepest-root tie-break, and the threshold/margin
|
|
13
|
+
* decision all live in the shared attribution-core so Codex and Claude Code
|
|
14
|
+
* attribute identically.
|
|
14
15
|
*/
|
|
16
|
+
// Re-exported for callers (and tests) that historically imported these from
|
|
17
|
+
// the codex adapter; the single implementation lives in attribution-core.
|
|
18
|
+
export { sanitizeSessionId, sessionIdFromFileName };
|
|
15
19
|
export const CODEX_ATTRIBUTION_DEFAULT_SINCE_MINUTES = 24 * 60;
|
|
16
20
|
export const CODEX_ATTRIBUTION_DEFAULT_SESSION_LIMIT = 50;
|
|
17
21
|
export const CODEX_SESSION_MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
18
|
-
const SECRET_FILE_SEGMENT_PATTERN = /(^|[/\\])(?:\.env(?:\..*)?|.*(?:secret|credential|private[_-]?key|service[_-]?role).*)$/i;
|
|
19
|
-
const SESSION_FILE_UUID_PATTERN = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i;
|
|
20
|
-
const SCORE_CWD_MATCH = 0.5;
|
|
21
|
-
const SCORE_WORKSPACE_ROOT_MATCH = 0.1;
|
|
22
|
-
const SCORE_ORIGIN_MATCH = 0.3;
|
|
23
|
-
const SCORE_BRANCH_MATCH = 0.15;
|
|
24
|
-
const SCORE_HEAD_SHA_MATCH = 0.05;
|
|
25
|
-
const ATTRIBUTION_MIN_SCORE = 0.4;
|
|
26
|
-
const ATTRIBUTION_MIN_MARGIN = 0.15;
|
|
27
22
|
export async function scanAndAttributeCodexSessions(options) {
|
|
28
23
|
const sinceMinutes = options.sinceMinutes ?? CODEX_ATTRIBUTION_DEFAULT_SINCE_MINUTES;
|
|
29
24
|
const limit = options.limit ?? CODEX_ATTRIBUTION_DEFAULT_SESSION_LIMIT;
|
|
@@ -134,7 +129,14 @@ async function attributeOneSession(file, worktrees) {
|
|
|
134
129
|
worktree: null,
|
|
135
130
|
};
|
|
136
131
|
}
|
|
137
|
-
|
|
132
|
+
const outcome = scoreSignalsAgainstWorktrees({
|
|
133
|
+
cwds: signals.cwds,
|
|
134
|
+
workspaceRoots: signals.workspace_roots,
|
|
135
|
+
originUrls: signals.repository_urls,
|
|
136
|
+
branches: signals.branches,
|
|
137
|
+
headShas: signals.commit_hashes,
|
|
138
|
+
}, worktrees);
|
|
139
|
+
return { ...base, ...outcome };
|
|
138
140
|
}
|
|
139
141
|
export function extractCodexSessionSignals(content) {
|
|
140
142
|
const sessionIds = new Set();
|
|
@@ -204,95 +206,6 @@ export function extractCodexSessionSignals(content) {
|
|
|
204
206
|
parse_error_count: parseErrorCount,
|
|
205
207
|
};
|
|
206
208
|
}
|
|
207
|
-
function scoreSessionAgainstWorktrees(base, signals, worktrees) {
|
|
208
|
-
const hasAnySignal = signals.cwds.length > 0 ||
|
|
209
|
-
signals.workspace_roots.length > 0 ||
|
|
210
|
-
signals.repository_urls.length > 0 ||
|
|
211
|
-
signals.branches.length > 0 ||
|
|
212
|
-
signals.commit_hashes.length > 0;
|
|
213
|
-
if (!hasAnySignal) {
|
|
214
|
-
return {
|
|
215
|
-
...base,
|
|
216
|
-
state: "unattributed",
|
|
217
|
-
reason: "no_repo_signals",
|
|
218
|
-
signals: [],
|
|
219
|
-
attribution_score: 0,
|
|
220
|
-
path_score: 0,
|
|
221
|
-
worktree: null,
|
|
222
|
-
};
|
|
223
|
-
}
|
|
224
|
-
const scored = worktrees.map((worktree) => {
|
|
225
|
-
const matched = [];
|
|
226
|
-
let score = 0;
|
|
227
|
-
let pathScore = 0;
|
|
228
|
-
if (signals.cwds.some((cwd) => isPathWithin(cwd, worktree.repo_root))) {
|
|
229
|
-
score += SCORE_CWD_MATCH;
|
|
230
|
-
pathScore += SCORE_CWD_MATCH;
|
|
231
|
-
matched.push("cwd_match");
|
|
232
|
-
}
|
|
233
|
-
if (signals.workspace_roots.some((root) => isPathWithin(root, worktree.repo_root))) {
|
|
234
|
-
score += SCORE_WORKSPACE_ROOT_MATCH;
|
|
235
|
-
pathScore += SCORE_WORKSPACE_ROOT_MATCH;
|
|
236
|
-
matched.push("workspace_root_match");
|
|
237
|
-
}
|
|
238
|
-
if (worktree.repo_origin_url &&
|
|
239
|
-
signals.repository_urls.includes(worktree.repo_origin_url)) {
|
|
240
|
-
score += SCORE_ORIGIN_MATCH;
|
|
241
|
-
matched.push("origin_url_match");
|
|
242
|
-
}
|
|
243
|
-
if (signals.branches.includes(worktree.branch)) {
|
|
244
|
-
score += SCORE_BRANCH_MATCH;
|
|
245
|
-
matched.push("branch_match");
|
|
246
|
-
}
|
|
247
|
-
if (worktree.head_sha && signals.commit_hashes.includes(worktree.head_sha)) {
|
|
248
|
-
score += SCORE_HEAD_SHA_MATCH;
|
|
249
|
-
matched.push("head_sha_match");
|
|
250
|
-
}
|
|
251
|
-
return { worktree, score, pathScore, matched };
|
|
252
|
-
});
|
|
253
|
-
scored.sort((a, b) => b.score - a.score);
|
|
254
|
-
const best = scored[0];
|
|
255
|
-
const secondBestScore = scored[1]?.score ?? 0;
|
|
256
|
-
if (!best || best.score === 0) {
|
|
257
|
-
const reason = signals.cwds.length > 0 || signals.workspace_roots.length > 0
|
|
258
|
-
? "cwd_outside_scanned_worktrees"
|
|
259
|
-
: "no_matching_worktree_signals";
|
|
260
|
-
return {
|
|
261
|
-
...base,
|
|
262
|
-
state: "unattributed",
|
|
263
|
-
reason,
|
|
264
|
-
signals: [],
|
|
265
|
-
attribution_score: 0,
|
|
266
|
-
path_score: 0,
|
|
267
|
-
worktree: null,
|
|
268
|
-
};
|
|
269
|
-
}
|
|
270
|
-
const score = clampScore(best.score);
|
|
271
|
-
const pathScore = clampScore(best.pathScore);
|
|
272
|
-
if (best.score >= ATTRIBUTION_MIN_SCORE &&
|
|
273
|
-
best.score - secondBestScore >= ATTRIBUTION_MIN_MARGIN) {
|
|
274
|
-
return {
|
|
275
|
-
...base,
|
|
276
|
-
state: "attributed",
|
|
277
|
-
reason: "deterministic_signal_match",
|
|
278
|
-
signals: best.matched,
|
|
279
|
-
attribution_score: score,
|
|
280
|
-
path_score: pathScore,
|
|
281
|
-
worktree: best.worktree,
|
|
282
|
-
};
|
|
283
|
-
}
|
|
284
|
-
return {
|
|
285
|
-
...base,
|
|
286
|
-
state: "ambiguous",
|
|
287
|
-
reason: best.score - secondBestScore < ATTRIBUTION_MIN_MARGIN
|
|
288
|
-
? "multiple_worktrees_close_scores"
|
|
289
|
-
: "signal_score_below_threshold",
|
|
290
|
-
signals: best.matched,
|
|
291
|
-
attribution_score: score,
|
|
292
|
-
path_score: pathScore,
|
|
293
|
-
worktree: null,
|
|
294
|
-
};
|
|
295
|
-
}
|
|
296
209
|
function skippedResult(base, reason) {
|
|
297
210
|
return {
|
|
298
211
|
...base,
|
|
@@ -304,41 +217,10 @@ function skippedResult(base, reason) {
|
|
|
304
217
|
worktree: null,
|
|
305
218
|
};
|
|
306
219
|
}
|
|
307
|
-
export function sessionIdFromFileName(fileName) {
|
|
308
|
-
const match = fileName.match(SESSION_FILE_UUID_PATTERN);
|
|
309
|
-
return match?.[1]?.toLowerCase() ?? null;
|
|
310
|
-
}
|
|
311
|
-
/**
|
|
312
|
-
* Session ids come from file content and end up inside remote object keys, so
|
|
313
|
-
* anything outside the safe charset falls back to a hash of the raw value
|
|
314
|
-
* instead of poisoning every begin request in the batch.
|
|
315
|
-
*/
|
|
316
|
-
export function sanitizeSessionId(value) {
|
|
317
|
-
if (!value)
|
|
318
|
-
return null;
|
|
319
|
-
if (/^[A-Za-z0-9._-]{4,80}$/.test(value))
|
|
320
|
-
return value;
|
|
321
|
-
return shortHash(value);
|
|
322
|
-
}
|
|
323
|
-
function isPathWithin(candidate, root) {
|
|
324
|
-
const normalizedCandidate = path.resolve(candidate);
|
|
325
|
-
const normalizedRoot = path.resolve(root);
|
|
326
|
-
return (normalizedCandidate === normalizedRoot ||
|
|
327
|
-
normalizedCandidate.startsWith(normalizedRoot + path.sep));
|
|
328
|
-
}
|
|
329
220
|
function addString(target, value) {
|
|
330
221
|
if (typeof value === "string" && value.trim())
|
|
331
222
|
target.add(value.trim());
|
|
332
223
|
}
|
|
333
|
-
function clampScore(value) {
|
|
334
|
-
return Math.min(1, Math.max(0, Number(value.toFixed(4))));
|
|
335
|
-
}
|
|
336
224
|
function isSecretLikePath(value) {
|
|
337
225
|
return SECRET_FILE_SEGMENT_PATTERN.test(value);
|
|
338
|
-
}
|
|
339
|
-
function shortHash(value) {
|
|
340
|
-
return crypto.createHash("sha256").update(value, "utf8").digest("hex").slice(0, 16);
|
|
341
|
-
}
|
|
342
|
-
function sha256(value) {
|
|
343
|
-
return crypto.createHash("sha256").update(value).digest("hex");
|
|
344
226
|
}
|
package/dist/adapters/common.js
CHANGED
|
@@ -20,10 +20,13 @@ export function makeCaptureProvenance(context, captureSource) {
|
|
|
20
20
|
};
|
|
21
21
|
}
|
|
22
22
|
export function makeUnavailableScan(context, captureSource, adapterName, reasonLabel) {
|
|
23
|
+
return makeSourceScan(context, captureSource, adapterName, "skipped", reasonLabel);
|
|
24
|
+
}
|
|
25
|
+
export function makeSourceScan(context, captureSource, adapterName, status, reasonLabel) {
|
|
23
26
|
return SourceScanResultSchema.parse({
|
|
24
27
|
adapter: makeSourceAdapterIdentity(captureSource, adapterName),
|
|
25
28
|
work_context_id: context.workContextId,
|
|
26
|
-
status
|
|
29
|
+
status,
|
|
27
30
|
started_at: context.now.toISOString(),
|
|
28
31
|
finished_at: context.now.toISOString(),
|
|
29
32
|
diagnostic_labels: [reasonLabel],
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { collectCarState } from "./car-state.js";
|
|
2
|
-
import
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import { makeSourceScan, makeUnavailableScan, } from "./common.js";
|
|
3
4
|
import { collectGitState } from "./git-state.js";
|
|
4
5
|
import { collectRawEvidencePack, } from "./raw-evidence.js";
|
|
5
6
|
import { generateDumbRiskFlags } from "./risk-flags.js";
|
|
@@ -21,8 +22,13 @@ export async function runLocalSourceCollectors(options) {
|
|
|
21
22
|
repoRoot: options.repoRoot,
|
|
22
23
|
sessionsDir: options.rawEvidenceSessionsDir,
|
|
23
24
|
includeCodexJsonl: options.rawEvidenceIncludeCodexJsonl,
|
|
25
|
+
includeClaudeJsonl: options.rawEvidenceIncludeClaudeJsonl,
|
|
24
26
|
codexSessionFiles: options.rawEvidenceCodexSessionFiles,
|
|
27
|
+
claudeSessionFiles: options.rawEvidenceClaudeSessionFiles,
|
|
25
28
|
skipContentHashes: options.rawEvidenceSkipContentHashes,
|
|
29
|
+
byteBudget: options.rawEvidenceByteBudget,
|
|
30
|
+
objectBudget: options.rawEvidenceObjectBudget,
|
|
31
|
+
budget: options.rawEvidenceBudget,
|
|
26
32
|
})
|
|
27
33
|
: {
|
|
28
34
|
scan: makeUnavailableScan(context, "codex_jsonl", "codex-jsonl", "raw_evidence_state_dir_not_configured"),
|
|
@@ -35,10 +41,16 @@ export async function runLocalSourceCollectors(options) {
|
|
|
35
41
|
makeUnavailableScan(context, "mcp_local", "mcp-local", "mcp_activity_not_configured"),
|
|
36
42
|
makeUnavailableScan(context, "claude_hooks", "claude-hooks", "claude_hooks_not_present"),
|
|
37
43
|
];
|
|
44
|
+
// claude_jsonl availability (B.4): "ok" when ~/.claude/projects exists, else
|
|
45
|
+
// skipped with claude_projects_dir_not_found so an operator on a non-Claude
|
|
46
|
+
// machine sees why the claude funnel is empty. Collection itself is gated
|
|
47
|
+
// separately by collect_claude_jsonl in the sync orchestrator.
|
|
48
|
+
const claudeJsonlScan = await makeClaudeJsonlScan(context, options.claudeProjectsDir);
|
|
38
49
|
const candidates = collectBindingCandidates([
|
|
39
50
|
git.scan,
|
|
40
51
|
car.scan,
|
|
41
52
|
rawEvidence.scan,
|
|
53
|
+
claudeJsonlScan,
|
|
42
54
|
...unavailableScans,
|
|
43
55
|
]);
|
|
44
56
|
const binding = resolveTicketBinding({
|
|
@@ -56,7 +68,7 @@ export async function runLocalSourceCollectors(options) {
|
|
|
56
68
|
binding,
|
|
57
69
|
});
|
|
58
70
|
return {
|
|
59
|
-
scans: [git.scan, car.scan, rawEvidence.scan, ...unavailableScans],
|
|
71
|
+
scans: [git.scan, car.scan, rawEvidence.scan, claudeJsonlScan, ...unavailableScans],
|
|
60
72
|
facts: {
|
|
61
73
|
git: gitFacts,
|
|
62
74
|
car: carFacts,
|
|
@@ -69,6 +81,13 @@ export async function runLocalSourceCollectors(options) {
|
|
|
69
81
|
function collectBindingCandidates(scans) {
|
|
70
82
|
return scans.flatMap((scan) => scan.ticket_binding_candidates);
|
|
71
83
|
}
|
|
84
|
+
async function makeClaudeJsonlScan(context, projectsDir) {
|
|
85
|
+
if (!projectsDir) {
|
|
86
|
+
return makeUnavailableScan(context, "claude_jsonl", "claude-jsonl", "claude_projects_dir_not_configured");
|
|
87
|
+
}
|
|
88
|
+
const present = await fs.stat(projectsDir).then((stat) => stat.isDirectory(), () => false);
|
|
89
|
+
return makeSourceScan(context, "claude_jsonl", "claude-jsonl", present ? "ok" : "skipped", present ? "claude_projects_dir_present" : "claude_projects_dir_not_found");
|
|
90
|
+
}
|
|
72
91
|
function isGitStateFacts(value) {
|
|
73
92
|
return Boolean(value && typeof value === "object" && "repo_root" in value);
|
|
74
93
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SourceScanResultSchema, containsSecretLikeContent, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { SECRET_FILE_SEGMENT_PATTERN, SourceScanResultSchema, containsSecretLikeContent, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import { execFile } from "node:child_process";
|
|
3
3
|
import crypto from "node:crypto";
|
|
4
4
|
import fs from "node:fs/promises";
|
|
@@ -12,7 +12,14 @@ const DEFAULT_SESSION_LIMIT = 50;
|
|
|
12
12
|
const MAX_GIT_DIFF_BYTES = 2 * 1024 * 1024;
|
|
13
13
|
export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
|
|
14
14
|
export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
|
|
15
|
-
|
|
15
|
+
// Per-sync upload budgets enforced at COLLECTION time (D7b). With sidecars a
|
|
16
|
+
// worst-case sync could buffer hundreds of MiB and fire thousands of chunk
|
|
17
|
+
// POSTs; the byte budget bounds buffered bytes and the object budget bounds
|
|
18
|
+
// request count. Overflow becomes a deferred-skip entry, picked up next sync
|
|
19
|
+
// (content addressing makes this convergent).
|
|
20
|
+
export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 512 * 1024 * 1024;
|
|
21
|
+
export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
|
|
22
|
+
const CLAUDE_MAX_COLLECT_FILE_BYTES = 10 * 1024 * 1024;
|
|
16
23
|
export async function collectRawEvidencePack(context, options) {
|
|
17
24
|
const startedAt = context.now.toISOString();
|
|
18
25
|
const packId = rawEvidencePackId(context);
|
|
@@ -21,35 +28,37 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
21
28
|
const entries = [];
|
|
22
29
|
const skipped = [];
|
|
23
30
|
const reused = [];
|
|
24
|
-
const
|
|
31
|
+
const collection = {
|
|
32
|
+
context,
|
|
33
|
+
filesDir,
|
|
34
|
+
packId,
|
|
35
|
+
entries,
|
|
36
|
+
skipped,
|
|
37
|
+
reused,
|
|
38
|
+
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
39
|
+
budget: options.budget ?? {
|
|
40
|
+
remainingBytes: options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET,
|
|
41
|
+
remainingObjects: options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET,
|
|
42
|
+
},
|
|
43
|
+
index: { value: 0 },
|
|
44
|
+
};
|
|
25
45
|
try {
|
|
26
46
|
await ensurePrivateDir(evidenceDir);
|
|
27
47
|
await ensurePrivateDir(filesDir);
|
|
28
48
|
if (options.includeCodexJsonl !== false) {
|
|
29
|
-
await collectCodexJsonlFiles({
|
|
30
|
-
context,
|
|
31
|
-
filesDir,
|
|
32
|
-
packId,
|
|
33
|
-
entries,
|
|
34
|
-
skipped,
|
|
35
|
-
reused,
|
|
36
|
-
skipContentHashes,
|
|
49
|
+
await collectCodexJsonlFiles(collection, {
|
|
37
50
|
codexSessionFiles: options.codexSessionFiles,
|
|
38
51
|
sessionsDir: options.sessionsDir,
|
|
39
52
|
sinceMinutes: options.sinceMinutes ?? DEFAULT_SINCE_MINUTES,
|
|
40
53
|
limit: options.sessionLimit ?? DEFAULT_SESSION_LIMIT,
|
|
41
54
|
});
|
|
42
55
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
skipped,
|
|
50
|
-
reused,
|
|
51
|
-
skipContentHashes,
|
|
52
|
-
});
|
|
56
|
+
if (options.includeClaudeJsonl !== false && options.claudeSessionFiles) {
|
|
57
|
+
await collectClaudeJsonlFiles(collection, options.claudeSessionFiles);
|
|
58
|
+
}
|
|
59
|
+
await collectGitDiffFiles(collection, options.repoRoot);
|
|
60
|
+
const deferredByteBudgetCount = skipped.filter((entry) => entry.reason === "deferred_byte_budget").length;
|
|
61
|
+
const deferredObjectBudgetCount = skipped.filter((entry) => entry.reason === "deferred_object_budget").length;
|
|
53
62
|
if (entries.length === 0) {
|
|
54
63
|
const facts = {
|
|
55
64
|
pack_id: packId,
|
|
@@ -60,6 +69,8 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
60
69
|
byte_size: 0,
|
|
61
70
|
skipped_count: skipped.length,
|
|
62
71
|
reused_count: reused.length,
|
|
72
|
+
deferred_byte_budget_count: deferredByteBudgetCount,
|
|
73
|
+
deferred_object_budget_count: deferredObjectBudgetCount,
|
|
63
74
|
content_kinds: [],
|
|
64
75
|
pointers: [],
|
|
65
76
|
upload_files: [],
|
|
@@ -78,7 +89,6 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
78
89
|
const manifestWithoutSelf = makeManifest({
|
|
79
90
|
context,
|
|
80
91
|
packId,
|
|
81
|
-
evidenceDir,
|
|
82
92
|
entries,
|
|
83
93
|
skipped,
|
|
84
94
|
reused,
|
|
@@ -108,6 +118,8 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
108
118
|
byte_size: entries.reduce((sum, entry) => sum + entry.byte_size, 0),
|
|
109
119
|
skipped_count: skipped.length,
|
|
110
120
|
reused_count: reused.length,
|
|
121
|
+
deferred_byte_budget_count: deferredByteBudgetCount,
|
|
122
|
+
deferred_object_budget_count: deferredObjectBudgetCount,
|
|
111
123
|
content_kinds: [...new Set(entries.map((entry) => entry.kind))],
|
|
112
124
|
pointers: entries.map(pointerFromEntry),
|
|
113
125
|
upload_files: entries.map((entry) => ({
|
|
@@ -174,88 +186,181 @@ function makeRawEvidenceScan(options) {
|
|
|
174
186
|
],
|
|
175
187
|
});
|
|
176
188
|
}
|
|
177
|
-
async function collectCodexJsonlFiles(options) {
|
|
189
|
+
async function collectCodexJsonlFiles(collection, options) {
|
|
178
190
|
const candidates = options.codexSessionFiles
|
|
179
191
|
? options.codexSessionFiles.map((file) => ({
|
|
180
192
|
filePath: file.local_path,
|
|
181
193
|
codexSessionId: file.codex_session_id,
|
|
182
194
|
}))
|
|
183
|
-
: (await walkJsonlFiles(options.sessionsDir ?? path.join(os.homedir(), ".codex", "sessions"),
|
|
195
|
+
: (await walkJsonlFiles(options.sessionsDir ?? path.join(os.homedir(), ".codex", "sessions"), collection.context.now.getTime() - options.sinceMinutes * 60 * 1000))
|
|
184
196
|
.slice(0, options.limit)
|
|
185
197
|
.map((filePath) => ({ filePath, codexSessionId: null }));
|
|
186
|
-
let index = 0;
|
|
187
198
|
for (const candidate of candidates) {
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
199
|
+
const codexSessionId = candidate.codexSessionId ?? shortHash(candidate.filePath);
|
|
200
|
+
await collectOneEvidenceFile(collection, {
|
|
201
|
+
filePath: candidate.filePath,
|
|
202
|
+
kind: "codex_jsonl",
|
|
203
|
+
sessionId: codexSessionId,
|
|
204
|
+
mediaType: "application/jsonl",
|
|
205
|
+
redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
206
|
+
contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async function collectClaudeJsonlFiles(collection, sessions) {
|
|
211
|
+
for (const session of sessions) {
|
|
212
|
+
const sessionId = session.claude_session_id;
|
|
213
|
+
if (session.main_file_oversized) {
|
|
214
|
+
// D7: the oversized main was attributed via a streamed read but its bytes
|
|
215
|
+
// are never uploaded (server commit assembles in memory). Its sidecars
|
|
216
|
+
// still collect below.
|
|
217
|
+
collection.skipped.push({
|
|
218
|
+
kind: "claude_jsonl",
|
|
219
|
+
label: path.basename(session.local_path),
|
|
220
|
+
reason: "file_too_large",
|
|
194
221
|
});
|
|
195
|
-
continue;
|
|
196
222
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
223
|
+
else if (session.skip_main) {
|
|
224
|
+
// D9 damped: prior durable copy is still good enough; collect sidecars
|
|
225
|
+
// only. The session reports reused_existing from cursor state, so no
|
|
226
|
+
// skipped entry is recorded here.
|
|
200
227
|
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
228
|
+
else {
|
|
229
|
+
await collectOneEvidenceFile(collection, {
|
|
230
|
+
filePath: session.local_path,
|
|
231
|
+
kind: "claude_jsonl",
|
|
232
|
+
sessionId,
|
|
233
|
+
mediaType: "application/jsonl",
|
|
234
|
+
// Re-check size at collection: a main that grew past the cap between
|
|
235
|
+
// attribution and collection is an honest file_too_large skip, not an
|
|
236
|
+
// upload_failed at the chunk client.
|
|
237
|
+
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
238
|
+
redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
239
|
+
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/${hash16}.jsonl`,
|
|
206
240
|
});
|
|
207
|
-
continue;
|
|
208
241
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
if (options.skipContentHashes.has(contentHash)) {
|
|
220
|
-
options.reused.push({
|
|
221
|
-
kind: "codex_jsonl",
|
|
222
|
-
label: fileName,
|
|
223
|
-
content_hash_sha256: contentHash,
|
|
224
|
-
codex_session_id: codexSessionId,
|
|
242
|
+
for (const sidecar of session.sidecar_files) {
|
|
243
|
+
const stem = path.basename(sidecar.local_path).replace(/\.jsonl$/i, "");
|
|
244
|
+
await collectOneEvidenceFile(collection, {
|
|
245
|
+
filePath: sidecar.local_path,
|
|
246
|
+
kind: "claude_jsonl_sidecar",
|
|
247
|
+
sessionId,
|
|
248
|
+
mediaType: "application/jsonl",
|
|
249
|
+
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
250
|
+
redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
251
|
+
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeKeySegment(stem)}-${hash16}.jsonl`,
|
|
225
252
|
});
|
|
226
|
-
continue;
|
|
227
253
|
}
|
|
228
|
-
index += 1;
|
|
229
|
-
const relativePath = path.join("files", `${String(index).padStart(3, "0")}-${shortHash(candidate.filePath)}-${fileName}`);
|
|
230
|
-
const destination = path.join(options.filesDir, path.basename(relativePath));
|
|
231
|
-
await fs.writeFile(destination, raw, { mode: 0o600 });
|
|
232
|
-
await chmodPrivate(destination, 0o600);
|
|
233
|
-
options.entries.push(evidenceEntry({
|
|
234
|
-
kind: "codex_jsonl",
|
|
235
|
-
packId: options.packId,
|
|
236
|
-
operatorId: options.context.operatorId,
|
|
237
|
-
workContextId: options.context.workContextId,
|
|
238
|
-
localPath: destination,
|
|
239
|
-
relativePath,
|
|
240
|
-
mediaType: "application/jsonl",
|
|
241
|
-
redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
242
|
-
bytes: raw,
|
|
243
|
-
codexSessionId,
|
|
244
|
-
contentAddress: `codex/${safeKeySegment(codexSessionId)}/${contentHash.slice(0, 16)}.jsonl`,
|
|
245
|
-
}));
|
|
246
254
|
}
|
|
247
255
|
}
|
|
248
|
-
|
|
256
|
+
/**
|
|
257
|
+
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
258
|
+
* attributed transcript into the pack. The secret guard runs again here even
|
|
259
|
+
* though attribution already guarded — three reads per file (attribution,
|
|
260
|
+
* collection, server commit) is the accepted defense-in-depth cost; do not
|
|
261
|
+
* "optimize" a layer away.
|
|
262
|
+
*/
|
|
263
|
+
async function collectOneEvidenceFile(collection, options) {
|
|
264
|
+
const fileName = path.basename(options.filePath);
|
|
265
|
+
if (isSecretLikePath(fileName)) {
|
|
266
|
+
collection.skipped.push({
|
|
267
|
+
kind: options.kind,
|
|
268
|
+
label: fileName,
|
|
269
|
+
reason: "secret_like_file_name",
|
|
270
|
+
});
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
let raw;
|
|
274
|
+
try {
|
|
275
|
+
raw = await fs.readFile(options.filePath);
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
collection.skipped.push({
|
|
279
|
+
kind: options.kind,
|
|
280
|
+
label: fileName,
|
|
281
|
+
reason: "file_read_failed",
|
|
282
|
+
});
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
if (options.maxFileBytes && raw.byteLength > options.maxFileBytes) {
|
|
286
|
+
collection.skipped.push({
|
|
287
|
+
kind: options.kind,
|
|
288
|
+
label: fileName,
|
|
289
|
+
reason: "file_too_large",
|
|
290
|
+
});
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (containsSecretLikeContent(raw.toString("utf8"))) {
|
|
294
|
+
collection.skipped.push({
|
|
295
|
+
kind: options.kind,
|
|
296
|
+
label: fileName,
|
|
297
|
+
reason: "secret_like_content_guard",
|
|
298
|
+
});
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
const contentHash = sha256(raw);
|
|
302
|
+
if (collection.skipContentHashes.has(contentHash)) {
|
|
303
|
+
collection.reused.push({
|
|
304
|
+
kind: options.kind,
|
|
305
|
+
label: fileName,
|
|
306
|
+
content_hash_sha256: contentHash,
|
|
307
|
+
codex_session_id: options.sessionId,
|
|
308
|
+
});
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
const deferReason = admitToBudget(collection.budget, raw.byteLength);
|
|
312
|
+
if (deferReason) {
|
|
313
|
+
collection.skipped.push({
|
|
314
|
+
kind: options.kind,
|
|
315
|
+
label: fileName,
|
|
316
|
+
reason: deferReason,
|
|
317
|
+
});
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
collection.index.value += 1;
|
|
321
|
+
const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(options.filePath)}-${fileName}`);
|
|
322
|
+
const destination = path.join(collection.filesDir, path.basename(relativePath));
|
|
323
|
+
await fs.writeFile(destination, raw, { mode: 0o600 });
|
|
324
|
+
await chmodPrivate(destination, 0o600);
|
|
325
|
+
collection.entries.push(evidenceEntry({
|
|
326
|
+
kind: options.kind,
|
|
327
|
+
packId: collection.packId,
|
|
328
|
+
operatorId: collection.context.operatorId,
|
|
329
|
+
workContextId: collection.context.workContextId,
|
|
330
|
+
localPath: destination,
|
|
331
|
+
relativePath,
|
|
332
|
+
mediaType: options.mediaType,
|
|
333
|
+
redactedSummary: options.redactedSummary,
|
|
334
|
+
bytes: raw,
|
|
335
|
+
codexSessionId: options.sessionId,
|
|
336
|
+
contentAddress: options.contentAddress(contentHash.slice(0, 16)),
|
|
337
|
+
}));
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Decrements the per-sync budget when a file fits, or returns a deferred-skip
|
|
341
|
+
* reason when it does not. The object budget bounds request count; the byte
|
|
342
|
+
* budget bounds buffered bytes.
|
|
343
|
+
*/
|
|
344
|
+
function admitToBudget(budget, byteLength) {
|
|
345
|
+
if (budget.remainingObjects <= 0)
|
|
346
|
+
return "deferred_object_budget";
|
|
347
|
+
if (byteLength > budget.remainingBytes)
|
|
348
|
+
return "deferred_byte_budget";
|
|
349
|
+
budget.remainingObjects -= 1;
|
|
350
|
+
budget.remainingBytes -= byteLength;
|
|
351
|
+
return null;
|
|
352
|
+
}
|
|
353
|
+
async function collectGitDiffFiles(collection, repoRoot) {
|
|
249
354
|
const diffTargets = [
|
|
250
355
|
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
251
356
|
{ label: "staged", args: ["diff", "--cached", "--no-ext-diff", "--"] },
|
|
252
357
|
];
|
|
253
358
|
for (const target of diffTargets) {
|
|
254
|
-
const diff = await runGitDiff(target.args,
|
|
359
|
+
const diff = await runGitDiff(target.args, repoRoot);
|
|
255
360
|
if (!diff.trim())
|
|
256
361
|
continue;
|
|
257
362
|
if (containsSecretLikeContent(diff)) {
|
|
258
|
-
|
|
363
|
+
collection.skipped.push({
|
|
259
364
|
kind: "git_diff",
|
|
260
365
|
label: target.label,
|
|
261
366
|
reason: "secret_like_content_guard",
|
|
@@ -264,8 +369,8 @@ async function collectGitDiffFiles(options) {
|
|
|
264
369
|
}
|
|
265
370
|
const raw = Buffer.from(diff.slice(0, MAX_GIT_DIFF_BYTES), "utf8");
|
|
266
371
|
const contentHash = sha256(raw);
|
|
267
|
-
if (
|
|
268
|
-
|
|
372
|
+
if (collection.skipContentHashes.has(contentHash)) {
|
|
373
|
+
collection.reused.push({
|
|
269
374
|
kind: "git_diff",
|
|
270
375
|
label: target.label,
|
|
271
376
|
content_hash_sha256: contentHash,
|
|
@@ -273,15 +378,24 @@ async function collectGitDiffFiles(options) {
|
|
|
273
378
|
});
|
|
274
379
|
continue;
|
|
275
380
|
}
|
|
381
|
+
const deferReason = admitToBudget(collection.budget, raw.byteLength);
|
|
382
|
+
if (deferReason) {
|
|
383
|
+
collection.skipped.push({
|
|
384
|
+
kind: "git_diff",
|
|
385
|
+
label: target.label,
|
|
386
|
+
reason: deferReason,
|
|
387
|
+
});
|
|
388
|
+
continue;
|
|
389
|
+
}
|
|
276
390
|
const relativePath = path.join("files", `git-${target.label}.diff`);
|
|
277
|
-
const destination = path.join(
|
|
391
|
+
const destination = path.join(collection.filesDir, path.basename(relativePath));
|
|
278
392
|
await fs.writeFile(destination, raw, { mode: 0o600 });
|
|
279
393
|
await chmodPrivate(destination, 0o600);
|
|
280
|
-
|
|
394
|
+
collection.entries.push(evidenceEntry({
|
|
281
395
|
kind: "git_diff",
|
|
282
|
-
packId:
|
|
283
|
-
operatorId:
|
|
284
|
-
workContextId:
|
|
396
|
+
packId: collection.packId,
|
|
397
|
+
operatorId: collection.context.operatorId,
|
|
398
|
+
workContextId: collection.context.workContextId,
|
|
285
399
|
localPath: destination,
|
|
286
400
|
relativePath,
|
|
287
401
|
mediaType: "text/x-diff",
|
|
@@ -358,6 +472,7 @@ function makeManifest(options) {
|
|
|
358
472
|
raw_prompts: "preserved_private_durable_remote",
|
|
359
473
|
raw_responses: "preserved_private_durable_remote",
|
|
360
474
|
transcripts: "preserved_private_durable_remote",
|
|
475
|
+
claude_transcripts: "preserved_private_durable_remote",
|
|
361
476
|
tool_payloads: "preserved_private_durable_remote",
|
|
362
477
|
git_diffs: "preserved_private_durable_remote_env_secret_paths_excluded",
|
|
363
478
|
env_files: "never_read",
|
|
@@ -417,10 +532,10 @@ function pointerFromEntry(entry) {
|
|
|
417
532
|
};
|
|
418
533
|
}
|
|
419
534
|
/**
|
|
420
|
-
* Codex JSONL and git diff objects are content-addressed (kind/id/hash)
|
|
421
|
-
* same content maps to the same remote key across syncs: interrupted
|
|
422
|
-
* resume and repeated syncs dedupe server-side. Pack-scoped keys remain
|
|
423
|
-
* the per-sync manifest.
|
|
535
|
+
* Codex/Claude JSONL and git diff objects are content-addressed (kind/id/hash)
|
|
536
|
+
* so the same content maps to the same remote key across syncs: interrupted
|
|
537
|
+
* uploads resume and repeated syncs dedupe server-side. Pack-scoped keys remain
|
|
538
|
+
* for the per-sync manifest.
|
|
424
539
|
*/
|
|
425
540
|
function remoteObjectKey(options) {
|
|
426
541
|
if (options.contentAddress) {
|