@bli-cockpit/cli 0.1.2 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -9
- package/dist/adapters/codex-attribution.js +344 -0
- package/dist/adapters/local-sources.js +3 -0
- package/dist/adapters/raw-evidence.js +97 -17
- package/dist/commands/local.js +389 -16
- package/dist/cursors/raw-evidence-cursor.js +129 -0
- package/dist/evidence-upload-client.js +362 -0
- package/dist/local-state.js +93 -9
- package/dist/repo-identity.js +203 -0
- package/dist/server.js +7 -3
- package/dist/upload.js +271 -95
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -4,8 +4,8 @@ Public BLI Cockpit command-line interface for approved operators and interns.
|
|
|
4
4
|
|
|
5
5
|
The npm package is public; the Cockpit backend and admin tooling are not. The
|
|
6
6
|
CLI pairs a local laptop with the private Cockpit dashboard, records safe
|
|
7
|
-
work context, uploads consented raw Codex/diff evidence to private
|
|
8
|
-
|
|
7
|
+
work context, uploads consented raw Codex/diff evidence to durable private
|
|
8
|
+
storage, and uploads metadata refs after dashboard-approved
|
|
9
9
|
device pairing.
|
|
10
10
|
|
|
11
11
|
## One-paste install
|
|
@@ -21,15 +21,38 @@ npm exec --yes --package=@bli-cockpit/cli@latest -- cockpit onboard \
|
|
|
21
21
|
--repo "$PWD"
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
+
If a work folder contains multiple repos, run the same command from the parent:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
cd ~/Downloads/bluepearl-workspace
|
|
28
|
+
COCKPIT_DEVICE_NAME="$(scutil --get ComputerName 2>/dev/null || hostname -s)"
|
|
29
|
+
npm exec --yes --package=@bli-cockpit/cli@latest -- cockpit onboard \
|
|
30
|
+
--dashboard-url <DASHBOARD_URL> \
|
|
31
|
+
--email <APPROVED_EMAIL> \
|
|
32
|
+
--device-name "$COCKPIT_DEVICE_NAME" \
|
|
33
|
+
--repo "$PWD"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Parent mode scans child git repos/worktrees, creates one stable work context per
|
|
37
|
+
worktree, and the dashboard rolls them up under one repo row with worktree
|
|
38
|
+
drilldown. Device approval is still one time per laptop/dashboard/email. Codex
|
|
39
|
+
JSONL transcripts are attributed to a repo/worktree deterministically (session
|
|
40
|
+
cwd, workspace roots, git origin/branch/commit signals) and upload only into
|
|
41
|
+
that repo's evidence lane; transcripts that cannot be attributed to exactly one
|
|
42
|
+
worktree are reported as ambiguous/unattributed with reason labels instead of
|
|
43
|
+
being duplicated across repos or dropped.
|
|
44
|
+
|
|
24
45
|
What happens:
|
|
25
46
|
|
|
26
47
|
1. npm downloads the public `@bli-cockpit/cli` package.
|
|
27
48
|
2. `cockpit onboard` writes local user config.
|
|
28
49
|
3. Cockpit prints a dashboard pairing URL and code.
|
|
29
|
-
4.
|
|
30
|
-
|
|
50
|
+
4. Admin approves the pending request from Ambient -> Collector approvals, or
|
|
51
|
+
pastes the printed code there. The signed-in intern can still use the
|
|
52
|
+
printed URL.
|
|
31
53
|
5. The CLI starts general ambient capture, uploads private raw evidence objects
|
|
32
|
-
when present
|
|
54
|
+
when present (chunked and resumable, with identical content acknowledged
|
|
55
|
+
instead of re-uploaded), then uploads one safe metadata/ref envelope.
|
|
33
56
|
6. The CLI prints `PASS: Cockpit collector is ready for harvest.`
|
|
34
57
|
|
|
35
58
|
`--device-name` is just a readable label in Cockpit. It can be
|
|
@@ -68,17 +91,23 @@ Local files:
|
|
|
68
91
|
- `~/.config/bli-cockpit/config.json`: dashboard URL and local install config.
|
|
69
92
|
- `~/.config/bli-cockpit/session.json`: normal paired device session.
|
|
70
93
|
- `~/.local/state/bli-cockpit/spool/`: safe retry records when upload fails.
|
|
94
|
+
- `~/.local/state/bli-cockpit/cursors/raw-evidence.json`: hashes and labels of
|
|
95
|
+
already-durable evidence so repeated syncs skip re-uploading the same
|
|
96
|
+
content (no raw content is stored in the cursor).
|
|
71
97
|
- `.codex-autorunner/contextspace/active_context.md` in the work repo when a
|
|
72
98
|
work context is active.
|
|
73
99
|
|
|
74
100
|
Remote dashboard:
|
|
75
101
|
|
|
76
102
|
- approved user and device identity;
|
|
77
|
-
- repo, branch, optional ticket, source
|
|
78
|
-
timestamps;
|
|
103
|
+
- repo/worktree fingerprints, branch, head SHA, optional ticket, source
|
|
104
|
+
availability, risk flags, and upload timestamps;
|
|
79
105
|
- raw evidence refs accepted by `/api/ambient/ingest`;
|
|
80
|
-
- durable private Storage objects accepted by
|
|
81
|
-
`/api/ambient/evidence/upload
|
|
106
|
+
- durable private Storage objects accepted by the chunked
|
|
107
|
+
`/api/ambient/evidence/upload/begin|chunk|commit` endpoints, tracked in a
|
|
108
|
+
durable per-object upload ledger;
|
|
109
|
+
- Codex session attribution records (session id, file hash, attribution state
|
|
110
|
+
and reason labels, scores) accepted by `/api/ambient/codex-sessions`.
|
|
82
111
|
|
|
83
112
|
Never provide Supabase service-role keys, raw DB URLs, root env files, cookies,
|
|
84
113
|
or deployment tokens to this CLI. The collector must never read env files.
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
import { containsSecretLikeContent } from "@bli-cockpit/telemetry-core";
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { normalizeGitOrigin } from "../repo-identity.js";
|
|
6
|
+
/**
|
|
7
|
+
* Deterministic Codex session JSONL -> repo/worktree attribution.
|
|
8
|
+
*
|
|
9
|
+
* Only metadata-bearing records are inspected (session_meta, turn_context):
|
|
10
|
+
* cwd, workspace roots, and git branch/commit/origin. Prompt, response,
|
|
11
|
+
* reasoning, and tool payload fields are never extracted, printed, or
|
|
12
|
+
* summarized. A session is attributed only when exactly one worktree clearly
|
|
13
|
+
* wins; close calls stay ambiguous instead of being duplicated across repos.
|
|
14
|
+
*/
|
|
15
|
+
export const CODEX_ATTRIBUTION_DEFAULT_SINCE_MINUTES = 24 * 60;
|
|
16
|
+
export const CODEX_ATTRIBUTION_DEFAULT_SESSION_LIMIT = 50;
|
|
17
|
+
export const CODEX_SESSION_MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
18
|
+
const SECRET_FILE_SEGMENT_PATTERN = /(^|[/\\])(?:\.env(?:\..*)?|.*(?:secret|credential|private[_-]?key|service[_-]?role).*)$/i;
|
|
19
|
+
const SESSION_FILE_UUID_PATTERN = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i;
|
|
20
|
+
const SCORE_CWD_MATCH = 0.5;
|
|
21
|
+
const SCORE_WORKSPACE_ROOT_MATCH = 0.1;
|
|
22
|
+
const SCORE_ORIGIN_MATCH = 0.3;
|
|
23
|
+
const SCORE_BRANCH_MATCH = 0.15;
|
|
24
|
+
const SCORE_HEAD_SHA_MATCH = 0.05;
|
|
25
|
+
const ATTRIBUTION_MIN_SCORE = 0.4;
|
|
26
|
+
const ATTRIBUTION_MIN_MARGIN = 0.15;
|
|
27
|
+
export async function scanAndAttributeCodexSessions(options) {
|
|
28
|
+
const sinceMinutes = options.sinceMinutes ?? CODEX_ATTRIBUTION_DEFAULT_SINCE_MINUTES;
|
|
29
|
+
const limit = options.limit ?? CODEX_ATTRIBUTION_DEFAULT_SESSION_LIMIT;
|
|
30
|
+
const cutoffMs = options.now.getTime() - sinceMinutes * 60 * 1000;
|
|
31
|
+
const files = (await walkCodexJsonlFiles(options.sessionsDir, cutoffMs)).slice(0, limit);
|
|
32
|
+
const results = [];
|
|
33
|
+
for (const file of files) {
|
|
34
|
+
results.push(await attributeOneSession(file, options.worktrees));
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
results,
|
|
38
|
+
scanned_file_count: files.length,
|
|
39
|
+
counts: {
|
|
40
|
+
attributed: results.filter((entry) => entry.state === "attributed").length,
|
|
41
|
+
ambiguous: results.filter((entry) => entry.state === "ambiguous").length,
|
|
42
|
+
unattributed: results.filter((entry) => entry.state === "unattributed")
|
|
43
|
+
.length,
|
|
44
|
+
skipped: results.filter((entry) => entry.state === "skipped").length,
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
export async function walkCodexJsonlFiles(dir, cutoffMs) {
|
|
49
|
+
const out = [];
|
|
50
|
+
const stack = [dir];
|
|
51
|
+
while (stack.length > 0) {
|
|
52
|
+
const current = stack.pop();
|
|
53
|
+
if (!current || isSecretLikePath(current))
|
|
54
|
+
continue;
|
|
55
|
+
let entries;
|
|
56
|
+
try {
|
|
57
|
+
entries = await fs.readdir(current, { withFileTypes: true });
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
for (const entry of entries) {
|
|
63
|
+
const full = path.join(current, entry.name);
|
|
64
|
+
if (entry.isDirectory()) {
|
|
65
|
+
// Never descend into secret-like directories.
|
|
66
|
+
if (!isSecretLikePath(full))
|
|
67
|
+
stack.push(full);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
// Secret-like file NAMES stay in the list so attribution can record a
|
|
71
|
+
// "skipped" observation with a reason label (the content is never read)
|
|
72
|
+
// instead of silently dropping the session.
|
|
73
|
+
if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
|
|
74
|
+
continue;
|
|
75
|
+
const stat = await fs.stat(full);
|
|
76
|
+
if (stat.mtimeMs >= cutoffMs) {
|
|
77
|
+
out.push({ file: full, mtimeMs: stat.mtimeMs, byteSize: stat.size });
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
async function attributeOneSession(file, worktrees) {
|
|
85
|
+
const fileName = path.basename(file.file);
|
|
86
|
+
const base = {
|
|
87
|
+
file_path: file.file,
|
|
88
|
+
file_name: fileName,
|
|
89
|
+
codex_session_id: sessionIdFromFileName(fileName) ?? shortHash(file.file),
|
|
90
|
+
cwd_basename: null,
|
|
91
|
+
cwd_hash: null,
|
|
92
|
+
session_file_mtime: new Date(file.mtimeMs).toISOString(),
|
|
93
|
+
session_file_mtime_ms: file.mtimeMs,
|
|
94
|
+
byte_size: file.byteSize,
|
|
95
|
+
content_hash_sha256: null,
|
|
96
|
+
};
|
|
97
|
+
if (isSecretLikePath(fileName)) {
|
|
98
|
+
return skippedResult(base, "secret_like_file_name");
|
|
99
|
+
}
|
|
100
|
+
if (file.byteSize > CODEX_SESSION_MAX_FILE_BYTES) {
|
|
101
|
+
return skippedResult(base, "file_too_large");
|
|
102
|
+
}
|
|
103
|
+
let raw;
|
|
104
|
+
try {
|
|
105
|
+
raw = await fs.readFile(file.file);
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
return skippedResult(base, "file_read_failed");
|
|
109
|
+
}
|
|
110
|
+
const content = raw.toString("utf8");
|
|
111
|
+
base.content_hash_sha256 = sha256(raw);
|
|
112
|
+
base.byte_size = raw.byteLength;
|
|
113
|
+
if (containsSecretLikeContent(content)) {
|
|
114
|
+
return skippedResult(base, "secret_like_content_guard");
|
|
115
|
+
}
|
|
116
|
+
const signals = extractCodexSessionSignals(content);
|
|
117
|
+
const metaSessionId = sanitizeSessionId(signals.session_ids[0]);
|
|
118
|
+
if (metaSessionId) {
|
|
119
|
+
base.codex_session_id = metaSessionId;
|
|
120
|
+
}
|
|
121
|
+
const primaryCwd = signals.cwds[0] ?? null;
|
|
122
|
+
if (primaryCwd) {
|
|
123
|
+
base.cwd_basename = path.basename(primaryCwd) || null;
|
|
124
|
+
base.cwd_hash = shortHash(primaryCwd);
|
|
125
|
+
}
|
|
126
|
+
if (signals.line_count === 0 || signals.parse_error_count === signals.line_count) {
|
|
127
|
+
return {
|
|
128
|
+
...base,
|
|
129
|
+
state: "unattributed",
|
|
130
|
+
reason: "jsonl_parse_failed",
|
|
131
|
+
signals: [],
|
|
132
|
+
attribution_score: 0,
|
|
133
|
+
path_score: 0,
|
|
134
|
+
worktree: null,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
return scoreSessionAgainstWorktrees(base, signals, worktrees);
|
|
138
|
+
}
|
|
139
|
+
export function extractCodexSessionSignals(content) {
|
|
140
|
+
const sessionIds = new Set();
|
|
141
|
+
const cwds = new Set();
|
|
142
|
+
const workspaceRoots = new Set();
|
|
143
|
+
const branches = new Set();
|
|
144
|
+
const commitHashes = new Set();
|
|
145
|
+
const repositoryUrls = new Set();
|
|
146
|
+
let lineCount = 0;
|
|
147
|
+
let parseErrorCount = 0;
|
|
148
|
+
for (const line of content.split("\n")) {
|
|
149
|
+
if (!line.trim())
|
|
150
|
+
continue;
|
|
151
|
+
lineCount += 1;
|
|
152
|
+
let record;
|
|
153
|
+
try {
|
|
154
|
+
record = JSON.parse(line);
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
parseErrorCount += 1;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
if (!record || typeof record !== "object")
|
|
161
|
+
continue;
|
|
162
|
+
const type = record.type;
|
|
163
|
+
const payload = record.payload;
|
|
164
|
+
if (!payload || typeof payload !== "object")
|
|
165
|
+
continue;
|
|
166
|
+
const payloadRecord = payload;
|
|
167
|
+
if (type === "session_meta") {
|
|
168
|
+
addString(sessionIds, payloadRecord["id"]);
|
|
169
|
+
addString(cwds, payloadRecord["cwd"]);
|
|
170
|
+
const git = payloadRecord["git"];
|
|
171
|
+
if (git && typeof git === "object") {
|
|
172
|
+
const gitRecord = git;
|
|
173
|
+
addString(branches, gitRecord["branch"] ?? gitRecord["current_branch"]);
|
|
174
|
+
addString(commitHashes, gitRecord["commit_hash"]);
|
|
175
|
+
const repositoryUrl = gitRecord["repository_url"];
|
|
176
|
+
if (typeof repositoryUrl === "string" && repositoryUrl.trim()) {
|
|
177
|
+
repositoryUrls.add(normalizeGitOrigin(repositoryUrl));
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
else if (type === "turn_context") {
|
|
182
|
+
addString(cwds, payloadRecord["cwd"]);
|
|
183
|
+
const roots = payloadRecord["workspace_roots"];
|
|
184
|
+
if (Array.isArray(roots)) {
|
|
185
|
+
for (const root of roots) {
|
|
186
|
+
if (typeof root === "string") {
|
|
187
|
+
addString(workspaceRoots, root);
|
|
188
|
+
}
|
|
189
|
+
else if (root && typeof root === "object") {
|
|
190
|
+
addString(workspaceRoots, root["path"]);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
session_ids: [...sessionIds],
|
|
198
|
+
cwds: [...cwds],
|
|
199
|
+
workspace_roots: [...workspaceRoots],
|
|
200
|
+
branches: [...branches],
|
|
201
|
+
commit_hashes: [...commitHashes],
|
|
202
|
+
repository_urls: [...repositoryUrls],
|
|
203
|
+
line_count: lineCount,
|
|
204
|
+
parse_error_count: parseErrorCount,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
function scoreSessionAgainstWorktrees(base, signals, worktrees) {
|
|
208
|
+
const hasAnySignal = signals.cwds.length > 0 ||
|
|
209
|
+
signals.workspace_roots.length > 0 ||
|
|
210
|
+
signals.repository_urls.length > 0 ||
|
|
211
|
+
signals.branches.length > 0 ||
|
|
212
|
+
signals.commit_hashes.length > 0;
|
|
213
|
+
if (!hasAnySignal) {
|
|
214
|
+
return {
|
|
215
|
+
...base,
|
|
216
|
+
state: "unattributed",
|
|
217
|
+
reason: "no_repo_signals",
|
|
218
|
+
signals: [],
|
|
219
|
+
attribution_score: 0,
|
|
220
|
+
path_score: 0,
|
|
221
|
+
worktree: null,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
const scored = worktrees.map((worktree) => {
|
|
225
|
+
const matched = [];
|
|
226
|
+
let score = 0;
|
|
227
|
+
let pathScore = 0;
|
|
228
|
+
if (signals.cwds.some((cwd) => isPathWithin(cwd, worktree.repo_root))) {
|
|
229
|
+
score += SCORE_CWD_MATCH;
|
|
230
|
+
pathScore += SCORE_CWD_MATCH;
|
|
231
|
+
matched.push("cwd_match");
|
|
232
|
+
}
|
|
233
|
+
if (signals.workspace_roots.some((root) => isPathWithin(root, worktree.repo_root))) {
|
|
234
|
+
score += SCORE_WORKSPACE_ROOT_MATCH;
|
|
235
|
+
pathScore += SCORE_WORKSPACE_ROOT_MATCH;
|
|
236
|
+
matched.push("workspace_root_match");
|
|
237
|
+
}
|
|
238
|
+
if (worktree.repo_origin_url &&
|
|
239
|
+
signals.repository_urls.includes(worktree.repo_origin_url)) {
|
|
240
|
+
score += SCORE_ORIGIN_MATCH;
|
|
241
|
+
matched.push("origin_url_match");
|
|
242
|
+
}
|
|
243
|
+
if (signals.branches.includes(worktree.branch)) {
|
|
244
|
+
score += SCORE_BRANCH_MATCH;
|
|
245
|
+
matched.push("branch_match");
|
|
246
|
+
}
|
|
247
|
+
if (worktree.head_sha && signals.commit_hashes.includes(worktree.head_sha)) {
|
|
248
|
+
score += SCORE_HEAD_SHA_MATCH;
|
|
249
|
+
matched.push("head_sha_match");
|
|
250
|
+
}
|
|
251
|
+
return { worktree, score, pathScore, matched };
|
|
252
|
+
});
|
|
253
|
+
scored.sort((a, b) => b.score - a.score);
|
|
254
|
+
const best = scored[0];
|
|
255
|
+
const secondBestScore = scored[1]?.score ?? 0;
|
|
256
|
+
if (!best || best.score === 0) {
|
|
257
|
+
const reason = signals.cwds.length > 0 || signals.workspace_roots.length > 0
|
|
258
|
+
? "cwd_outside_scanned_worktrees"
|
|
259
|
+
: "no_matching_worktree_signals";
|
|
260
|
+
return {
|
|
261
|
+
...base,
|
|
262
|
+
state: "unattributed",
|
|
263
|
+
reason,
|
|
264
|
+
signals: [],
|
|
265
|
+
attribution_score: 0,
|
|
266
|
+
path_score: 0,
|
|
267
|
+
worktree: null,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
const score = clampScore(best.score);
|
|
271
|
+
const pathScore = clampScore(best.pathScore);
|
|
272
|
+
if (best.score >= ATTRIBUTION_MIN_SCORE &&
|
|
273
|
+
best.score - secondBestScore >= ATTRIBUTION_MIN_MARGIN) {
|
|
274
|
+
return {
|
|
275
|
+
...base,
|
|
276
|
+
state: "attributed",
|
|
277
|
+
reason: "deterministic_signal_match",
|
|
278
|
+
signals: best.matched,
|
|
279
|
+
attribution_score: score,
|
|
280
|
+
path_score: pathScore,
|
|
281
|
+
worktree: best.worktree,
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
return {
|
|
285
|
+
...base,
|
|
286
|
+
state: "ambiguous",
|
|
287
|
+
reason: best.score - secondBestScore < ATTRIBUTION_MIN_MARGIN
|
|
288
|
+
? "multiple_worktrees_close_scores"
|
|
289
|
+
: "signal_score_below_threshold",
|
|
290
|
+
signals: best.matched,
|
|
291
|
+
attribution_score: score,
|
|
292
|
+
path_score: pathScore,
|
|
293
|
+
worktree: null,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
function skippedResult(base, reason) {
|
|
297
|
+
return {
|
|
298
|
+
...base,
|
|
299
|
+
state: "skipped",
|
|
300
|
+
reason,
|
|
301
|
+
signals: [],
|
|
302
|
+
attribution_score: 0,
|
|
303
|
+
path_score: 0,
|
|
304
|
+
worktree: null,
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
export function sessionIdFromFileName(fileName) {
|
|
308
|
+
const match = fileName.match(SESSION_FILE_UUID_PATTERN);
|
|
309
|
+
return match?.[1]?.toLowerCase() ?? null;
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Session ids come from file content and end up inside remote object keys, so
|
|
313
|
+
* anything outside the safe charset falls back to a hash of the raw value
|
|
314
|
+
* instead of poisoning every begin request in the batch.
|
|
315
|
+
*/
|
|
316
|
+
export function sanitizeSessionId(value) {
|
|
317
|
+
if (!value)
|
|
318
|
+
return null;
|
|
319
|
+
if (/^[A-Za-z0-9._-]{4,80}$/.test(value))
|
|
320
|
+
return value;
|
|
321
|
+
return shortHash(value);
|
|
322
|
+
}
|
|
323
|
+
function isPathWithin(candidate, root) {
|
|
324
|
+
const normalizedCandidate = path.resolve(candidate);
|
|
325
|
+
const normalizedRoot = path.resolve(root);
|
|
326
|
+
return (normalizedCandidate === normalizedRoot ||
|
|
327
|
+
normalizedCandidate.startsWith(normalizedRoot + path.sep));
|
|
328
|
+
}
|
|
329
|
+
function addString(target, value) {
|
|
330
|
+
if (typeof value === "string" && value.trim())
|
|
331
|
+
target.add(value.trim());
|
|
332
|
+
}
|
|
333
|
+
function clampScore(value) {
|
|
334
|
+
return Math.min(1, Math.max(0, Number(value.toFixed(4))));
|
|
335
|
+
}
|
|
336
|
+
function isSecretLikePath(value) {
|
|
337
|
+
return SECRET_FILE_SEGMENT_PATTERN.test(value);
|
|
338
|
+
}
|
|
339
|
+
function shortHash(value) {
|
|
340
|
+
return crypto.createHash("sha256").update(value, "utf8").digest("hex").slice(0, 16);
|
|
341
|
+
}
|
|
342
|
+
function sha256(value) {
|
|
343
|
+
return crypto.createHash("sha256").update(value).digest("hex");
|
|
344
|
+
}
|
|
@@ -20,6 +20,9 @@ export async function runLocalSourceCollectors(options) {
|
|
|
20
20
|
stateDir: options.rawEvidenceStateDir,
|
|
21
21
|
repoRoot: options.repoRoot,
|
|
22
22
|
sessionsDir: options.rawEvidenceSessionsDir,
|
|
23
|
+
includeCodexJsonl: options.rawEvidenceIncludeCodexJsonl,
|
|
24
|
+
codexSessionFiles: options.rawEvidenceCodexSessionFiles,
|
|
25
|
+
skipContentHashes: options.rawEvidenceSkipContentHashes,
|
|
23
26
|
})
|
|
24
27
|
: {
|
|
25
28
|
scan: makeUnavailableScan(context, "codex_jsonl", "codex-jsonl", "raw_evidence_state_dir_not_configured"),
|
|
@@ -20,19 +20,26 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
20
20
|
const filesDir = path.join(evidenceDir, "files");
|
|
21
21
|
const entries = [];
|
|
22
22
|
const skipped = [];
|
|
23
|
+
const reused = [];
|
|
24
|
+
const skipContentHashes = options.skipContentHashes ?? new Set();
|
|
23
25
|
try {
|
|
24
26
|
await ensurePrivateDir(evidenceDir);
|
|
25
27
|
await ensurePrivateDir(filesDir);
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
if (options.includeCodexJsonl !== false) {
|
|
29
|
+
await collectCodexJsonlFiles({
|
|
30
|
+
context,
|
|
31
|
+
filesDir,
|
|
32
|
+
packId,
|
|
33
|
+
entries,
|
|
34
|
+
skipped,
|
|
35
|
+
reused,
|
|
36
|
+
skipContentHashes,
|
|
37
|
+
codexSessionFiles: options.codexSessionFiles,
|
|
38
|
+
sessionsDir: options.sessionsDir,
|
|
39
|
+
sinceMinutes: options.sinceMinutes ?? DEFAULT_SINCE_MINUTES,
|
|
40
|
+
limit: options.sessionLimit ?? DEFAULT_SESSION_LIMIT,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
36
43
|
await collectGitDiffFiles({
|
|
37
44
|
context,
|
|
38
45
|
filesDir,
|
|
@@ -40,6 +47,8 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
40
47
|
repoRoot: options.repoRoot,
|
|
41
48
|
entries,
|
|
42
49
|
skipped,
|
|
50
|
+
reused,
|
|
51
|
+
skipContentHashes,
|
|
43
52
|
});
|
|
44
53
|
if (entries.length === 0) {
|
|
45
54
|
const facts = {
|
|
@@ -50,9 +59,11 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
50
59
|
file_count: 0,
|
|
51
60
|
byte_size: 0,
|
|
52
61
|
skipped_count: skipped.length,
|
|
62
|
+
reused_count: reused.length,
|
|
53
63
|
content_kinds: [],
|
|
54
64
|
pointers: [],
|
|
55
65
|
upload_files: [],
|
|
66
|
+
reused,
|
|
56
67
|
};
|
|
57
68
|
return {
|
|
58
69
|
facts,
|
|
@@ -70,6 +81,7 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
70
81
|
evidenceDir,
|
|
71
82
|
entries,
|
|
72
83
|
skipped,
|
|
84
|
+
reused,
|
|
73
85
|
});
|
|
74
86
|
const manifestPath = path.join(evidenceDir, "manifest.json");
|
|
75
87
|
const manifestBytes = Buffer.from(`${JSON.stringify(manifestWithoutSelf, null, 2)}\n`, "utf8");
|
|
@@ -95,12 +107,16 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
95
107
|
file_count: entries.length,
|
|
96
108
|
byte_size: entries.reduce((sum, entry) => sum + entry.byte_size, 0),
|
|
97
109
|
skipped_count: skipped.length,
|
|
110
|
+
reused_count: reused.length,
|
|
98
111
|
content_kinds: [...new Set(entries.map((entry) => entry.kind))],
|
|
99
112
|
pointers: entries.map(pointerFromEntry),
|
|
100
113
|
upload_files: entries.map((entry) => ({
|
|
101
114
|
pointer: pointerFromEntry(entry),
|
|
102
115
|
local_path: entry.local_path,
|
|
116
|
+
kind: entry.kind,
|
|
117
|
+
codex_session_id: entry.codex_session_id ?? null,
|
|
103
118
|
})),
|
|
119
|
+
reused,
|
|
104
120
|
};
|
|
105
121
|
return {
|
|
106
122
|
facts,
|
|
@@ -153,17 +169,23 @@ function makeRawEvidenceScan(options) {
|
|
|
153
169
|
`files:${options.facts.file_count}`,
|
|
154
170
|
`bytes:${options.facts.byte_size}`,
|
|
155
171
|
`skipped:${options.facts.skipped_count}`,
|
|
172
|
+
`reused:${options.facts.reused_count}`,
|
|
156
173
|
...options.facts.content_kinds.map((kind) => `kind:${kind}`),
|
|
157
174
|
],
|
|
158
175
|
});
|
|
159
176
|
}
|
|
160
177
|
async function collectCodexJsonlFiles(options) {
|
|
161
|
-
const
|
|
162
|
-
|
|
163
|
-
|
|
178
|
+
const candidates = options.codexSessionFiles
|
|
179
|
+
? options.codexSessionFiles.map((file) => ({
|
|
180
|
+
filePath: file.local_path,
|
|
181
|
+
codexSessionId: file.codex_session_id,
|
|
182
|
+
}))
|
|
183
|
+
: (await walkJsonlFiles(options.sessionsDir ?? path.join(os.homedir(), ".codex", "sessions"), options.context.now.getTime() - options.sinceMinutes * 60 * 1000))
|
|
184
|
+
.slice(0, options.limit)
|
|
185
|
+
.map((filePath) => ({ filePath, codexSessionId: null }));
|
|
164
186
|
let index = 0;
|
|
165
|
-
for (const
|
|
166
|
-
const fileName = path.basename(filePath);
|
|
187
|
+
for (const candidate of candidates) {
|
|
188
|
+
const fileName = path.basename(candidate.filePath);
|
|
167
189
|
if (isSecretLikePath(fileName)) {
|
|
168
190
|
options.skipped.push({
|
|
169
191
|
kind: "codex_jsonl",
|
|
@@ -172,7 +194,18 @@ async function collectCodexJsonlFiles(options) {
|
|
|
172
194
|
});
|
|
173
195
|
continue;
|
|
174
196
|
}
|
|
175
|
-
|
|
197
|
+
let raw;
|
|
198
|
+
try {
|
|
199
|
+
raw = await fs.readFile(candidate.filePath);
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
options.skipped.push({
|
|
203
|
+
kind: "codex_jsonl",
|
|
204
|
+
label: fileName,
|
|
205
|
+
reason: "file_read_failed",
|
|
206
|
+
});
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
176
209
|
if (containsSecretLikeContent(raw.toString("utf8"))) {
|
|
177
210
|
options.skipped.push({
|
|
178
211
|
kind: "codex_jsonl",
|
|
@@ -181,8 +214,19 @@ async function collectCodexJsonlFiles(options) {
|
|
|
181
214
|
});
|
|
182
215
|
continue;
|
|
183
216
|
}
|
|
217
|
+
const contentHash = sha256(raw);
|
|
218
|
+
const codexSessionId = candidate.codexSessionId ?? shortHash(candidate.filePath);
|
|
219
|
+
if (options.skipContentHashes.has(contentHash)) {
|
|
220
|
+
options.reused.push({
|
|
221
|
+
kind: "codex_jsonl",
|
|
222
|
+
label: fileName,
|
|
223
|
+
content_hash_sha256: contentHash,
|
|
224
|
+
codex_session_id: codexSessionId,
|
|
225
|
+
});
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
184
228
|
index += 1;
|
|
185
|
-
const relativePath = path.join("files", `${String(index).padStart(3, "0")}-${shortHash(filePath)}-${fileName}`);
|
|
229
|
+
const relativePath = path.join("files", `${String(index).padStart(3, "0")}-${shortHash(candidate.filePath)}-${fileName}`);
|
|
186
230
|
const destination = path.join(options.filesDir, path.basename(relativePath));
|
|
187
231
|
await fs.writeFile(destination, raw, { mode: 0o600 });
|
|
188
232
|
await chmodPrivate(destination, 0o600);
|
|
@@ -196,6 +240,8 @@ async function collectCodexJsonlFiles(options) {
|
|
|
196
240
|
mediaType: "application/jsonl",
|
|
197
241
|
redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
198
242
|
bytes: raw,
|
|
243
|
+
codexSessionId,
|
|
244
|
+
contentAddress: `codex/${safeKeySegment(codexSessionId)}/${contentHash.slice(0, 16)}.jsonl`,
|
|
199
245
|
}));
|
|
200
246
|
}
|
|
201
247
|
}
|
|
@@ -217,6 +263,16 @@ async function collectGitDiffFiles(options) {
|
|
|
217
263
|
continue;
|
|
218
264
|
}
|
|
219
265
|
const raw = Buffer.from(diff.slice(0, MAX_GIT_DIFF_BYTES), "utf8");
|
|
266
|
+
const contentHash = sha256(raw);
|
|
267
|
+
if (options.skipContentHashes.has(contentHash)) {
|
|
268
|
+
options.reused.push({
|
|
269
|
+
kind: "git_diff",
|
|
270
|
+
label: target.label,
|
|
271
|
+
content_hash_sha256: contentHash,
|
|
272
|
+
codex_session_id: null,
|
|
273
|
+
});
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
220
276
|
const relativePath = path.join("files", `git-${target.label}.diff`);
|
|
221
277
|
const destination = path.join(options.filesDir, path.basename(relativePath));
|
|
222
278
|
await fs.writeFile(destination, raw, { mode: 0o600 });
|
|
@@ -231,6 +287,7 @@ async function collectGitDiffFiles(options) {
|
|
|
231
287
|
mediaType: "text/x-diff",
|
|
232
288
|
redactedSummary: `Raw git ${target.label} diff preserved locally with env/secret paths excluded.`,
|
|
233
289
|
bytes: raw,
|
|
290
|
+
contentAddress: `git-diff/${target.label}-${contentHash.slice(0, 16)}.diff`,
|
|
234
291
|
}));
|
|
235
292
|
}
|
|
236
293
|
}
|
|
@@ -308,6 +365,7 @@ function makeManifest(options) {
|
|
|
308
365
|
},
|
|
309
366
|
files: options.entries.map(redactManifestEntry),
|
|
310
367
|
skipped: options.skipped,
|
|
368
|
+
reused: options.reused,
|
|
311
369
|
};
|
|
312
370
|
}
|
|
313
371
|
function evidenceEntry(options) {
|
|
@@ -321,11 +379,13 @@ function evidenceEntry(options) {
|
|
|
321
379
|
workContextId: options.workContextId,
|
|
322
380
|
packId: options.packId,
|
|
323
381
|
relativePath: options.relativePath,
|
|
382
|
+
contentAddress: options.contentAddress,
|
|
324
383
|
}),
|
|
325
384
|
content_hash_sha256: digest,
|
|
326
385
|
byte_size: options.bytes.byteLength,
|
|
327
386
|
media_type: options.mediaType,
|
|
328
387
|
redacted_summary: options.redactedSummary,
|
|
388
|
+
codex_session_id: options.codexSessionId ?? null,
|
|
329
389
|
};
|
|
330
390
|
}
|
|
331
391
|
function redactManifestEntry(entry) {
|
|
@@ -356,7 +416,20 @@ function pointerFromEntry(entry) {
|
|
|
356
416
|
redacted_summary: entry.redacted_summary,
|
|
357
417
|
};
|
|
358
418
|
}
|
|
419
|
+
/**
|
|
420
|
+
* Codex JSONL and git diff objects are content-addressed (kind/id/hash) so the
|
|
421
|
+
* same content maps to the same remote key across syncs: interrupted uploads
|
|
422
|
+
* resume and repeated syncs dedupe server-side. Pack-scoped keys remain for
|
|
423
|
+
* the per-sync manifest.
|
|
424
|
+
*/
|
|
359
425
|
function remoteObjectKey(options) {
|
|
426
|
+
if (options.contentAddress) {
|
|
427
|
+
return posixPath([
|
|
428
|
+
options.operatorId,
|
|
429
|
+
options.workContextId,
|
|
430
|
+
options.contentAddress,
|
|
431
|
+
]);
|
|
432
|
+
}
|
|
360
433
|
return posixPath([
|
|
361
434
|
options.operatorId,
|
|
362
435
|
options.workContextId,
|
|
@@ -378,6 +451,13 @@ function rawEvidencePackId(context) {
|
|
|
378
451
|
function shortHash(value) {
|
|
379
452
|
return crypto.createHash("sha256").update(value, "utf8").digest("hex").slice(0, 12);
|
|
380
453
|
}
|
|
454
|
+
/**
|
|
455
|
+
* Object keys must satisfy the server's key pattern; ids derived from file
|
|
456
|
+
* content fall back to a hash rather than failing the whole upload batch.
|
|
457
|
+
*/
|
|
458
|
+
function safeKeySegment(value) {
|
|
459
|
+
return /^[A-Za-z0-9._-]{1,80}$/.test(value) ? value : shortHash(value);
|
|
460
|
+
}
|
|
381
461
|
function sha256(value) {
|
|
382
462
|
return crypto.createHash("sha256").update(value).digest("hex");
|
|
383
463
|
}
|