@bli-cockpit/cli 0.2.48 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-attribution-gaps.js +133 -0
- package/dist/adapters/raw-evidence-claude-reader.js +108 -0
- package/dist/adapters/raw-evidence-codex-reader.js +147 -0
- package/dist/adapters/raw-evidence-collection-state.js +199 -0
- package/dist/adapters/raw-evidence-facts.js +338 -0
- package/dist/adapters/raw-evidence-git-diff-reader.js +187 -0
- package/dist/adapters/raw-evidence-image-reader.js +107 -0
- package/dist/adapters/raw-evidence-sanitize.js +56 -0
- package/dist/adapters/raw-evidence-transcript-file.js +182 -0
- package/dist/adapters/raw-evidence.js +94 -1203
- package/dist/autostart-contract.js +79 -0
- package/dist/autostart-darwin-plist.js +265 -0
- package/dist/autostart-darwin.js +171 -0
- package/dist/autostart-windows-scripts.js +310 -0
- package/dist/autostart-windows-task-xml.js +260 -0
- package/dist/autostart-windows.js +237 -0
- package/dist/autostart-xml.js +23 -0
- package/dist/autostart.js +35 -1148
- package/dist/commands/agent-rules-command.js +55 -0
- package/dist/commands/agent-session-report.js +290 -0
- package/dist/commands/analyze.js +131 -0
- package/dist/commands/autostart-command.js +105 -0
- package/dist/commands/backfill-batches.js +34 -0
- package/dist/commands/backfill-candidates.js +54 -0
- package/dist/commands/backfill-checkpoint.js +101 -0
- package/dist/commands/backfill-command-line.js +70 -0
- package/dist/commands/backfill-evidence-outcomes.js +104 -0
- package/dist/commands/backfill-issues.js +265 -0
- package/dist/commands/backfill-output.js +75 -0
- package/dist/commands/backfill-plan.js +71 -0
- package/dist/commands/backfill-reasons.js +107 -0
- package/dist/commands/backfill-report.js +298 -0
- package/dist/commands/backfill-result.js +150 -0
- package/dist/commands/backfill-scan.js +274 -0
- package/dist/commands/backfill-scope.js +114 -0
- package/dist/commands/backfill-session-report.js +145 -0
- package/dist/commands/backfill-types.js +1 -0
- package/dist/commands/backfill-upload.js +212 -0
- package/dist/commands/backfill.js +58 -1705
- package/dist/commands/cli-io.js +13 -0
- package/dist/commands/doctor.js +57 -0
- package/dist/commands/jarvis-trace.js +184 -0
- package/dist/commands/jarvis.js +323 -7
- package/dist/commands/local-arg-values.js +169 -0
- package/dist/commands/local-args-collector.js +604 -0
- package/dist/commands/local-args-tower.js +891 -0
- package/dist/commands/local-args.js +10 -1549
- package/dist/commands/local-help.js +30 -5
- package/dist/commands/local.js +21 -1786
- package/dist/commands/login.js +53 -0
- package/dist/commands/logout.js +66 -0
- package/dist/commands/memory-install-claude.js +294 -0
- package/dist/commands/memory-install-codex.js +205 -0
- package/dist/commands/memory-install-contract.js +231 -0
- package/dist/commands/memory-install-files.js +63 -0
- package/dist/commands/memory-install-skills.js +121 -0
- package/dist/commands/memory-install-toml.js +265 -0
- package/dist/commands/memory-install.js +378 -0
- package/dist/commands/onboard-receipts.js +66 -0
- package/dist/commands/onboard-report.js +274 -0
- package/dist/commands/onboard.js +449 -0
- package/dist/commands/ops-render.js +36 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/serve.js +13 -0
- package/dist/commands/session-sync.js +513 -534
- package/dist/commands/settings-render.js +28 -0
- package/dist/commands/settings.js +66 -2
- package/dist/commands/start.js +47 -0
- package/dist/commands/sync-followups.js +308 -0
- package/dist/commands/sync.js +387 -0
- package/dist/local-state-attributed-target.js +75 -0
- package/dist/local-state-config.js +147 -0
- package/dist/local-state-files.js +59 -0
- package/dist/local-state-identity.js +73 -0
- package/dist/local-state-pairing.js +263 -0
- package/dist/local-state-paths.js +61 -0
- package/dist/local-state-session.js +68 -0
- package/dist/local-state-status.js +163 -0
- package/dist/local-state-work-context.js +190 -0
- package/dist/local-state.js +34 -848
- package/dist/tower-client.js +3 -2
- package/dist/tower-stream.js +76 -6
- package/package.json +2 -1
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-policy.js";
|
|
2
|
+
import { recordScanned, recordSkipCount, recordTruncationCount, } from "./raw-evidence-completeness.js";
|
|
3
|
+
/**
|
|
4
|
+
* Record every Codex session the attribution scan saw, capped or lost, so the
|
|
5
|
+
* pass reports them as its own gaps.
|
|
6
|
+
*/
|
|
7
|
+
export function recordCodexAttributionCompleteness(ledger, scan, selectedPaths) {
|
|
8
|
+
recordScanned(ledger, "codex_attribution", scan.scanned_file_count);
|
|
9
|
+
ledger.caps.push(...codexAttributionCaps(scan));
|
|
10
|
+
recordSkipCount(ledger, "codex_attribution", "session_limit_overflow", Math.max(0, scan.discovered_file_count - scan.scanned_file_count));
|
|
11
|
+
recordSkipCount(ledger, "codex_attribution", "directory_read_failed", scan.directory_read_failed_count);
|
|
12
|
+
recordSkipCount(ledger, "codex_attribution", "file_stat_failed", scan.stat_failed_count);
|
|
13
|
+
recordSkipCount(ledger, "codex_attribution", "secret_like_directory", scan.secret_path_skipped_count);
|
|
14
|
+
recordAttributionResultSkips(ledger, "codex_attribution", scan.results, selectedPaths);
|
|
15
|
+
}
|
|
16
|
+
/** Every limit the Codex attribution scan ran under, and whether it bit. */
|
|
17
|
+
function codexAttributionCaps(scan) {
|
|
18
|
+
return [
|
|
19
|
+
{
|
|
20
|
+
source: "codex_attribution",
|
|
21
|
+
cap_type: "scan_window_minutes",
|
|
22
|
+
limit: scan.since_minutes,
|
|
23
|
+
observed: scan.since_minutes,
|
|
24
|
+
applied: false,
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
source: "codex_attribution",
|
|
28
|
+
cap_type: "session_limit",
|
|
29
|
+
limit: scan.session_limit,
|
|
30
|
+
observed: scan.discovered_file_count,
|
|
31
|
+
applied: scan.session_limit_applied,
|
|
32
|
+
},
|
|
33
|
+
];
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Record every Claude session the attribution scan saw, capped or lost —
|
|
37
|
+
* sidecars included, because a subagent transcript nobody could read is missing
|
|
38
|
+
* evidence exactly like a main.
|
|
39
|
+
*/
|
|
40
|
+
export function recordClaudeAttributionCompleteness(ledger, scan, selectedPaths) {
|
|
41
|
+
recordScanned(ledger, "claude_attribution", scan.scanned_session_count);
|
|
42
|
+
if (scan.disabled_reason) {
|
|
43
|
+
recordSkipCount(ledger, "claude_attribution", scan.disabled_reason, 1);
|
|
44
|
+
}
|
|
45
|
+
ledger.caps.push(...claudeAttributionCaps(scan));
|
|
46
|
+
recordClaudeAttributionSkips(ledger, scan);
|
|
47
|
+
recordTruncationCount(ledger, "claude_attribution", "oversized_jsonl_line", scan.counts.oversized_lines_skipped, { max_bytes: scan.max_line_buffer_bytes });
|
|
48
|
+
recordAttributionResultSkips(ledger, "claude_attribution", scan.results, selectedPaths);
|
|
49
|
+
recordClaudeSidecarSkips(ledger, scan);
|
|
50
|
+
}
|
|
51
|
+
/** Every limit the Claude attribution scan ran under, and whether it bit. */
|
|
52
|
+
function claudeAttributionCaps(scan) {
|
|
53
|
+
return [
|
|
54
|
+
{
|
|
55
|
+
source: "claude_attribution",
|
|
56
|
+
cap_type: "scan_window_minutes",
|
|
57
|
+
limit: scan.since_minutes,
|
|
58
|
+
observed: scan.since_minutes,
|
|
59
|
+
applied: false,
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
source: "claude_attribution",
|
|
63
|
+
cap_type: "session_limit",
|
|
64
|
+
limit: scan.session_limit,
|
|
65
|
+
observed: scan.discovered_session_count,
|
|
66
|
+
applied: scan.session_limit_applied,
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
source: "claude_attribution",
|
|
70
|
+
cap_type: "max_file_bytes",
|
|
71
|
+
limit: scan.max_file_bytes,
|
|
72
|
+
applied: scan.counts.mains_oversized > 0 ||
|
|
73
|
+
scan.results.some((result) => result.reason === "file_too_large"),
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
source: "claude_attribution",
|
|
77
|
+
cap_type: "max_sidecar_files",
|
|
78
|
+
limit: scan.max_sidecar_files,
|
|
79
|
+
observed: scan.max_sidecar_files + scan.counts.sidecars_capped,
|
|
80
|
+
applied: scan.counts.sidecars_capped > 0,
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
source: "claude_attribution",
|
|
84
|
+
cap_type: "max_line_buffer_bytes",
|
|
85
|
+
limit: scan.max_line_buffer_bytes,
|
|
86
|
+
applied: scan.counts.oversized_lines_skipped > 0,
|
|
87
|
+
},
|
|
88
|
+
];
|
|
89
|
+
}
|
|
90
|
+
/** Each way the Claude scan lost whole sessions, one named skip count apiece. */
|
|
91
|
+
function recordClaudeAttributionSkips(ledger, scan) {
|
|
92
|
+
recordSkipCount(ledger, "claude_attribution", "session_limit_overflow", Math.max(0, scan.discovered_session_count - scan.scanned_session_count));
|
|
93
|
+
recordSkipCount(ledger, "claude_attribution", "secret_like_project_dir", scan.project_dirs_skipped);
|
|
94
|
+
recordSkipCount(ledger, "claude_attribution", "project_dir_read_failed", scan.project_dir_read_failed_count);
|
|
95
|
+
recordSkipCount(ledger, "claude_attribution", "session_stat_failed", scan.session_stat_failed_count);
|
|
96
|
+
recordSkipCount(ledger, "claude_attribution", "sidecar_dir_read_failed", scan.sidecar_dir_read_failed_count);
|
|
97
|
+
recordSkipCount(ledger, "claude_attribution", "sidecar_stat_failed", scan.sidecar_stat_failed_count);
|
|
98
|
+
recordSkipCount(ledger, "claude_attribution", "sidecar_limit_overflow", scan.counts.sidecars_capped);
|
|
99
|
+
}
|
|
100
|
+
/** One skip per sidecar the scan declined, labelled with its own reason. */
|
|
101
|
+
function recordClaudeSidecarSkips(ledger, scan) {
|
|
102
|
+
for (const result of scan.results) {
|
|
103
|
+
for (const sidecar of result.sidecar_files) {
|
|
104
|
+
if (!sidecar.skipped_reason)
|
|
105
|
+
continue;
|
|
106
|
+
recordSkipCount(ledger, "claude_attribution", `sidecar_${sidecar.skipped_reason}`, 1);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
/** One skip per scanned session this pass is neither collecting nor excusing. */
|
|
111
|
+
function recordAttributionResultSkips(ledger, source, results, selectedPaths) {
|
|
112
|
+
for (const result of results) {
|
|
113
|
+
if (isAttributionAccountedFor(result, selectedPaths))
|
|
114
|
+
continue;
|
|
115
|
+
const reason = result.state === "skipped"
|
|
116
|
+
? result.reason
|
|
117
|
+
: `attribution_${result.state}:${result.reason}`;
|
|
118
|
+
recordSkipCount(ledger, source, reason, 1);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Not every unattributed result is a gap. A selected session is being collected
|
|
123
|
+
* by this pass, and a live-sync-safe synthetic target is owned by another
|
|
124
|
+
* workspace's pack — neither is missing evidence.
|
|
125
|
+
*/
|
|
126
|
+
function isAttributionAccountedFor(result, selectedPaths) {
|
|
127
|
+
if (result.state === "attributed")
|
|
128
|
+
return true;
|
|
129
|
+
if (selectedPaths.has(result.file_path))
|
|
130
|
+
return true;
|
|
131
|
+
return (result.worktree !== null &&
|
|
132
|
+
isLiveRawEvidenceSyncAttribution(result.state, result.reason, true));
|
|
133
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claude Code sessions: one main transcript, its subagent sidecars, and the
|
|
3
|
+
* images either of them attached.
|
|
4
|
+
*
|
|
5
|
+
* Sync always attributes first, so there is no directory-walk fallback here —
|
|
6
|
+
* the caller either names the sessions or Claude is not collected this pass.
|
|
7
|
+
*
|
|
8
|
+
* The main file has three named outcomes and the sidecars are independent of
|
|
9
|
+
* all of them: an oversized main (D7) is skipped for its bytes while its
|
|
10
|
+
* sidecars still collect, and a growth-damped main (D9) is not re-collected at
|
|
11
|
+
* all while the session still reports the durable copy it already has.
|
|
12
|
+
*/
|
|
13
|
+
import path from "node:path";
|
|
14
|
+
import { isSecretLikePath, safeKeySegment } from "./raw-evidence-keys.js";
|
|
15
|
+
import { recordScanned } from "./raw-evidence-completeness.js";
|
|
16
|
+
import { CLAUDE_MAX_COLLECT_FILE_BYTES, } from "./raw-evidence-collection-state.js";
|
|
17
|
+
import { collectOneEvidenceFile } from "./raw-evidence-transcript-file.js";
|
|
18
|
+
import { collectAgentImagesFromTranscript } from "./raw-evidence-image-reader.js";
|
|
19
|
+
export async function collectClaudeJsonlFiles(collection, sessions) {
|
|
20
|
+
recordScanned(collection, "claude_jsonl", sessions.reduce((count, session) => count + 1 + session.sidecar_files.length, 0));
|
|
21
|
+
for (const session of sessions) {
|
|
22
|
+
await collectOneClaudeSession(collection, session);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
async function collectOneClaudeSession(collection, session) {
|
|
26
|
+
const sessionId = session.claude_session_id;
|
|
27
|
+
const mainOutcome = await collectClaudeMainFile(collection, session);
|
|
28
|
+
if (mainOutcome === "collected" || mainOutcome === "damped_reuse") {
|
|
29
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
30
|
+
filePath: session.local_path,
|
|
31
|
+
source: "claude_code",
|
|
32
|
+
sessionId,
|
|
33
|
+
kind: "claude_image_attachment",
|
|
34
|
+
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/images/${hash16}.${extension}`,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
for (const sidecar of session.sidecar_files) {
|
|
38
|
+
await collectOneClaudeSidecar(collection, session, sidecar.local_path);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* What happens to a Claude main file, as a decision table:
|
|
43
|
+
*
|
|
44
|
+
* | condition | main bytes | outcome |
|
|
45
|
+
* | --------------------- | ------------------------- | ------------------- |
|
|
46
|
+
* | `main_file_oversized` | skipped `file_too_large` | `skipped_too_large` |
|
|
47
|
+
* | `skip_main` (D9) | not re-collected | `damped_reuse` |
|
|
48
|
+
* | otherwise | collected, or named skip | `collected` / `not_collected` |
|
|
49
|
+
*
|
|
50
|
+
* Its images are collected for every outcome except `skipped_too_large` and
|
|
51
|
+
* `not_collected` — the caller decides that, this function only reports.
|
|
52
|
+
*
|
|
53
|
+
* D7: an oversized main was attributed via a streamed read but its bytes are
|
|
54
|
+
* never uploaded (server commit assembles in memory). Its sidecars still
|
|
55
|
+
* collect. D9 damped: the prior durable copy is still good enough, and the
|
|
56
|
+
* session reports `reused_existing` from cursor state, so no skip is recorded.
|
|
57
|
+
*/
|
|
58
|
+
async function collectClaudeMainFile(collection, session) {
|
|
59
|
+
if (session.main_file_oversized) {
|
|
60
|
+
collection.skipped.push({
|
|
61
|
+
kind: "claude_jsonl",
|
|
62
|
+
label: path.basename(session.local_path),
|
|
63
|
+
reason: "file_too_large",
|
|
64
|
+
});
|
|
65
|
+
return "skipped_too_large";
|
|
66
|
+
}
|
|
67
|
+
if (session.skip_main)
|
|
68
|
+
return "damped_reuse";
|
|
69
|
+
const accepted = await collectOneEvidenceFile(collection, {
|
|
70
|
+
filePath: session.local_path,
|
|
71
|
+
kind: "claude_jsonl",
|
|
72
|
+
sessionId: session.claude_session_id,
|
|
73
|
+
mediaType: "application/jsonl",
|
|
74
|
+
// Re-check size at collection: a main that grew past the cap between
|
|
75
|
+
// attribution and collection is an honest file_too_large skip, not an
|
|
76
|
+
// upload_failed at the chunk client.
|
|
77
|
+
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
78
|
+
redactedSummary: "Raw Claude Code JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
79
|
+
contentAddress: (hash16) => `claude/${safeKeySegment(session.claude_session_id)}/${hash16}.jsonl`,
|
|
80
|
+
});
|
|
81
|
+
return accepted ? "collected" : "not_collected";
|
|
82
|
+
}
|
|
83
|
+
async function collectOneClaudeSidecar(collection, session, sidecarPath) {
|
|
84
|
+
const sessionId = session.claude_session_id;
|
|
85
|
+
const stem = path.basename(sidecarPath).replace(/\.jsonl$/i, "");
|
|
86
|
+
const safeSidecarId = isSecretLikePath(stem)
|
|
87
|
+
? "redacted-file-name"
|
|
88
|
+
: safeKeySegment(stem);
|
|
89
|
+
const sidecarAccepted = await collectOneEvidenceFile(collection, {
|
|
90
|
+
filePath: sidecarPath,
|
|
91
|
+
kind: "claude_jsonl_sidecar",
|
|
92
|
+
sessionId,
|
|
93
|
+
mediaType: "application/jsonl",
|
|
94
|
+
maxFileBytes: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
95
|
+
redactedSummary: "Raw Claude Code subagent transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
96
|
+
contentAddress: (hash16) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}-${hash16}.jsonl`,
|
|
97
|
+
});
|
|
98
|
+
if (!sidecarAccepted)
|
|
99
|
+
return;
|
|
100
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
101
|
+
filePath: sidecarPath,
|
|
102
|
+
source: "claude_code",
|
|
103
|
+
sessionId,
|
|
104
|
+
sidecarId: safeSidecarId,
|
|
105
|
+
kind: "claude_image_attachment",
|
|
106
|
+
contentAddress: (hash16, extension) => `claude/${safeKeySegment(sessionId)}/subagents/${safeSidecarId}/images/${hash16}.${extension}`,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which Codex transcripts this pass reads, and the walk it falls back to.
|
|
3
|
+
*
|
|
4
|
+
* Attribution normally hands over an exact list, and that list is never
|
|
5
|
+
* trimmed: the session limit guards only the fallback walk, which can turn up
|
|
6
|
+
* every Codex session on the machine and would otherwise let one busy laptop
|
|
7
|
+
* spend the whole sync. Either way the cap is declared, observed and recorded,
|
|
8
|
+
* so a trimmed pass says how many it left out.
|
|
9
|
+
*
|
|
10
|
+
* A session is a transcript first and its attached images second, never the
|
|
11
|
+
* reverse — images are only collected once the transcript itself was accepted.
|
|
12
|
+
*/
|
|
13
|
+
import fs from "node:fs/promises";
|
|
14
|
+
import os from "node:os";
|
|
15
|
+
import path from "node:path";
|
|
16
|
+
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
17
|
+
import { describeError } from "../health-detail.js";
|
|
18
|
+
import { defaultCodexSessionDirs } from "./codex-attribution.js";
|
|
19
|
+
import { isSecretLikePath, safeKeySegment, shortHash, } from "./raw-evidence-keys.js";
|
|
20
|
+
import { recordScanned } from "./raw-evidence-completeness.js";
|
|
21
|
+
import { collectOneEvidenceFile } from "./raw-evidence-transcript-file.js";
|
|
22
|
+
import { collectAgentImagesFromTranscript } from "./raw-evidence-image-reader.js";
|
|
23
|
+
export async function collectCodexJsonlFiles(collection, options) {
|
|
24
|
+
for (const candidate of await chooseCodexCandidates(collection, options)) {
|
|
25
|
+
recordScanned(collection, "codex_jsonl");
|
|
26
|
+
await collectOneCodexSession(collection, candidate);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Which Codex transcripts this pass will look at, and the record of how many it
|
|
31
|
+
* had to leave out. Attribution's own list is never trimmed — the session limit
|
|
32
|
+
* only guards the fallback walk, which can turn up every session on the machine.
|
|
33
|
+
*/
|
|
34
|
+
async function chooseCodexCandidates(collection, options) {
|
|
35
|
+
const attributed = Boolean(options.codexSessionFiles);
|
|
36
|
+
const candidates = options.codexSessionFiles
|
|
37
|
+
? options.codexSessionFiles.map((file) => ({
|
|
38
|
+
filePath: file.local_path,
|
|
39
|
+
codexSessionId: file.codex_session_id,
|
|
40
|
+
}))
|
|
41
|
+
: (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
|
|
42
|
+
collection.caps.push({
|
|
43
|
+
source: "codex_jsonl",
|
|
44
|
+
cap_type: "session_limit",
|
|
45
|
+
limit: options.limit,
|
|
46
|
+
observed: candidates.length,
|
|
47
|
+
applied: !attributed && candidates.length > options.limit,
|
|
48
|
+
});
|
|
49
|
+
return attributed ? candidates : candidates.slice(0, options.limit);
|
|
50
|
+
}
|
|
51
|
+
/** The transcript, then the images it explicitly attached — never the reverse. */
|
|
52
|
+
async function collectOneCodexSession(collection, candidate) {
|
|
53
|
+
const codexSessionId = candidate.codexSessionId ?? shortHash(candidate.filePath);
|
|
54
|
+
const transcriptAccepted = await collectOneEvidenceFile(collection, {
|
|
55
|
+
filePath: candidate.filePath,
|
|
56
|
+
kind: "codex_jsonl",
|
|
57
|
+
sessionId: codexSessionId,
|
|
58
|
+
mediaType: "application/jsonl",
|
|
59
|
+
// Guard before read/toString: Codex files can exceed the buffered commit
|
|
60
|
+
// ceiling just like Claude mains.
|
|
61
|
+
maxFileBytes: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
62
|
+
redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
63
|
+
contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
|
|
64
|
+
});
|
|
65
|
+
if (!transcriptAccepted)
|
|
66
|
+
return;
|
|
67
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
68
|
+
filePath: candidate.filePath,
|
|
69
|
+
source: "codex",
|
|
70
|
+
sessionId: codexSessionId,
|
|
71
|
+
kind: "codex_image_attachment",
|
|
72
|
+
contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
/** The unattributed fallback: every recent Codex transcript on this machine. */
|
|
76
|
+
function walkRecentCodexJsonlFiles(collection, options) {
|
|
77
|
+
const dirs = options.sessionsDirs ??
|
|
78
|
+
(options.sessionsDir
|
|
79
|
+
? [options.sessionsDir]
|
|
80
|
+
: defaultCodexSessionDirs(os.homedir()));
|
|
81
|
+
return walkJsonlFiles(dirs, collection.context.now.getTime() - options.sinceMinutes * 60 * 1000);
|
|
82
|
+
}
|
|
83
|
+
// ---------------------------------------------------------------------------
|
|
84
|
+
// Walking the session store
|
|
85
|
+
// ---------------------------------------------------------------------------
|
|
86
|
+
/**
|
|
87
|
+
* Newest-first `.jsonl` files under these directories, modified since `cutoffMs`.
|
|
88
|
+
*
|
|
89
|
+
* Secret-like directory and file names are never descended into or opened;
|
|
90
|
+
* symlinked duplicates are collapsed by real path so one transcript reachable
|
|
91
|
+
* two ways is collected once.
|
|
92
|
+
*/
|
|
93
|
+
async function walkJsonlFiles(dir, cutoffMs) {
|
|
94
|
+
const out = [];
|
|
95
|
+
const stack = Array.isArray(dir) ? [...dir] : [dir];
|
|
96
|
+
const seen = new Set();
|
|
97
|
+
// Counted rather than logged per directory: a wide walk can hit many, and
|
|
98
|
+
// the useful signal is "N directories in the session store were skipped and
|
|
99
|
+
// here is the first reason", not N near-identical lines (BLI-3238).
|
|
100
|
+
let unreadableDirCount = 0;
|
|
101
|
+
let firstUnreadableDir = null;
|
|
102
|
+
while (stack.length > 0) {
|
|
103
|
+
const current = stack.pop();
|
|
104
|
+
if (!current || isSecretLikePath(current))
|
|
105
|
+
continue;
|
|
106
|
+
let entries;
|
|
107
|
+
try {
|
|
108
|
+
entries = await fs.readdir(current, { withFileTypes: true });
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
// A directory that cannot be listed hides every session under it, and
|
|
112
|
+
// the walk's only visible effect is a smaller file count.
|
|
113
|
+
unreadableDirCount += 1;
|
|
114
|
+
firstUnreadableDir ??= describeError(error);
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
for (const entry of entries) {
|
|
118
|
+
const full = path.join(current, entry.name);
|
|
119
|
+
if (isSecretLikePath(full))
|
|
120
|
+
continue;
|
|
121
|
+
if (entry.isDirectory()) {
|
|
122
|
+
stack.push(full);
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
|
|
126
|
+
continue;
|
|
127
|
+
const stat = await fs.stat(full);
|
|
128
|
+
if (stat.mtimeMs < cutoffMs)
|
|
129
|
+
continue;
|
|
130
|
+
const dedupeKey = await fs.realpath(full).catch(() => path.resolve(full));
|
|
131
|
+
if (seen.has(dedupeKey))
|
|
132
|
+
continue;
|
|
133
|
+
seen.add(dedupeKey);
|
|
134
|
+
out.push({ file: full, mtimeMs: stat.mtimeMs });
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if (unreadableDirCount > 0) {
|
|
138
|
+
console.error("[raw-evidence] session-store directories skipped during the walk", JSON.stringify({
|
|
139
|
+
reason: "session_dir_unreadable",
|
|
140
|
+
unreadable_dir_count: unreadableDirCount,
|
|
141
|
+
found_file_count: out.length,
|
|
142
|
+
...firstUnreadableDir,
|
|
143
|
+
}));
|
|
144
|
+
}
|
|
145
|
+
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
146
|
+
return out.map((entry) => entry.file);
|
|
147
|
+
}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What one collection pass carries, what it may spend, and where its bytes land.
|
|
3
|
+
*
|
|
4
|
+
* `CollectionState` is the single object every source path in this family
|
|
5
|
+
* threads through. Its gap-ledger fields (`scanned`, `skipped`, `truncated`,
|
|
6
|
+
* `failed`, `redacted`, `reused`, `caps`) are exactly the `EvidenceGapLedger`
|
|
7
|
+
* contract, so `raw-evidence-completeness.ts` can read and write them without
|
|
8
|
+
* knowing about packs, budgets or disks.
|
|
9
|
+
*
|
|
10
|
+
* Two mechanics live here rather than in any one reader, because the Codex
|
|
11
|
+
* reader, the Claude reader, the image reader and the git-diff reader all spend
|
|
12
|
+
* them:
|
|
13
|
+
*
|
|
14
|
+
* - **`admitToBudget`** — the per-sync byte and object budget (D7b),
|
|
15
|
+
* decremented in place so one `cockpit sync` across many worktrees honours
|
|
16
|
+
* one cap.
|
|
17
|
+
* - **`stageEvidenceBytes`** — bytes go on disk exactly once; content an
|
|
18
|
+
* earlier sync already staged is adopted rather than copied again.
|
|
19
|
+
*
|
|
20
|
+
* Opening a pass is staging-first (BLI-3066): a pack is named by its content,
|
|
21
|
+
* so its id cannot be known yet and the bytes land in a private
|
|
22
|
+
* `.staging-<pid>-<rand>` directory that `raw-evidence-facts.ts` promotes once
|
|
23
|
+
* collection is done.
|
|
24
|
+
*/
|
|
25
|
+
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
26
|
+
import crypto from "node:crypto";
|
|
27
|
+
import fs from "node:fs/promises";
|
|
28
|
+
import path from "node:path";
|
|
29
|
+
import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, } from "../raw-evidence-staging.js";
|
|
30
|
+
import { chmodPrivate } from "./raw-evidence-pack-store.js";
|
|
31
|
+
import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, } from "./raw-evidence-git-diff.js";
|
|
32
|
+
export const DEFAULT_SINCE_MINUTES = 24 * 60;
|
|
33
|
+
export const DEFAULT_SESSION_LIMIT = 50;
|
|
34
|
+
// Per-sync upload budgets enforced at COLLECTION time (D7b). A single marathon
|
|
35
|
+
// transcript can approach the 500 MiB wire cap (RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES),
|
|
36
|
+
// so 2 GiB leaves room for several files without starving the sync; overflow
|
|
37
|
+
// still defers and converges.
|
|
38
|
+
export const RAW_EVIDENCE_DEFAULT_BYTE_BUDGET = 2 * 1024 * 1024 * 1024;
|
|
39
|
+
export const RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET = 300;
|
|
40
|
+
export const CLAUDE_MAX_COLLECT_FILE_BYTES = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES;
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
// Opening a pass
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
/**
|
|
45
|
+
* Staging first, promotion second (BLI-3066). The pack id cannot be known until
|
|
46
|
+
* the content is, so bytes land in a private per-attempt directory, and that
|
|
47
|
+
* directory is then renamed to its content-keyed name — or dropped, when an
|
|
48
|
+
* identical pack is already there.
|
|
49
|
+
*/
|
|
50
|
+
export function newStagingDir(rawEvidenceRoot) {
|
|
51
|
+
return path.join(rawEvidenceRoot, `.staging-${process.pid}-${crypto.randomUUID().slice(0, 8)}`);
|
|
52
|
+
}
|
|
53
|
+
export async function openCollection(context, options, places) {
|
|
54
|
+
const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
|
|
55
|
+
const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
|
|
56
|
+
const staging = await readRawEvidenceStagingState(options.stateDir);
|
|
57
|
+
return {
|
|
58
|
+
context,
|
|
59
|
+
filesDir: path.join(places.stagingDir, "files"),
|
|
60
|
+
rawEvidenceRoot: places.rawEvidenceRoot,
|
|
61
|
+
packId: "",
|
|
62
|
+
staging,
|
|
63
|
+
heldSources: collectionHeldSources(staging, context.now, options),
|
|
64
|
+
stagedReusedCount: 0,
|
|
65
|
+
stagedNewCount: 0,
|
|
66
|
+
deliveryHeldCount: 0,
|
|
67
|
+
entries: [],
|
|
68
|
+
skipped: [],
|
|
69
|
+
truncated: [],
|
|
70
|
+
failed: [],
|
|
71
|
+
redacted: [],
|
|
72
|
+
reused: [],
|
|
73
|
+
scanned: new Map(),
|
|
74
|
+
caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
|
|
75
|
+
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
76
|
+
budget: options.budget ?? {
|
|
77
|
+
remainingBytes: byteBudget,
|
|
78
|
+
remainingObjects: objectBudget,
|
|
79
|
+
},
|
|
80
|
+
index: { value: 0 },
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Which sources this pass refuses to even read because their delivery is inside
|
|
85
|
+
* a backoff window — none of them, when a person asked for this pass.
|
|
86
|
+
*
|
|
87
|
+
* The bypass is logged rather than assumed: an operator retry that quietly
|
|
88
|
+
* ignored a hold would be as unreadable as the hold that quietly blocked it.
|
|
89
|
+
*/
|
|
90
|
+
function collectionHeldSources(staging, now, options) {
|
|
91
|
+
const held = heldSourceKeys(staging, now);
|
|
92
|
+
if (deliveryBackoffApplies(options.deliveryMode))
|
|
93
|
+
return held;
|
|
94
|
+
if (held.size > 0) {
|
|
95
|
+
console.error("[raw-evidence] delivery backoff bypassed for operator retry", JSON.stringify({
|
|
96
|
+
reason: DELIVERY_BACKOFF_BYPASS_REASON,
|
|
97
|
+
source_count: held.size,
|
|
98
|
+
}));
|
|
99
|
+
}
|
|
100
|
+
return new Set();
|
|
101
|
+
}
|
|
102
|
+
/** Every cap this pass could hit, declared up front and flipped when applied. */
|
|
103
|
+
function startingCaps(options) {
|
|
104
|
+
return [
|
|
105
|
+
{
|
|
106
|
+
source: "raw_evidence",
|
|
107
|
+
cap_type: "byte_budget",
|
|
108
|
+
limit: options.byteBudget,
|
|
109
|
+
observed: options.budget?.remainingBytes ?? options.byteBudget,
|
|
110
|
+
applied: false,
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
source: "raw_evidence",
|
|
114
|
+
cap_type: "object_budget",
|
|
115
|
+
limit: options.objectBudget,
|
|
116
|
+
observed: options.budget?.remainingObjects ?? options.objectBudget,
|
|
117
|
+
applied: false,
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
source: "git_diff",
|
|
121
|
+
cap_type: "max_bytes_per_diff",
|
|
122
|
+
limit: MAX_GIT_DIFF_BYTES,
|
|
123
|
+
applied: false,
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
source: "git_diff",
|
|
127
|
+
cap_type: "timeout_ms",
|
|
128
|
+
limit: GIT_DIFF_TIMEOUT_MS,
|
|
129
|
+
applied: false,
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
source: "codex_jsonl",
|
|
133
|
+
cap_type: "max_file_bytes",
|
|
134
|
+
limit: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
135
|
+
applied: false,
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
source: "claude_jsonl",
|
|
139
|
+
cap_type: "max_file_bytes",
|
|
140
|
+
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
141
|
+
applied: false,
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
source: "claude_jsonl_sidecar",
|
|
145
|
+
cap_type: "max_file_bytes",
|
|
146
|
+
limit: CLAUDE_MAX_COLLECT_FILE_BYTES,
|
|
147
|
+
applied: false,
|
|
148
|
+
},
|
|
149
|
+
];
|
|
150
|
+
}
|
|
151
|
+
export async function discardStagingDir(stagingDir) {
|
|
152
|
+
await fs
|
|
153
|
+
.rm(stagingDir, { recursive: true, force: true })
|
|
154
|
+
.catch(() => undefined);
|
|
155
|
+
}
|
|
156
|
+
// ---------------------------------------------------------------------------
|
|
157
|
+
// Staging and budgets
|
|
158
|
+
// ---------------------------------------------------------------------------
|
|
159
|
+
/**
|
|
160
|
+
* Put these bytes on disk once.
|
|
161
|
+
*
|
|
162
|
+
* If an earlier sync already staged this exact content and the copy is still
|
|
163
|
+
* there, that copy is used — the uploader only needs a readable path, and it
|
|
164
|
+
* does not care which pack directory holds it. This is the branch that stops
|
|
165
|
+
* one 334 MB rollout from becoming 559 copies while its commit keeps failing.
|
|
166
|
+
*/
|
|
167
|
+
export async function stageEvidenceBytes(collection, options) {
|
|
168
|
+
const existing = await resolveStagedObject(collection.rawEvidenceRoot, collection.staging, options.contentHash);
|
|
169
|
+
if (existing) {
|
|
170
|
+
collection.stagedReusedCount += 1;
|
|
171
|
+
console.error("[raw-evidence] staged copy reused", JSON.stringify({
|
|
172
|
+
reason: "staged_reused",
|
|
173
|
+
kind: options.kind,
|
|
174
|
+
content_hash_prefix: options.contentHash.slice(0, 16),
|
|
175
|
+
byte_size: existing.entry.byte_size,
|
|
176
|
+
pack_id: existing.entry.pack_id,
|
|
177
|
+
}));
|
|
178
|
+
return { local_path: existing.local_path, staged_in_pack: false };
|
|
179
|
+
}
|
|
180
|
+
const destination = path.join(collection.filesDir, options.fileName);
|
|
181
|
+
await fs.writeFile(destination, options.bytes, { mode: 0o600 });
|
|
182
|
+
await chmodPrivate(destination, 0o600);
|
|
183
|
+
collection.stagedNewCount += 1;
|
|
184
|
+
return { local_path: destination, staged_in_pack: true };
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Decrements the per-sync budget when a file fits, or returns a deferred-skip
|
|
188
|
+
* reason when it does not. The object budget bounds request count; the byte
|
|
189
|
+
* budget bounds buffered bytes.
|
|
190
|
+
*/
|
|
191
|
+
export function admitToBudget(budget, byteLength) {
|
|
192
|
+
if (budget.remainingObjects <= 0)
|
|
193
|
+
return "deferred_object_budget";
|
|
194
|
+
if (byteLength > budget.remainingBytes)
|
|
195
|
+
return "deferred_byte_budget";
|
|
196
|
+
budget.remainingObjects -= 1;
|
|
197
|
+
budget.remainingBytes -= byteLength;
|
|
198
|
+
return null;
|
|
199
|
+
}
|