@bli-cockpit/cli 0.2.27 → 0.2.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/raw-evidence-completeness.js +226 -0
- package/dist/adapters/raw-evidence-git-diff.js +90 -0
- package/dist/adapters/raw-evidence-keys.js +92 -0
- package/dist/adapters/raw-evidence-manifest.js +132 -0
- package/dist/adapters/raw-evidence-pack-store.js +136 -0
- package/dist/adapters/raw-evidence-sanitize.js +190 -0
- package/dist/adapters/raw-evidence.js +740 -1044
- package/dist/commands/backfill.js +7 -0
- package/dist/commands/cli-io.js +92 -0
- package/dist/commands/collection-report.js +135 -0
- package/dist/commands/collection-roots.js +153 -0
- package/dist/commands/install-receipts.js +193 -0
- package/dist/commands/install-update.js +305 -0
- package/dist/commands/local-auth.js +268 -0
- package/dist/commands/local-discovery.js +100 -0
- package/dist/commands/local-help.js +281 -0
- package/dist/commands/local.js +183 -1842
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sessions.js +162 -0
- package/dist/commands/status.js +191 -0
- package/dist/evidence-upload-client.js +43 -2
- package/dist/local-state.js +12 -1
- package/dist/raw-evidence-gc.js +178 -0
- package/dist/raw-evidence-staging.js +322 -0
- package/dist/upload-agent-artifacts.js +153 -0
- package/dist/upload-envelope.js +407 -0
- package/dist/upload-evidence-delivery.js +505 -0
- package/dist/upload-http.js +46 -0
- package/dist/upload-session-reports.js +404 -0
- package/dist/upload.js +153 -1159
- package/package.json +2 -2
|
@@ -36,6 +36,7 @@ export async function runLocalSourceCollectors(options) {
|
|
|
36
36
|
byteBudget: options.rawEvidenceByteBudget,
|
|
37
37
|
objectBudget: options.rawEvidenceObjectBudget,
|
|
38
38
|
budget: options.rawEvidenceBudget,
|
|
39
|
+
deliveryMode: options.rawEvidenceDeliveryMode,
|
|
39
40
|
})
|
|
40
41
|
: {
|
|
41
42
|
scan: makeUnavailableScan(context, "codex_jsonl", "codex-jsonl", "raw_evidence_state_dir_not_configured"),
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import { EvidenceCompletenessPayloadSchema } from "@bli-cockpit/telemetry-core";
|
|
2
|
+
// ---------------------------------------------------------------------------
|
|
3
|
+
// Writing to the ledger
|
|
4
|
+
// ---------------------------------------------------------------------------
|
|
5
|
+
export function recordScanned(ledger, source, count = 1) {
|
|
6
|
+
ledger.scanned.set(source, (ledger.scanned.get(source) ?? 0) + count);
|
|
7
|
+
}
|
|
8
|
+
export function recordSkipCount(ledger, source, reason, count) {
|
|
9
|
+
if (count <= 0)
|
|
10
|
+
return;
|
|
11
|
+
ledger.skipped.push({ kind: source, label: reason, reason, count });
|
|
12
|
+
}
|
|
13
|
+
export function recordTruncationCount(ledger, source, reason, count, details = {}) {
|
|
14
|
+
if (count <= 0)
|
|
15
|
+
return;
|
|
16
|
+
ledger.truncated.push({ kind: source, reason, count, ...details });
|
|
17
|
+
}
|
|
18
|
+
export function markCapApplied(ledger, source, capType) {
|
|
19
|
+
const cap = ledger.caps.find((candidate) => candidate.source === source && candidate.cap_type === capType);
|
|
20
|
+
if (cap)
|
|
21
|
+
cap.applied = true;
|
|
22
|
+
}
|
|
23
|
+
export function markBudgetCapApplied(ledger, reason) {
|
|
24
|
+
markCapApplied(ledger, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
|
|
25
|
+
}
|
|
26
|
+
// ---------------------------------------------------------------------------
|
|
27
|
+
// Counting
|
|
28
|
+
// ---------------------------------------------------------------------------
|
|
29
|
+
/** A record with no explicit `count` stands for exactly one file. */
|
|
30
|
+
export function evidenceEntryCount(entry) {
|
|
31
|
+
return entry.count ?? 1;
|
|
32
|
+
}
|
|
33
|
+
export function countEvidenceEntries(entries, predicate = () => true) {
|
|
34
|
+
return entries.reduce((sum, entry) => sum + (predicate(entry) ? evidenceEntryCount(entry) : 0), 0);
|
|
35
|
+
}
|
|
36
|
+
export function makeEvidenceCompleteness(ledger, options) {
|
|
37
|
+
const sourceCounts = listSourceNames(ledger).map((source) => summarizeSource(ledger, source));
|
|
38
|
+
const totals = sumSourceTotals(sourceCounts);
|
|
39
|
+
const hasGaps = hasAnyGap(ledger, totals);
|
|
40
|
+
return EvidenceCompletenessPayloadSchema.parse({
|
|
41
|
+
schema_version: "evidence-completeness.v1",
|
|
42
|
+
status: decideCompletenessStatus(totals, hasGaps),
|
|
43
|
+
generated_at: options.finishedAt,
|
|
44
|
+
scan_window: {
|
|
45
|
+
started_at: options.startedAt,
|
|
46
|
+
finished_at: options.finishedAt,
|
|
47
|
+
since_minutes: options.sinceMinutes,
|
|
48
|
+
},
|
|
49
|
+
source_counts: sourceCounts,
|
|
50
|
+
totals,
|
|
51
|
+
caps: ledger.caps,
|
|
52
|
+
skip_reasons: sortedBySourceAndReason(tallyByReason(ledger.skipped)),
|
|
53
|
+
failure_reasons: sortedBySourceAndReason(tallyByReason(ledger.failed)),
|
|
54
|
+
truncation_markers: sortedBySourceAndReason(tallyTruncationMarkers(ledger)),
|
|
55
|
+
redaction_markers: tallyRedactionMarkers(ledger).sort((a, b) => `${a.source}:${a.mode}`.localeCompare(`${b.source}:${b.mode}`)),
|
|
56
|
+
notes: completenessNotes(ledger, totals, hasGaps),
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
/** Every source that either offered a candidate or produced a record. */
|
|
60
|
+
function listSourceNames(ledger) {
|
|
61
|
+
const sources = new Set(ledger.scanned.keys());
|
|
62
|
+
for (const entry of ledger.entries)
|
|
63
|
+
sources.add(entry.kind);
|
|
64
|
+
for (const entry of ledger.skipped)
|
|
65
|
+
sources.add(entry.kind);
|
|
66
|
+
for (const entry of ledger.reused)
|
|
67
|
+
sources.add(entry.kind);
|
|
68
|
+
for (const entry of ledger.truncated)
|
|
69
|
+
sources.add(entry.kind);
|
|
70
|
+
for (const entry of ledger.failed)
|
|
71
|
+
sources.add(entry.kind);
|
|
72
|
+
for (const entry of ledger.redacted)
|
|
73
|
+
sources.add(entry.kind);
|
|
74
|
+
return [...sources].sort();
|
|
75
|
+
}
|
|
76
|
+
function summarizeSource(ledger, source) {
|
|
77
|
+
const skipped = ledger.skipped.filter((entry) => entry.kind === source);
|
|
78
|
+
return {
|
|
79
|
+
source,
|
|
80
|
+
scanned_count: ledger.scanned.get(source) ?? 0,
|
|
81
|
+
included_count: ledger.entries.filter((entry) => entry.kind === source)
|
|
82
|
+
.length,
|
|
83
|
+
skipped_count: countEvidenceEntries(skipped),
|
|
84
|
+
truncated_count: countEvidenceEntries(ledger.truncated, (entry) => entry.kind === source),
|
|
85
|
+
deferred_count: countEvidenceEntries(skipped, (entry) => entry.reason.startsWith("deferred_")),
|
|
86
|
+
reused_count: ledger.reused.filter((entry) => entry.kind === source).length,
|
|
87
|
+
failed_count: countEvidenceEntries(ledger.failed, (entry) => entry.kind === source),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
function sumSourceTotals(sourceCounts) {
|
|
91
|
+
return sourceCounts.reduce((sum, count) => ({
|
|
92
|
+
scanned_count: sum.scanned_count + count.scanned_count,
|
|
93
|
+
included_count: sum.included_count + count.included_count,
|
|
94
|
+
skipped_count: sum.skipped_count + count.skipped_count,
|
|
95
|
+
truncated_count: sum.truncated_count + count.truncated_count,
|
|
96
|
+
deferred_count: sum.deferred_count + count.deferred_count,
|
|
97
|
+
reused_count: sum.reused_count + count.reused_count,
|
|
98
|
+
failed_count: sum.failed_count + count.failed_count,
|
|
99
|
+
}), {
|
|
100
|
+
scanned_count: 0,
|
|
101
|
+
included_count: 0,
|
|
102
|
+
skipped_count: 0,
|
|
103
|
+
truncated_count: 0,
|
|
104
|
+
deferred_count: 0,
|
|
105
|
+
reused_count: 0,
|
|
106
|
+
failed_count: 0,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
/** Collapse records to one row per (source, reason), summing their counts. */
|
|
110
|
+
function tallyByReason(records) {
|
|
111
|
+
const counts = new Map();
|
|
112
|
+
for (const record of records) {
|
|
113
|
+
const key = `${record.kind}:${record.reason}`;
|
|
114
|
+
const existing = counts.get(key);
|
|
115
|
+
if (existing) {
|
|
116
|
+
existing.count += evidenceEntryCount(record);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
counts.set(key, {
|
|
120
|
+
source: record.kind,
|
|
121
|
+
reason: record.reason,
|
|
122
|
+
count: evidenceEntryCount(record),
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
return [...counts.values()];
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Truncations of the same source and reason merge, keeping the WORST numbers
|
|
129
|
+
* seen: the largest observed size and the largest amount actually included.
|
|
130
|
+
*/
|
|
131
|
+
function tallyTruncationMarkers(ledger) {
|
|
132
|
+
const counts = new Map();
|
|
133
|
+
for (const truncated of ledger.truncated) {
|
|
134
|
+
const key = `${truncated.kind}:${truncated.reason}`;
|
|
135
|
+
const existing = counts.get(key);
|
|
136
|
+
if (existing) {
|
|
137
|
+
existing.count += evidenceEntryCount(truncated);
|
|
138
|
+
existing.observed_bytes = Math.max(existing.observed_bytes ?? 0, truncated.observed_bytes ?? 0);
|
|
139
|
+
existing.included_bytes = Math.max(existing.included_bytes ?? 0, truncated.included_bytes ?? 0);
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
counts.set(key, {
|
|
143
|
+
source: truncated.kind,
|
|
144
|
+
reason: truncated.reason,
|
|
145
|
+
count: evidenceEntryCount(truncated),
|
|
146
|
+
...(truncated.max_bytes !== undefined
|
|
147
|
+
? { max_bytes: truncated.max_bytes }
|
|
148
|
+
: {}),
|
|
149
|
+
...(truncated.observed_bytes !== undefined
|
|
150
|
+
? { observed_bytes: truncated.observed_bytes }
|
|
151
|
+
: {}),
|
|
152
|
+
...(truncated.included_bytes !== undefined
|
|
153
|
+
? { included_bytes: truncated.included_bytes }
|
|
154
|
+
: {}),
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
return [...counts.values()];
|
|
158
|
+
}
|
|
159
|
+
/** Grouped by source, masking mode, and the exact set of rules that fired. */
|
|
160
|
+
function tallyRedactionMarkers(ledger) {
|
|
161
|
+
const counts = new Map();
|
|
162
|
+
for (const redacted of ledger.redacted) {
|
|
163
|
+
const ruleIds = redacted.redaction.rule_counts.map((rule) => rule.rule_id);
|
|
164
|
+
const key = `${redacted.kind}:${redacted.completenessLabel}:${ruleIds.sort().join(",")}`;
|
|
165
|
+
const existing = counts.get(key);
|
|
166
|
+
if (existing) {
|
|
167
|
+
existing.count += 1;
|
|
168
|
+
existing.rule_ids = [
|
|
169
|
+
...new Set([...existing.rule_ids, ...ruleIds]),
|
|
170
|
+
].sort();
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
counts.set(key, {
|
|
174
|
+
source: redacted.kind,
|
|
175
|
+
status: "sanitized",
|
|
176
|
+
mode: redacted.completenessLabel,
|
|
177
|
+
count: 1,
|
|
178
|
+
rule_ids: [...new Set(ruleIds)].sort(),
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
return [...counts.values()];
|
|
182
|
+
}
|
|
183
|
+
function sortedBySourceAndReason(markers) {
|
|
184
|
+
return markers.sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`));
|
|
185
|
+
}
|
|
186
|
+
/** Anything at all that keeps this pass from being raw-complete. */
|
|
187
|
+
function hasAnyGap(ledger, totals) {
|
|
188
|
+
return (totals.skipped_count > 0 ||
|
|
189
|
+
totals.truncated_count > 0 ||
|
|
190
|
+
totals.deferred_count > 0 ||
|
|
191
|
+
totals.failed_count > 0 ||
|
|
192
|
+
ledger.redacted.length > 0 ||
|
|
193
|
+
ledger.caps.some((cap) => cap.applied));
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* | condition | status |
|
|
197
|
+
* | ------------------------------------------ | ---------- |
|
|
198
|
+
* | a failure and nothing landed at all | `failed` |
|
|
199
|
+
* | nothing landed and nothing was missing | `empty` |
|
|
200
|
+
* | anything was skipped, cut, deferred, masked| `partial` |
|
|
201
|
+
* | otherwise | `complete` |
|
|
202
|
+
*/
|
|
203
|
+
function decideCompletenessStatus(totals, hasGaps) {
|
|
204
|
+
const landed = totals.included_count + totals.reused_count;
|
|
205
|
+
if (totals.failed_count > 0 && landed === 0)
|
|
206
|
+
return "failed";
|
|
207
|
+
if (landed === 0 && !hasGaps)
|
|
208
|
+
return "empty";
|
|
209
|
+
return hasGaps ? "partial" : "complete";
|
|
210
|
+
}
|
|
211
|
+
function completenessNotes(ledger, totals, hasGaps) {
|
|
212
|
+
if (totals.failed_count > 0) {
|
|
213
|
+
return [
|
|
214
|
+
"Evidence collection failed; downstream analysis should not infer confidence.",
|
|
215
|
+
];
|
|
216
|
+
}
|
|
217
|
+
if (ledger.redacted.length > 0) {
|
|
218
|
+
return [
|
|
219
|
+
"Evidence was sanitized before upload; downstream analysis should not treat it as raw-complete.",
|
|
220
|
+
];
|
|
221
|
+
}
|
|
222
|
+
if (hasGaps) {
|
|
223
|
+
return ["Evidence is incomplete; downstream analysis should lower confidence."];
|
|
224
|
+
}
|
|
225
|
+
return [];
|
|
226
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a working tree's diff without letting it read a secret or run away.
|
|
3
|
+
*
|
|
4
|
+
* Three protections, all of them caps that name themselves when they fire:
|
|
5
|
+
*
|
|
6
|
+
* - **the pathspec** never even asks git for `.env*`, `*secret*`, `*credential*`,
|
|
7
|
+
* `*private-key*`, `*.pem` or `*.key`; those files are not read, not skipped.
|
|
8
|
+
* - **`MAX_GIT_DIFF_BYTES`** stops buffering at 2 MiB and kills the child once
|
|
9
|
+
* more than that has been observed, so a giant rebase cannot exhaust memory.
|
|
10
|
+
* - **`GIT_DIFF_TIMEOUT_MS`** kills a diff that hangs.
|
|
11
|
+
*
|
|
12
|
+
* Either cap yields a *truncated* result rather than a failure: the bytes that
|
|
13
|
+
* were captured are real evidence, and the truncation reason travels with them.
|
|
14
|
+
*/
|
|
15
|
+
import { spawn } from "node:child_process";
|
|
16
|
+
export const MAX_GIT_DIFF_BYTES = 2 * 1024 * 1024;
|
|
17
|
+
export const GIT_DIFF_TIMEOUT_MS = 3_000;
|
|
18
|
+
/** Paths Cockpit never reads, excluded at the git level rather than after. */
|
|
19
|
+
const SECRET_EXCLUDING_PATHSPEC = [
|
|
20
|
+
".",
|
|
21
|
+
":(exclude).env",
|
|
22
|
+
":(exclude).env.*",
|
|
23
|
+
":(exclude)**/.env",
|
|
24
|
+
":(exclude)**/.env.*",
|
|
25
|
+
":(exclude)**/*secret*",
|
|
26
|
+
":(exclude)**/*credential*",
|
|
27
|
+
":(exclude)**/*private-key*",
|
|
28
|
+
":(exclude)**/*.pem",
|
|
29
|
+
":(exclude)**/*.key",
|
|
30
|
+
];
|
|
31
|
+
export async function runGitDiff(args, repoRoot) {
|
|
32
|
+
return new Promise((resolve, reject) => {
|
|
33
|
+
const child = spawn("git", [...args, ...SECRET_EXCLUDING_PATHSPEC], {
|
|
34
|
+
cwd: repoRoot,
|
|
35
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
36
|
+
});
|
|
37
|
+
const stdoutChunks = [];
|
|
38
|
+
const stderrChunks = [];
|
|
39
|
+
let observedBytes = 0;
|
|
40
|
+
let includedBytes = 0;
|
|
41
|
+
let truncated = false;
|
|
42
|
+
let timedOut = false;
|
|
43
|
+
const timeout = setTimeout(() => {
|
|
44
|
+
timedOut = true;
|
|
45
|
+
truncated = true;
|
|
46
|
+
child.kill("SIGTERM");
|
|
47
|
+
}, GIT_DIFF_TIMEOUT_MS);
|
|
48
|
+
child.stdout.on("data", (chunk) => {
|
|
49
|
+
observedBytes += chunk.byteLength;
|
|
50
|
+
if (includedBytes < MAX_GIT_DIFF_BYTES) {
|
|
51
|
+
const next = chunk.subarray(0, MAX_GIT_DIFF_BYTES - includedBytes);
|
|
52
|
+
stdoutChunks.push(next);
|
|
53
|
+
includedBytes += next.byteLength;
|
|
54
|
+
}
|
|
55
|
+
if (observedBytes > MAX_GIT_DIFF_BYTES) {
|
|
56
|
+
truncated = true;
|
|
57
|
+
child.kill("SIGTERM");
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
// Only enough stderr to name a failure; never enough to hold content.
|
|
61
|
+
child.stderr.on("data", (chunk) => {
|
|
62
|
+
if (stderrChunks.reduce((sum, item) => sum + item.byteLength, 0) < 4096) {
|
|
63
|
+
stderrChunks.push(chunk.subarray(0, 4096));
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
child.on("error", (error) => {
|
|
67
|
+
clearTimeout(timeout);
|
|
68
|
+
reject(error);
|
|
69
|
+
});
|
|
70
|
+
child.on("close", (code, signal) => {
|
|
71
|
+
clearTimeout(timeout);
|
|
72
|
+
// A cap that fired killed the child on purpose, so SIGTERM and a
|
|
73
|
+
// non-zero code are both success here, not failure.
|
|
74
|
+
if (code === 0 || truncated || signal === "SIGTERM") {
|
|
75
|
+
resolve({
|
|
76
|
+
stdout: Buffer.concat(stdoutChunks).toString("utf8"),
|
|
77
|
+
truncated,
|
|
78
|
+
observedBytes,
|
|
79
|
+
truncationReason: timedOut
|
|
80
|
+
? "git_diff_timeout"
|
|
81
|
+
: "max_git_diff_bytes",
|
|
82
|
+
truncationCapType: timedOut ? "timeout_ms" : "max_bytes_per_diff",
|
|
83
|
+
});
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const stderr = Buffer.concat(stderrChunks).toString("utf8").trim();
|
|
87
|
+
reject(new Error(stderr || `git diff failed with code ${code ?? signal}`));
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a piece of raw evidence is called.
|
|
3
|
+
*
|
|
4
|
+
* Two naming jobs live here and nothing else:
|
|
5
|
+
*
|
|
6
|
+
* 1. **Hashing** — `sha256` names bytes, `shortHash` names a string in twelve
|
|
7
|
+
* hex characters. Content hashes are how a pack recognises bytes it has
|
|
8
|
+
* already staged, so they must never depend on a clock or a path.
|
|
9
|
+
* 2. **Remote object keys** — the readable `operators/…/repos/…/sessions/…`
|
|
10
|
+
* namespace an operator reads during an incident, followed by an immutable
|
|
11
|
+
* content address (or a pack-relative manifest path).
|
|
12
|
+
*
|
|
13
|
+
* Everything here is pure: same input, same name, on every machine and every
|
|
14
|
+
* platform. Nothing in this file reads a file, spawns a process or logs.
|
|
15
|
+
*/
|
|
16
|
+
import { SECRET_FILE_SEGMENT_PATTERN } from "@bli-cockpit/telemetry-core";
|
|
17
|
+
import crypto from "node:crypto";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
export function sha256(value) {
|
|
20
|
+
return crypto.createHash("sha256").update(value).digest("hex");
|
|
21
|
+
}
|
|
22
|
+
export function shortHash(value) {
|
|
23
|
+
return crypto
|
|
24
|
+
.createHash("sha256")
|
|
25
|
+
.update(value, "utf8")
|
|
26
|
+
.digest("hex")
|
|
27
|
+
.slice(0, 12);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Object keys must satisfy the server's key pattern; ids derived from file
|
|
31
|
+
* content fall back to a hash rather than failing the whole upload batch.
|
|
32
|
+
*/
|
|
33
|
+
export function safeKeySegment(value) {
|
|
34
|
+
return /^[A-Za-z0-9._-]{1,80}$/.test(value) ? value : shortHash(value);
|
|
35
|
+
}
|
|
36
|
+
export function isSecretLikePath(value) {
|
|
37
|
+
return SECRET_FILE_SEGMENT_PATTERN.test(value);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Raw evidence keys start with human-readable context, then end in immutable
|
|
41
|
+
* content addresses or pack-relative manifest paths. The local cursor reuses
|
|
42
|
+
* prior content hashes across syncs; the readable date/session folders are for
|
|
43
|
+
* operator debugging and incident response.
|
|
44
|
+
*/
|
|
45
|
+
export function remoteObjectKey(options) {
|
|
46
|
+
const namespace = readableEvidenceNamespace(options.context);
|
|
47
|
+
if (options.contentAddress) {
|
|
48
|
+
return posixPath([...namespace, options.contentAddress]);
|
|
49
|
+
}
|
|
50
|
+
return posixPath([...namespace, options.packId, options.relativePath]);
|
|
51
|
+
}
|
|
52
|
+
function readableEvidenceNamespace(context) {
|
|
53
|
+
return [
|
|
54
|
+
"operators",
|
|
55
|
+
operatorSlug(context),
|
|
56
|
+
"repos",
|
|
57
|
+
readableKeySegment(context.repoLabel ?? path.basename(context.repoRoot), "repo"),
|
|
58
|
+
"worktrees",
|
|
59
|
+
readableKeySegment(context.worktreeLabel ?? path.basename(context.repoRoot), "worktree"),
|
|
60
|
+
"tickets",
|
|
61
|
+
readableKeySegment(context.activeTicketId ?? "unbound", "unbound", {
|
|
62
|
+
lowercase: false,
|
|
63
|
+
}),
|
|
64
|
+
"dates",
|
|
65
|
+
context.now.toISOString().slice(0, 10),
|
|
66
|
+
"sessions",
|
|
67
|
+
readableKeySegment(context.sessionId, "session"),
|
|
68
|
+
"ids",
|
|
69
|
+
safeKeySegment(context.operatorId),
|
|
70
|
+
safeKeySegment(context.workContextId),
|
|
71
|
+
];
|
|
72
|
+
}
|
|
73
|
+
function operatorSlug(context) {
|
|
74
|
+
const labelBeforeDomain = (context.operatorLabel ?? context.operatorId)
|
|
75
|
+
.split("@", 1)[0]
|
|
76
|
+
.trim();
|
|
77
|
+
const readable = readableKeySegment(labelBeforeDomain, "operator");
|
|
78
|
+
return `${readable}-${shortHash(context.operatorId).slice(0, 6)}`;
|
|
79
|
+
}
|
|
80
|
+
function readableKeySegment(value, fallback, options = {}) {
|
|
81
|
+
const base = options.lowercase === false ? value : value.toLowerCase();
|
|
82
|
+
const slug = base
|
|
83
|
+
.trim()
|
|
84
|
+
.replace(/[^A-Za-z0-9._-]+/g, "-")
|
|
85
|
+
.replace(/^-+|-+$/g, "")
|
|
86
|
+
.replace(/-{2,}/g, "-")
|
|
87
|
+
.slice(0, 80);
|
|
88
|
+
return slug || fallback;
|
|
89
|
+
}
|
|
90
|
+
function posixPath(parts) {
|
|
91
|
+
return parts.join("/").replace(/\\/g, "/").replace(/\/+/g, "/");
|
|
92
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { remoteObjectKey, sha256 } from "./raw-evidence-keys.js";
|
|
3
|
+
export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
|
|
4
|
+
export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
|
|
5
|
+
export function evidenceEntry(options) {
|
|
6
|
+
const digest = sha256(options.bytes);
|
|
7
|
+
const objectKey = remoteObjectKey({
|
|
8
|
+
context: options.context,
|
|
9
|
+
packId: options.packId,
|
|
10
|
+
relativePath: options.relativePath,
|
|
11
|
+
contentAddress: options.contentAddress,
|
|
12
|
+
});
|
|
13
|
+
return {
|
|
14
|
+
kind: options.kind,
|
|
15
|
+
local_path: options.localPath,
|
|
16
|
+
relative_path: options.relativePath,
|
|
17
|
+
object_key: objectKey,
|
|
18
|
+
content_hash_sha256: digest,
|
|
19
|
+
byte_size: options.bytes.byteLength,
|
|
20
|
+
media_type: options.mediaType,
|
|
21
|
+
redacted_summary: options.redactedSummary,
|
|
22
|
+
...(options.redaction ? { redaction: options.redaction } : {}),
|
|
23
|
+
codex_session_id: options.codexSessionId ?? null,
|
|
24
|
+
staged_in_pack: options.stagedInPack !== false,
|
|
25
|
+
source_key: options.sourceKey ?? null,
|
|
26
|
+
...(options.artifactMetadata
|
|
27
|
+
? {
|
|
28
|
+
artifact_metadata: {
|
|
29
|
+
...options.artifactMetadata,
|
|
30
|
+
raw_evidence_pointer_id: objectKey,
|
|
31
|
+
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
32
|
+
object_key: objectKey,
|
|
33
|
+
content_hash_sha256: digest,
|
|
34
|
+
byte_size: options.bytes.byteLength,
|
|
35
|
+
},
|
|
36
|
+
}
|
|
37
|
+
: {}),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export function pointerFromEntry(entry) {
|
|
41
|
+
return {
|
|
42
|
+
raw_evidence_pointer_id: entry.object_key,
|
|
43
|
+
privacy_classification: "remote_durable_raw_evidence",
|
|
44
|
+
retention_policy: {
|
|
45
|
+
mode: RAW_EVIDENCE_RETENTION_MODE,
|
|
46
|
+
privacy_classification: "remote_durable_raw_evidence",
|
|
47
|
+
},
|
|
48
|
+
storage_scope: "remote_object",
|
|
49
|
+
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
50
|
+
object_key: entry.object_key,
|
|
51
|
+
content_hash_sha256: entry.content_hash_sha256,
|
|
52
|
+
byte_size: entry.byte_size,
|
|
53
|
+
media_type: entry.media_type,
|
|
54
|
+
redacted_summary: entry.redacted_summary,
|
|
55
|
+
...(entry.redaction ? { redaction: entry.redaction } : {}),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
export function makeManifest(options) {
|
|
59
|
+
return {
|
|
60
|
+
schema: "bli.local_raw_evidence_pack.v1",
|
|
61
|
+
pack_id: options.packId,
|
|
62
|
+
created_at: options.context.now.toISOString(),
|
|
63
|
+
work_context_id: options.context.workContextId,
|
|
64
|
+
session_id: options.context.sessionId,
|
|
65
|
+
operator_id: options.context.operatorId,
|
|
66
|
+
operator_label: options.context.operatorLabel,
|
|
67
|
+
repo_label: options.context.repoLabel ?? path.basename(options.context.repoRoot),
|
|
68
|
+
worktree_label: options.context.worktreeLabel,
|
|
69
|
+
active_ticket_id: options.context.activeTicketId ?? null,
|
|
70
|
+
repo_basename: path.basename(options.context.repoRoot),
|
|
71
|
+
branch: options.context.branch,
|
|
72
|
+
storage_bucket: RAW_EVIDENCE_BUCKET,
|
|
73
|
+
raw_policy: {
|
|
74
|
+
raw_prompts: "preserved_private_durable_remote",
|
|
75
|
+
raw_responses: "preserved_private_durable_remote",
|
|
76
|
+
transcripts: "preserved_private_durable_remote",
|
|
77
|
+
claude_transcripts: "preserved_private_durable_remote",
|
|
78
|
+
tool_payloads: "preserved_private_durable_remote",
|
|
79
|
+
git_diffs: "preserved_private_durable_remote_env_secret_paths_excluded",
|
|
80
|
+
agent_image_attachments: "preserved_private_durable_remote_explicit_agent_session_attachment_only",
|
|
81
|
+
env_files: "never_read",
|
|
82
|
+
stdout: "manifest_only_no_raw_content",
|
|
83
|
+
},
|
|
84
|
+
files: options.entries.map(redactManifestEntry),
|
|
85
|
+
skipped: options.skipped,
|
|
86
|
+
redacted: options.redacted,
|
|
87
|
+
reused: options.reused,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
// `local_path` is redacted out, and so are the staging bookkeeping fields:
|
|
91
|
+
// where a copy happens to live on this disk is not part of the pack's identity,
|
|
92
|
+
// and putting it in the manifest would make byte-identical content produce
|
|
93
|
+
// different manifests.
|
|
94
|
+
function redactManifestEntry(entry) {
|
|
95
|
+
return {
|
|
96
|
+
kind: entry.kind,
|
|
97
|
+
relative_path: entry.relative_path,
|
|
98
|
+
object_key: entry.object_key,
|
|
99
|
+
content_hash_sha256: entry.content_hash_sha256,
|
|
100
|
+
byte_size: entry.byte_size,
|
|
101
|
+
media_type: entry.media_type,
|
|
102
|
+
redacted_summary: entry.redacted_summary,
|
|
103
|
+
...(entry.redaction ? { redaction: entry.redaction } : {}),
|
|
104
|
+
...(entry.artifact_metadata
|
|
105
|
+
? { artifact_metadata: entry.artifact_metadata }
|
|
106
|
+
: {}),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Does a manifest already on disk describe exactly this set of files?
|
|
111
|
+
*
|
|
112
|
+
* Compared by sorted content hash, not by order or by name: the question is
|
|
113
|
+
* whether the reused pack's manifest is still true, and an unreadable or
|
|
114
|
+
* unparseable manifest answers no.
|
|
115
|
+
*/
|
|
116
|
+
export function manifestDescribesEntries(manifestBytes, entries) {
|
|
117
|
+
try {
|
|
118
|
+
const parsed = JSON.parse(manifestBytes.toString("utf8"));
|
|
119
|
+
const recorded = (parsed.files ?? [])
|
|
120
|
+
.map((file) => typeof file.content_hash_sha256 === "string"
|
|
121
|
+
? file.content_hash_sha256
|
|
122
|
+
: "")
|
|
123
|
+
.filter(Boolean)
|
|
124
|
+
.sort();
|
|
125
|
+
const expected = entries.map((entry) => entry.content_hash_sha256).sort();
|
|
126
|
+
return (recorded.length === expected.length &&
|
|
127
|
+
recorded.every((hash, index) => hash === expected[index]));
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where the bytes live on this disk.
|
|
3
|
+
*
|
|
4
|
+
* Staging first, promotion second (BLI-3066). A pack's id is derived from the
|
|
5
|
+
* content in it, so it cannot be known until collection is done: bytes land in
|
|
6
|
+
* a private `.staging-<pid>-<rand>` directory, and only then does the directory
|
|
7
|
+
* become its content-keyed name. Identical content on the next sync resolves to
|
|
8
|
+
* the identical directory instead of a 560th copy.
|
|
9
|
+
*
|
|
10
|
+
* Everything written here is mode `0600` inside a `0700` directory, on every
|
|
11
|
+
* platform except Windows, where the chmod is a no-op.
|
|
12
|
+
*/
|
|
13
|
+
import fs from "node:fs/promises";
|
|
14
|
+
import path from "node:path";
|
|
15
|
+
import { makeManifest, manifestDescribesEntries, } from "./raw-evidence-manifest.js";
|
|
16
|
+
import { writeRawEvidenceStagingState, } from "../raw-evidence-staging.js";
|
|
17
|
+
/**
|
|
18
|
+
* Turn the staging directory into the content-keyed pack directory.
|
|
19
|
+
*
|
|
20
|
+
* Three outcomes, all of them named:
|
|
21
|
+
* - the pack does not exist yet: rename staging into place (`new`);
|
|
22
|
+
* - it exists and holds every file this pass staged: adopt it and delete the
|
|
23
|
+
* staging copy (`reused`) — nothing is copied twice;
|
|
24
|
+
* - it exists but is missing files: fill the gaps from staging
|
|
25
|
+
* (`restaged_incomplete`), because a half-written pack must not be trusted.
|
|
26
|
+
*
|
|
27
|
+
* Never deletes an existing pack directory wholesale: another pack's entries
|
|
28
|
+
* can point into it through the staged-object index.
|
|
29
|
+
*/
|
|
30
|
+
export async function promoteStagedPack(options) {
|
|
31
|
+
const evidenceDir = path.join(options.rawEvidenceRoot, options.packId);
|
|
32
|
+
const priorPackCount = await countPriorPacks(options.rawEvidenceRoot, options.workContextId, options.packId);
|
|
33
|
+
const existing = await fs.stat(evidenceDir).catch(() => null);
|
|
34
|
+
if (!existing?.isDirectory()) {
|
|
35
|
+
try {
|
|
36
|
+
await fs.rename(options.stagingDir, evidenceDir);
|
|
37
|
+
return {
|
|
38
|
+
state: "new",
|
|
39
|
+
evidenceDir,
|
|
40
|
+
priorPackCount,
|
|
41
|
+
refilledFileCount: 0,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// A concurrent sync can win the race to the same content-keyed name.
|
|
46
|
+
// Losing it is fine: the winner staged the identical bytes.
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const refilledFileCount = await refillMissingPackFiles(evidenceDir, options.entries);
|
|
50
|
+
await fs
|
|
51
|
+
.rm(options.stagingDir, { recursive: true, force: true })
|
|
52
|
+
.catch(() => undefined);
|
|
53
|
+
return {
|
|
54
|
+
state: refilledFileCount > 0 ? "restaged_incomplete" : "reused",
|
|
55
|
+
evidenceDir,
|
|
56
|
+
priorPackCount,
|
|
57
|
+
refilledFileCount,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/** A file already there at the right size is left alone; anything else is recopied. */
|
|
61
|
+
async function refillMissingPackFiles(evidenceDir, entries) {
|
|
62
|
+
let refilledFileCount = 0;
|
|
63
|
+
await ensurePrivateDir(path.join(evidenceDir, "files"));
|
|
64
|
+
for (const entry of entries) {
|
|
65
|
+
if (!entry.staged_in_pack)
|
|
66
|
+
continue;
|
|
67
|
+
const target = path.join(evidenceDir, "files", path.basename(entry.local_path));
|
|
68
|
+
const info = await fs.stat(target).catch(() => null);
|
|
69
|
+
if (info?.isFile() && info.size === entry.byte_size)
|
|
70
|
+
continue;
|
|
71
|
+
await fs.copyFile(entry.local_path, target).catch(() => undefined);
|
|
72
|
+
await chmodPrivate(target, 0o600);
|
|
73
|
+
refilledFileCount += 1;
|
|
74
|
+
}
|
|
75
|
+
return refilledFileCount;
|
|
76
|
+
}
|
|
77
|
+
async function countPriorPacks(rawEvidenceRoot, workContextId, packId) {
|
|
78
|
+
const entries = await fs
|
|
79
|
+
.readdir(rawEvidenceRoot, { withFileTypes: true })
|
|
80
|
+
.catch(() => []);
|
|
81
|
+
return entries.filter((entry) => entry.isDirectory() &&
|
|
82
|
+
entry.name !== packId &&
|
|
83
|
+
entry.name.startsWith(`${workContextId}-`)).length;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Write the manifest, or keep the one already in a reused pack.
|
|
87
|
+
*
|
|
88
|
+
* Byte-stability matters here: the manifest's object key is the only key that
|
|
89
|
+
* embeds the pack id, so rewriting it with a fresh `created_at` every sync
|
|
90
|
+
* would push different bytes at the same content-addressed key forever. A
|
|
91
|
+
* reused pack whose manifest already describes exactly these files keeps it.
|
|
92
|
+
*/
|
|
93
|
+
export async function stageManifest(options) {
|
|
94
|
+
if (options.reusePack) {
|
|
95
|
+
const existing = await fs.readFile(options.manifestPath).catch(() => null);
|
|
96
|
+
if (existing && manifestDescribesEntries(existing, options.entries)) {
|
|
97
|
+
return existing;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const manifestBytes = Buffer.from(`${JSON.stringify(makeManifest({
|
|
101
|
+
context: options.context,
|
|
102
|
+
packId: options.packId,
|
|
103
|
+
entries: options.entries,
|
|
104
|
+
skipped: options.skipped,
|
|
105
|
+
redacted: options.redacted,
|
|
106
|
+
reused: options.reused,
|
|
107
|
+
}), null, 2)}\n`, "utf8");
|
|
108
|
+
await fs.writeFile(options.manifestPath, manifestBytes, { mode: 0o600 });
|
|
109
|
+
await chmodPrivate(options.manifestPath, 0o600);
|
|
110
|
+
return manifestBytes;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Record which content hashes are staged where, so the next sync can adopt the
|
|
114
|
+
* copy instead of writing it again. Losing this file costs reuse and attempt
|
|
115
|
+
* counts, not evidence, so it must never fail a collection — but it must never
|
|
116
|
+
* be silent either.
|
|
117
|
+
*/
|
|
118
|
+
export async function persistStagingState(stateDir, staging, nowIso) {
|
|
119
|
+
staging.updated_at = nowIso;
|
|
120
|
+
await writeRawEvidenceStagingState(stateDir, staging).catch((error) => {
|
|
121
|
+
console.error("[raw-evidence] staging state write failed", JSON.stringify({
|
|
122
|
+
reason: "staging_state_write_failed",
|
|
123
|
+
detail: error instanceof Error ? error.name : typeof error,
|
|
124
|
+
staged_count: Object.keys(staging.staged).length,
|
|
125
|
+
}));
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
export async function ensurePrivateDir(dir) {
|
|
129
|
+
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
|
|
130
|
+
await chmodPrivate(dir, 0o700);
|
|
131
|
+
}
|
|
132
|
+
export async function chmodPrivate(target, mode) {
|
|
133
|
+
if (process.platform === "win32")
|
|
134
|
+
return;
|
|
135
|
+
await fs.chmod(target, mode).catch(() => undefined);
|
|
136
|
+
}
|