@bli-cockpit/cli 0.2.28 → 0.2.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -36,6 +36,7 @@ export async function runLocalSourceCollectors(options) {
36
36
  byteBudget: options.rawEvidenceByteBudget,
37
37
  objectBudget: options.rawEvidenceObjectBudget,
38
38
  budget: options.rawEvidenceBudget,
39
+ deliveryMode: options.rawEvidenceDeliveryMode,
39
40
  })
40
41
  : {
41
42
  scan: makeUnavailableScan(context, "codex_jsonl", "codex-jsonl", "raw_evidence_state_dir_not_configured"),
@@ -0,0 +1,226 @@
1
+ import { EvidenceCompletenessPayloadSchema } from "@bli-cockpit/telemetry-core";
2
+ // ---------------------------------------------------------------------------
3
+ // Writing to the ledger
4
+ // ---------------------------------------------------------------------------
5
+ export function recordScanned(ledger, source, count = 1) {
6
+ ledger.scanned.set(source, (ledger.scanned.get(source) ?? 0) + count);
7
+ }
8
+ export function recordSkipCount(ledger, source, reason, count) {
9
+ if (count <= 0)
10
+ return;
11
+ ledger.skipped.push({ kind: source, label: reason, reason, count });
12
+ }
13
+ export function recordTruncationCount(ledger, source, reason, count, details = {}) {
14
+ if (count <= 0)
15
+ return;
16
+ ledger.truncated.push({ kind: source, reason, count, ...details });
17
+ }
18
+ export function markCapApplied(ledger, source, capType) {
19
+ const cap = ledger.caps.find((candidate) => candidate.source === source && candidate.cap_type === capType);
20
+ if (cap)
21
+ cap.applied = true;
22
+ }
23
+ export function markBudgetCapApplied(ledger, reason) {
24
+ markCapApplied(ledger, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
25
+ }
26
+ // ---------------------------------------------------------------------------
27
+ // Counting
28
+ // ---------------------------------------------------------------------------
29
+ /** A record with no explicit `count` stands for exactly one file. */
30
+ export function evidenceEntryCount(entry) {
31
+ return entry.count ?? 1;
32
+ }
33
+ export function countEvidenceEntries(entries, predicate = () => true) {
34
+ return entries.reduce((sum, entry) => sum + (predicate(entry) ? evidenceEntryCount(entry) : 0), 0);
35
+ }
36
+ export function makeEvidenceCompleteness(ledger, options) {
37
+ const sourceCounts = listSourceNames(ledger).map((source) => summarizeSource(ledger, source));
38
+ const totals = sumSourceTotals(sourceCounts);
39
+ const hasGaps = hasAnyGap(ledger, totals);
40
+ return EvidenceCompletenessPayloadSchema.parse({
41
+ schema_version: "evidence-completeness.v1",
42
+ status: decideCompletenessStatus(totals, hasGaps),
43
+ generated_at: options.finishedAt,
44
+ scan_window: {
45
+ started_at: options.startedAt,
46
+ finished_at: options.finishedAt,
47
+ since_minutes: options.sinceMinutes,
48
+ },
49
+ source_counts: sourceCounts,
50
+ totals,
51
+ caps: ledger.caps,
52
+ skip_reasons: sortedBySourceAndReason(tallyByReason(ledger.skipped)),
53
+ failure_reasons: sortedBySourceAndReason(tallyByReason(ledger.failed)),
54
+ truncation_markers: sortedBySourceAndReason(tallyTruncationMarkers(ledger)),
55
+ redaction_markers: tallyRedactionMarkers(ledger).sort((a, b) => `${a.source}:${a.mode}`.localeCompare(`${b.source}:${b.mode}`)),
56
+ notes: completenessNotes(ledger, totals, hasGaps),
57
+ });
58
+ }
59
+ /** Every source that either offered a candidate or produced a record. */
60
+ function listSourceNames(ledger) {
61
+ const sources = new Set(ledger.scanned.keys());
62
+ for (const entry of ledger.entries)
63
+ sources.add(entry.kind);
64
+ for (const entry of ledger.skipped)
65
+ sources.add(entry.kind);
66
+ for (const entry of ledger.reused)
67
+ sources.add(entry.kind);
68
+ for (const entry of ledger.truncated)
69
+ sources.add(entry.kind);
70
+ for (const entry of ledger.failed)
71
+ sources.add(entry.kind);
72
+ for (const entry of ledger.redacted)
73
+ sources.add(entry.kind);
74
+ return [...sources].sort();
75
+ }
76
+ function summarizeSource(ledger, source) {
77
+ const skipped = ledger.skipped.filter((entry) => entry.kind === source);
78
+ return {
79
+ source,
80
+ scanned_count: ledger.scanned.get(source) ?? 0,
81
+ included_count: ledger.entries.filter((entry) => entry.kind === source)
82
+ .length,
83
+ skipped_count: countEvidenceEntries(skipped),
84
+ truncated_count: countEvidenceEntries(ledger.truncated, (entry) => entry.kind === source),
85
+ deferred_count: countEvidenceEntries(skipped, (entry) => entry.reason.startsWith("deferred_")),
86
+ reused_count: ledger.reused.filter((entry) => entry.kind === source).length,
87
+ failed_count: countEvidenceEntries(ledger.failed, (entry) => entry.kind === source),
88
+ };
89
+ }
90
+ function sumSourceTotals(sourceCounts) {
91
+ return sourceCounts.reduce((sum, count) => ({
92
+ scanned_count: sum.scanned_count + count.scanned_count,
93
+ included_count: sum.included_count + count.included_count,
94
+ skipped_count: sum.skipped_count + count.skipped_count,
95
+ truncated_count: sum.truncated_count + count.truncated_count,
96
+ deferred_count: sum.deferred_count + count.deferred_count,
97
+ reused_count: sum.reused_count + count.reused_count,
98
+ failed_count: sum.failed_count + count.failed_count,
99
+ }), {
100
+ scanned_count: 0,
101
+ included_count: 0,
102
+ skipped_count: 0,
103
+ truncated_count: 0,
104
+ deferred_count: 0,
105
+ reused_count: 0,
106
+ failed_count: 0,
107
+ });
108
+ }
109
+ /** Collapse records to one row per (source, reason), summing their counts. */
110
+ function tallyByReason(records) {
111
+ const counts = new Map();
112
+ for (const record of records) {
113
+ const key = `${record.kind}:${record.reason}`;
114
+ const existing = counts.get(key);
115
+ if (existing) {
116
+ existing.count += evidenceEntryCount(record);
117
+ continue;
118
+ }
119
+ counts.set(key, {
120
+ source: record.kind,
121
+ reason: record.reason,
122
+ count: evidenceEntryCount(record),
123
+ });
124
+ }
125
+ return [...counts.values()];
126
+ }
127
+ /**
128
+ * Truncations of the same source and reason merge, keeping the WORST numbers
129
+ * seen: the largest observed size and the largest amount actually included.
130
+ */
131
+ function tallyTruncationMarkers(ledger) {
132
+ const counts = new Map();
133
+ for (const truncated of ledger.truncated) {
134
+ const key = `${truncated.kind}:${truncated.reason}`;
135
+ const existing = counts.get(key);
136
+ if (existing) {
137
+ existing.count += evidenceEntryCount(truncated);
138
+ existing.observed_bytes = Math.max(existing.observed_bytes ?? 0, truncated.observed_bytes ?? 0);
139
+ existing.included_bytes = Math.max(existing.included_bytes ?? 0, truncated.included_bytes ?? 0);
140
+ continue;
141
+ }
142
+ counts.set(key, {
143
+ source: truncated.kind,
144
+ reason: truncated.reason,
145
+ count: evidenceEntryCount(truncated),
146
+ ...(truncated.max_bytes !== undefined
147
+ ? { max_bytes: truncated.max_bytes }
148
+ : {}),
149
+ ...(truncated.observed_bytes !== undefined
150
+ ? { observed_bytes: truncated.observed_bytes }
151
+ : {}),
152
+ ...(truncated.included_bytes !== undefined
153
+ ? { included_bytes: truncated.included_bytes }
154
+ : {}),
155
+ });
156
+ }
157
+ return [...counts.values()];
158
+ }
159
+ /** Grouped by source, masking mode, and the exact set of rules that fired. */
160
+ function tallyRedactionMarkers(ledger) {
161
+ const counts = new Map();
162
+ for (const redacted of ledger.redacted) {
163
+ const ruleIds = redacted.redaction.rule_counts.map((rule) => rule.rule_id);
164
+ const key = `${redacted.kind}:${redacted.completenessLabel}:${ruleIds.sort().join(",")}`;
165
+ const existing = counts.get(key);
166
+ if (existing) {
167
+ existing.count += 1;
168
+ existing.rule_ids = [
169
+ ...new Set([...existing.rule_ids, ...ruleIds]),
170
+ ].sort();
171
+ continue;
172
+ }
173
+ counts.set(key, {
174
+ source: redacted.kind,
175
+ status: "sanitized",
176
+ mode: redacted.completenessLabel,
177
+ count: 1,
178
+ rule_ids: [...new Set(ruleIds)].sort(),
179
+ });
180
+ }
181
+ return [...counts.values()];
182
+ }
183
+ function sortedBySourceAndReason(markers) {
184
+ return markers.sort((a, b) => `${a.source}:${a.reason}`.localeCompare(`${b.source}:${b.reason}`));
185
+ }
186
+ /** Anything at all that keeps this pass from being raw-complete. */
187
+ function hasAnyGap(ledger, totals) {
188
+ return (totals.skipped_count > 0 ||
189
+ totals.truncated_count > 0 ||
190
+ totals.deferred_count > 0 ||
191
+ totals.failed_count > 0 ||
192
+ ledger.redacted.length > 0 ||
193
+ ledger.caps.some((cap) => cap.applied));
194
+ }
195
+ /**
196
+ * | condition | status |
197
+ * | ------------------------------------------ | ---------- |
198
+ * | a failure and nothing landed at all | `failed` |
199
+ * | nothing landed and nothing was missing | `empty` |
200
+ * | anything was skipped, cut, deferred, masked| `partial` |
201
+ * | otherwise | `complete` |
202
+ */
203
+ function decideCompletenessStatus(totals, hasGaps) {
204
+ const landed = totals.included_count + totals.reused_count;
205
+ if (totals.failed_count > 0 && landed === 0)
206
+ return "failed";
207
+ if (landed === 0 && !hasGaps)
208
+ return "empty";
209
+ return hasGaps ? "partial" : "complete";
210
+ }
211
+ function completenessNotes(ledger, totals, hasGaps) {
212
+ if (totals.failed_count > 0) {
213
+ return [
214
+ "Evidence collection failed; downstream analysis should not infer confidence.",
215
+ ];
216
+ }
217
+ if (ledger.redacted.length > 0) {
218
+ return [
219
+ "Evidence was sanitized before upload; downstream analysis should not treat it as raw-complete.",
220
+ ];
221
+ }
222
+ if (hasGaps) {
223
+ return ["Evidence is incomplete; downstream analysis should lower confidence."];
224
+ }
225
+ return [];
226
+ }
@@ -0,0 +1,90 @@
1
+ /**
2
+ * Reading a working tree's diff without letting it read a secret or run away.
3
+ *
4
+ * Three protections, all of them caps that name themselves when they fire:
5
+ *
6
+ * - **the pathspec** never even asks git for `.env*`, `*secret*`, `*credential*`,
7
+ * `*private-key*`, `*.pem` or `*.key`; those files are not read, not skipped.
8
+ * - **`MAX_GIT_DIFF_BYTES`** stops buffering at 2 MiB and kills the child once
9
+ * more than that has been observed, so a giant rebase cannot exhaust memory.
10
+ * - **`GIT_DIFF_TIMEOUT_MS`** kills a diff that hangs.
11
+ *
12
+ * Either cap yields a *truncated* result rather than a failure: the bytes that
13
+ * were captured are real evidence, and the truncation reason travels with them.
14
+ */
15
+ import { spawn } from "node:child_process";
16
+ export const MAX_GIT_DIFF_BYTES = 2 * 1024 * 1024;
17
+ export const GIT_DIFF_TIMEOUT_MS = 3_000;
18
+ /** Paths Cockpit never reads, excluded at the git level rather than after. */
19
+ const SECRET_EXCLUDING_PATHSPEC = [
20
+ ".",
21
+ ":(exclude).env",
22
+ ":(exclude).env.*",
23
+ ":(exclude)**/.env",
24
+ ":(exclude)**/.env.*",
25
+ ":(exclude)**/*secret*",
26
+ ":(exclude)**/*credential*",
27
+ ":(exclude)**/*private-key*",
28
+ ":(exclude)**/*.pem",
29
+ ":(exclude)**/*.key",
30
+ ];
31
+ export async function runGitDiff(args, repoRoot) {
32
+ return new Promise((resolve, reject) => {
33
+ const child = spawn("git", [...args, ...SECRET_EXCLUDING_PATHSPEC], {
34
+ cwd: repoRoot,
35
+ stdio: ["ignore", "pipe", "pipe"],
36
+ });
37
+ const stdoutChunks = [];
38
+ const stderrChunks = [];
39
+ let observedBytes = 0;
40
+ let includedBytes = 0;
41
+ let truncated = false;
42
+ let timedOut = false;
43
+ const timeout = setTimeout(() => {
44
+ timedOut = true;
45
+ truncated = true;
46
+ child.kill("SIGTERM");
47
+ }, GIT_DIFF_TIMEOUT_MS);
48
+ child.stdout.on("data", (chunk) => {
49
+ observedBytes += chunk.byteLength;
50
+ if (includedBytes < MAX_GIT_DIFF_BYTES) {
51
+ const next = chunk.subarray(0, MAX_GIT_DIFF_BYTES - includedBytes);
52
+ stdoutChunks.push(next);
53
+ includedBytes += next.byteLength;
54
+ }
55
+ if (observedBytes > MAX_GIT_DIFF_BYTES) {
56
+ truncated = true;
57
+ child.kill("SIGTERM");
58
+ }
59
+ });
60
+ // Only enough stderr to name a failure; never enough to hold content.
61
+ child.stderr.on("data", (chunk) => {
62
+ if (stderrChunks.reduce((sum, item) => sum + item.byteLength, 0) < 4096) {
63
+ stderrChunks.push(chunk.subarray(0, 4096));
64
+ }
65
+ });
66
+ child.on("error", (error) => {
67
+ clearTimeout(timeout);
68
+ reject(error);
69
+ });
70
+ child.on("close", (code, signal) => {
71
+ clearTimeout(timeout);
72
+ // A cap that fired killed the child on purpose, so SIGTERM and a
73
+ // non-zero code are both success here, not failure.
74
+ if (code === 0 || truncated || signal === "SIGTERM") {
75
+ resolve({
76
+ stdout: Buffer.concat(stdoutChunks).toString("utf8"),
77
+ truncated,
78
+ observedBytes,
79
+ truncationReason: timedOut
80
+ ? "git_diff_timeout"
81
+ : "max_git_diff_bytes",
82
+ truncationCapType: timedOut ? "timeout_ms" : "max_bytes_per_diff",
83
+ });
84
+ return;
85
+ }
86
+ const stderr = Buffer.concat(stderrChunks).toString("utf8").trim();
87
+ reject(new Error(stderr || `git diff failed with code ${code ?? signal}`));
88
+ });
89
+ });
90
+ }
@@ -0,0 +1,92 @@
1
+ /**
2
+ * What a piece of raw evidence is called.
3
+ *
4
+ * Two naming jobs live here and nothing else:
5
+ *
6
+ * 1. **Hashing** — `sha256` names bytes, `shortHash` names a string in twelve
7
+ * hex characters. Content hashes are how a pack recognises bytes it has
8
+ * already staged, so they must never depend on a clock or a path.
9
+ * 2. **Remote object keys** — the readable `operators/…/repos/…/sessions/…`
10
+ * namespace an operator reads during an incident, followed by an immutable
11
+ * content address (or a pack-relative manifest path).
12
+ *
13
+ * Everything here is pure: same input, same name, on every machine and every
14
+ * platform. Nothing in this file reads a file, spawns a process or logs.
15
+ */
16
+ import { SECRET_FILE_SEGMENT_PATTERN } from "@bli-cockpit/telemetry-core";
17
+ import crypto from "node:crypto";
18
+ import path from "node:path";
19
+ export function sha256(value) {
20
+ return crypto.createHash("sha256").update(value).digest("hex");
21
+ }
22
+ export function shortHash(value) {
23
+ return crypto
24
+ .createHash("sha256")
25
+ .update(value, "utf8")
26
+ .digest("hex")
27
+ .slice(0, 12);
28
+ }
29
+ /**
30
+ * Object keys must satisfy the server's key pattern; ids derived from file
31
+ * content fall back to a hash rather than failing the whole upload batch.
32
+ */
33
+ export function safeKeySegment(value) {
34
+ return /^[A-Za-z0-9._-]{1,80}$/.test(value) ? value : shortHash(value);
35
+ }
36
+ export function isSecretLikePath(value) {
37
+ return SECRET_FILE_SEGMENT_PATTERN.test(value);
38
+ }
39
+ /**
40
+ * Raw evidence keys start with human-readable context, then end in immutable
41
+ * content addresses or pack-relative manifest paths. The local cursor reuses
42
+ * prior content hashes across syncs; the readable date/session folders are for
43
+ * operator debugging and incident response.
44
+ */
45
+ export function remoteObjectKey(options) {
46
+ const namespace = readableEvidenceNamespace(options.context);
47
+ if (options.contentAddress) {
48
+ return posixPath([...namespace, options.contentAddress]);
49
+ }
50
+ return posixPath([...namespace, options.packId, options.relativePath]);
51
+ }
52
+ function readableEvidenceNamespace(context) {
53
+ return [
54
+ "operators",
55
+ operatorSlug(context),
56
+ "repos",
57
+ readableKeySegment(context.repoLabel ?? path.basename(context.repoRoot), "repo"),
58
+ "worktrees",
59
+ readableKeySegment(context.worktreeLabel ?? path.basename(context.repoRoot), "worktree"),
60
+ "tickets",
61
+ readableKeySegment(context.activeTicketId ?? "unbound", "unbound", {
62
+ lowercase: false,
63
+ }),
64
+ "dates",
65
+ context.now.toISOString().slice(0, 10),
66
+ "sessions",
67
+ readableKeySegment(context.sessionId, "session"),
68
+ "ids",
69
+ safeKeySegment(context.operatorId),
70
+ safeKeySegment(context.workContextId),
71
+ ];
72
+ }
73
+ function operatorSlug(context) {
74
+ const labelBeforeDomain = (context.operatorLabel ?? context.operatorId)
75
+ .split("@", 1)[0]
76
+ .trim();
77
+ const readable = readableKeySegment(labelBeforeDomain, "operator");
78
+ return `${readable}-${shortHash(context.operatorId).slice(0, 6)}`;
79
+ }
80
+ function readableKeySegment(value, fallback, options = {}) {
81
+ const base = options.lowercase === false ? value : value.toLowerCase();
82
+ const slug = base
83
+ .trim()
84
+ .replace(/[^A-Za-z0-9._-]+/g, "-")
85
+ .replace(/^-+|-+$/g, "")
86
+ .replace(/-{2,}/g, "-")
87
+ .slice(0, 80);
88
+ return slug || fallback;
89
+ }
90
+ function posixPath(parts) {
91
+ return parts.join("/").replace(/\\/g, "/").replace(/\/+/g, "/");
92
+ }
@@ -0,0 +1,132 @@
1
+ import path from "node:path";
2
+ import { remoteObjectKey, sha256 } from "./raw-evidence-keys.js";
3
+ export const RAW_EVIDENCE_BUCKET = "ambient-raw-evidence";
4
+ export const RAW_EVIDENCE_RETENTION_MODE = "remote_durable";
5
+ export function evidenceEntry(options) {
6
+ const digest = sha256(options.bytes);
7
+ const objectKey = remoteObjectKey({
8
+ context: options.context,
9
+ packId: options.packId,
10
+ relativePath: options.relativePath,
11
+ contentAddress: options.contentAddress,
12
+ });
13
+ return {
14
+ kind: options.kind,
15
+ local_path: options.localPath,
16
+ relative_path: options.relativePath,
17
+ object_key: objectKey,
18
+ content_hash_sha256: digest,
19
+ byte_size: options.bytes.byteLength,
20
+ media_type: options.mediaType,
21
+ redacted_summary: options.redactedSummary,
22
+ ...(options.redaction ? { redaction: options.redaction } : {}),
23
+ codex_session_id: options.codexSessionId ?? null,
24
+ staged_in_pack: options.stagedInPack !== false,
25
+ source_key: options.sourceKey ?? null,
26
+ ...(options.artifactMetadata
27
+ ? {
28
+ artifact_metadata: {
29
+ ...options.artifactMetadata,
30
+ raw_evidence_pointer_id: objectKey,
31
+ storage_bucket: RAW_EVIDENCE_BUCKET,
32
+ object_key: objectKey,
33
+ content_hash_sha256: digest,
34
+ byte_size: options.bytes.byteLength,
35
+ },
36
+ }
37
+ : {}),
38
+ };
39
+ }
40
+ export function pointerFromEntry(entry) {
41
+ return {
42
+ raw_evidence_pointer_id: entry.object_key,
43
+ privacy_classification: "remote_durable_raw_evidence",
44
+ retention_policy: {
45
+ mode: RAW_EVIDENCE_RETENTION_MODE,
46
+ privacy_classification: "remote_durable_raw_evidence",
47
+ },
48
+ storage_scope: "remote_object",
49
+ storage_bucket: RAW_EVIDENCE_BUCKET,
50
+ object_key: entry.object_key,
51
+ content_hash_sha256: entry.content_hash_sha256,
52
+ byte_size: entry.byte_size,
53
+ media_type: entry.media_type,
54
+ redacted_summary: entry.redacted_summary,
55
+ ...(entry.redaction ? { redaction: entry.redaction } : {}),
56
+ };
57
+ }
58
+ export function makeManifest(options) {
59
+ return {
60
+ schema: "bli.local_raw_evidence_pack.v1",
61
+ pack_id: options.packId,
62
+ created_at: options.context.now.toISOString(),
63
+ work_context_id: options.context.workContextId,
64
+ session_id: options.context.sessionId,
65
+ operator_id: options.context.operatorId,
66
+ operator_label: options.context.operatorLabel,
67
+ repo_label: options.context.repoLabel ?? path.basename(options.context.repoRoot),
68
+ worktree_label: options.context.worktreeLabel,
69
+ active_ticket_id: options.context.activeTicketId ?? null,
70
+ repo_basename: path.basename(options.context.repoRoot),
71
+ branch: options.context.branch,
72
+ storage_bucket: RAW_EVIDENCE_BUCKET,
73
+ raw_policy: {
74
+ raw_prompts: "preserved_private_durable_remote",
75
+ raw_responses: "preserved_private_durable_remote",
76
+ transcripts: "preserved_private_durable_remote",
77
+ claude_transcripts: "preserved_private_durable_remote",
78
+ tool_payloads: "preserved_private_durable_remote",
79
+ git_diffs: "preserved_private_durable_remote_env_secret_paths_excluded",
80
+ agent_image_attachments: "preserved_private_durable_remote_explicit_agent_session_attachment_only",
81
+ env_files: "never_read",
82
+ stdout: "manifest_only_no_raw_content",
83
+ },
84
+ files: options.entries.map(redactManifestEntry),
85
+ skipped: options.skipped,
86
+ redacted: options.redacted,
87
+ reused: options.reused,
88
+ };
89
+ }
90
+ // `local_path` is redacted out, and so are the staging bookkeeping fields:
91
+ // where a copy happens to live on this disk is not part of the pack's identity,
92
+ // and putting it in the manifest would make byte-identical content produce
93
+ // different manifests.
94
+ function redactManifestEntry(entry) {
95
+ return {
96
+ kind: entry.kind,
97
+ relative_path: entry.relative_path,
98
+ object_key: entry.object_key,
99
+ content_hash_sha256: entry.content_hash_sha256,
100
+ byte_size: entry.byte_size,
101
+ media_type: entry.media_type,
102
+ redacted_summary: entry.redacted_summary,
103
+ ...(entry.redaction ? { redaction: entry.redaction } : {}),
104
+ ...(entry.artifact_metadata
105
+ ? { artifact_metadata: entry.artifact_metadata }
106
+ : {}),
107
+ };
108
+ }
109
+ /**
110
+ * Does a manifest already on disk describe exactly this set of files?
111
+ *
112
+ * Compared by sorted content hash, not by order or by name: the question is
113
+ * whether the reused pack's manifest is still true, and an unreadable or
114
+ * unparseable manifest answers no.
115
+ */
116
+ export function manifestDescribesEntries(manifestBytes, entries) {
117
+ try {
118
+ const parsed = JSON.parse(manifestBytes.toString("utf8"));
119
+ const recorded = (parsed.files ?? [])
120
+ .map((file) => typeof file.content_hash_sha256 === "string"
121
+ ? file.content_hash_sha256
122
+ : "")
123
+ .filter(Boolean)
124
+ .sort();
125
+ const expected = entries.map((entry) => entry.content_hash_sha256).sort();
126
+ return (recorded.length === expected.length &&
127
+ recorded.every((hash, index) => hash === expected[index]));
128
+ }
129
+ catch {
130
+ return false;
131
+ }
132
+ }
@@ -0,0 +1,136 @@
1
+ /**
2
+ * Where the bytes live on this disk.
3
+ *
4
+ * Staging first, promotion second (BLI-3066). A pack's id is derived from the
5
+ * content in it, so it cannot be known until collection is done: bytes land in
6
+ * a private `.staging-<pid>-<rand>` directory, and only then does the directory
7
+ * become its content-keyed name. Identical content on the next sync resolves to
8
+ * the identical directory instead of a 560th copy.
9
+ *
10
+ * Everything written here is mode `0600` inside a `0700` directory, on every
11
+ * platform except Windows, where the chmod is a no-op.
12
+ */
13
+ import fs from "node:fs/promises";
14
+ import path from "node:path";
15
+ import { makeManifest, manifestDescribesEntries, } from "./raw-evidence-manifest.js";
16
+ import { writeRawEvidenceStagingState, } from "../raw-evidence-staging.js";
17
+ /**
18
+ * Turn the staging directory into the content-keyed pack directory.
19
+ *
20
+ * Three outcomes, all of them named:
21
+ * - the pack does not exist yet: rename staging into place (`new`);
22
+ * - it exists and holds every file this pass staged: adopt it and delete the
23
+ * staging copy (`reused`) — nothing is copied twice;
24
+ * - it exists but is missing files: fill the gaps from staging
25
+ * (`restaged_incomplete`), because a half-written pack must not be trusted.
26
+ *
27
+ * Never deletes an existing pack directory wholesale: another pack's entries
28
+ * can point into it through the staged-object index.
29
+ */
30
+ export async function promoteStagedPack(options) {
31
+ const evidenceDir = path.join(options.rawEvidenceRoot, options.packId);
32
+ const priorPackCount = await countPriorPacks(options.rawEvidenceRoot, options.workContextId, options.packId);
33
+ const existing = await fs.stat(evidenceDir).catch(() => null);
34
+ if (!existing?.isDirectory()) {
35
+ try {
36
+ await fs.rename(options.stagingDir, evidenceDir);
37
+ return {
38
+ state: "new",
39
+ evidenceDir,
40
+ priorPackCount,
41
+ refilledFileCount: 0,
42
+ };
43
+ }
44
+ catch {
45
+ // A concurrent sync can win the race to the same content-keyed name.
46
+ // Losing it is fine: the winner staged the identical bytes.
47
+ }
48
+ }
49
+ const refilledFileCount = await refillMissingPackFiles(evidenceDir, options.entries);
50
+ await fs
51
+ .rm(options.stagingDir, { recursive: true, force: true })
52
+ .catch(() => undefined);
53
+ return {
54
+ state: refilledFileCount > 0 ? "restaged_incomplete" : "reused",
55
+ evidenceDir,
56
+ priorPackCount,
57
+ refilledFileCount,
58
+ };
59
+ }
60
+ /** A file already there at the right size is left alone; anything else is recopied. */
61
+ async function refillMissingPackFiles(evidenceDir, entries) {
62
+ let refilledFileCount = 0;
63
+ await ensurePrivateDir(path.join(evidenceDir, "files"));
64
+ for (const entry of entries) {
65
+ if (!entry.staged_in_pack)
66
+ continue;
67
+ const target = path.join(evidenceDir, "files", path.basename(entry.local_path));
68
+ const info = await fs.stat(target).catch(() => null);
69
+ if (info?.isFile() && info.size === entry.byte_size)
70
+ continue;
71
+ await fs.copyFile(entry.local_path, target).catch(() => undefined);
72
+ await chmodPrivate(target, 0o600);
73
+ refilledFileCount += 1;
74
+ }
75
+ return refilledFileCount;
76
+ }
77
+ async function countPriorPacks(rawEvidenceRoot, workContextId, packId) {
78
+ const entries = await fs
79
+ .readdir(rawEvidenceRoot, { withFileTypes: true })
80
+ .catch(() => []);
81
+ return entries.filter((entry) => entry.isDirectory() &&
82
+ entry.name !== packId &&
83
+ entry.name.startsWith(`${workContextId}-`)).length;
84
+ }
85
+ /**
86
+ * Write the manifest, or keep the one already in a reused pack.
87
+ *
88
+ * Byte-stability matters here: the manifest's object key is the only key that
89
+ * embeds the pack id, so rewriting it with a fresh `created_at` every sync
90
+ * would push different bytes at the same content-addressed key forever. A
91
+ * reused pack whose manifest already describes exactly these files keeps it.
92
+ */
93
+ export async function stageManifest(options) {
94
+ if (options.reusePack) {
95
+ const existing = await fs.readFile(options.manifestPath).catch(() => null);
96
+ if (existing && manifestDescribesEntries(existing, options.entries)) {
97
+ return existing;
98
+ }
99
+ }
100
+ const manifestBytes = Buffer.from(`${JSON.stringify(makeManifest({
101
+ context: options.context,
102
+ packId: options.packId,
103
+ entries: options.entries,
104
+ skipped: options.skipped,
105
+ redacted: options.redacted,
106
+ reused: options.reused,
107
+ }), null, 2)}\n`, "utf8");
108
+ await fs.writeFile(options.manifestPath, manifestBytes, { mode: 0o600 });
109
+ await chmodPrivate(options.manifestPath, 0o600);
110
+ return manifestBytes;
111
+ }
112
+ /**
113
+ * Record which content hashes are staged where, so the next sync can adopt the
114
+ * copy instead of writing it again. Losing this file costs reuse and attempt
115
+ * counts, not evidence, so it must never fail a collection — but it must never
116
+ * be silent either.
117
+ */
118
+ export async function persistStagingState(stateDir, staging, nowIso) {
119
+ staging.updated_at = nowIso;
120
+ await writeRawEvidenceStagingState(stateDir, staging).catch((error) => {
121
+ console.error("[raw-evidence] staging state write failed", JSON.stringify({
122
+ reason: "staging_state_write_failed",
123
+ detail: error instanceof Error ? error.name : typeof error,
124
+ staged_count: Object.keys(staging.staged).length,
125
+ }));
126
+ });
127
+ }
128
+ export async function ensurePrivateDir(dir) {
129
+ await fs.mkdir(dir, { recursive: true, mode: 0o700 });
130
+ await chmodPrivate(dir, 0o700);
131
+ }
132
+ export async function chmodPrivate(target, mode) {
133
+ if (process.platform === "win32")
134
+ return;
135
+ await fs.chmod(target, mode).catch(() => undefined);
136
+ }