@bli-cockpit/cli 0.2.251 → 0.2.253

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,85 @@
1
+ import { captureSessionFactsDelta, observeSessionFactsContent, observeSessionFactsFile, parseSessionFactsDelta, SESSION_FACTS_VERSION, SKILL_USE_VERSION, } from "@bli-cockpit/telemetry-core";
2
+ import crypto from "node:crypto";
3
+ import fs from "node:fs/promises";
4
+ import { extractClaudeSessionSignals } from "./claude-attribution-signals.js";
5
+ import { statSourceForMemo } from "./transcript-read-memo.js";
6
+ /**
7
+ * The reads attribution does of one Claude file, remembered by the file's stamp (BLI-22797;
8
+ * `transcript-read-memo.ts`). Every read goes through the same capture-then-merge path whether the file was
9
+ * opened or recalled, so a recalled file leaves the session's facts exactly as a fresh read does by construction
10
+ * (`session-facts-delta.ts`). Bump SIGNALS_VERSION when `claude-attribution-signals.ts` changes what it extracts.
11
+ */
12
+ const SIGNALS_VERSION = "signals.v1";
13
+ export const ATTRIBUTION_READ_RECIPE = ["attribution-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION].join("|");
14
+ const sha256 = (value) => crypto.createHash("sha256").update(value).digest("hex");
15
+ const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
16
+ const isNum = (v) => typeof v === "number" && Number.isFinite(v);
17
+ const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
18
+ function parseSidecarRead(data) {
19
+ if (!isObj(data) || !isStrings(data["cwds"]) || !isNum(data["byte_size"]) || typeof data["sha256"] !== "string")
20
+ return null;
21
+ const facts = parseSessionFactsDelta(data["facts"]);
22
+ return facts ? { cwds: data["cwds"], byte_size: data["byte_size"], sha256: data["sha256"], facts } : null;
23
+ }
24
+ function parseMainRead(data) {
25
+ if (!isObj(data) || !isNum(data["byte_size"]) || typeof data["sha256"] !== "string" || !isObj(data["signals"]))
26
+ return null;
27
+ const s = data["signals"];
28
+ if (!isStrings(s["session_ids"]) || !isStrings(s["cwds"]) || !isStrings(s["branches"]) || !isStrings(s["repository_urls"]))
29
+ return null;
30
+ if (!isNum(s["line_count"]) || !isNum(s["parse_error_count"]) || !isNum(s["content_record_count"]) || !isNum(s["envelope_field_hit_rate"]))
31
+ return null;
32
+ const facts = parseSessionFactsDelta(data["facts"]);
33
+ return facts ? { signals: s, byte_size: data["byte_size"], sha256: data["sha256"], facts } : null;
34
+ }
35
+ /** The sidecar's cwds, size, hash and facts contribution: recalled when its stamp is unchanged, else read (a read failure throws). */
36
+ export async function readSidecar(memo, localPath) {
37
+ const stamp = memo ? await statSourceForMemo(localPath) : null;
38
+ if (memo && stamp) {
39
+ const hit = memo.recall(localPath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseSidecarRead);
40
+ if (hit)
41
+ return hit;
42
+ }
43
+ else
44
+ memo?.note("stamp_missing");
45
+ const raw = await fs.readFile(localPath);
46
+ const content = raw.toString("utf8");
47
+ const read = {
48
+ cwds: extractClaudeSessionSignals(content).cwds,
49
+ byte_size: raw.byteLength,
50
+ sha256: sha256(raw),
51
+ facts: captureSessionFactsDelta("claude_code", (acc) => void observeSessionFactsContent(acc, content)),
52
+ };
53
+ if (memo && stamp) {
54
+ memo.note("fresh");
55
+ memo.remember(localPath, stamp, ATTRIBUTION_READ_RECIPE, read);
56
+ }
57
+ return read;
58
+ }
59
+ /** The main transcript's signals, size, hash and facts contribution. `{ empty: true }` when it holds only whitespace (never remembered). */
60
+ export async function readMain(memo, filePath) {
61
+ const stamp = memo ? await statSourceForMemo(filePath) : null;
62
+ if (memo && stamp) {
63
+ const hit = memo.recall(filePath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseMainRead);
64
+ if (hit)
65
+ return hit;
66
+ }
67
+ else
68
+ memo?.note("stamp_missing");
69
+ const raw = await fs.readFile(filePath);
70
+ const content = raw.toString("utf8");
71
+ const hash = sha256(raw);
72
+ if (content.trim() === "")
73
+ return { empty: true, byte_size: raw.byteLength, sha256: hash };
74
+ let signals;
75
+ const facts = captureSessionFactsDelta("claude_code", (acc) => {
76
+ signals = extractClaudeSessionSignals(content, acc);
77
+ observeSessionFactsFile(acc, { bytes: raw.byteLength, lines: signals.line_count });
78
+ });
79
+ const read = { signals: signals, byte_size: raw.byteLength, sha256: hash, facts };
80
+ if (memo && stamp) {
81
+ memo.note("fresh");
82
+ memo.remember(filePath, stamp, ATTRIBUTION_READ_RECIPE, read);
83
+ }
84
+ return read;
85
+ }
@@ -1,12 +1,11 @@
1
- import { finalizeSessionFacts, newSessionFactsAccumulator, observeSessionFactsContent, observeSessionFactsFile, } from "@bli-cockpit/telemetry-core";
2
- import crypto from "node:crypto";
1
+ import { finalizeSessionFacts, mergeSessionFactsDelta, newSessionFactsAccumulator, observeSessionFactsFile, } from "@bli-cockpit/telemetry-core";
3
2
  import { existsSync } from "node:fs";
4
- import fs from "node:fs/promises";
5
3
  import path from "node:path";
6
4
  import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
7
5
  import { describeError } from "../health-detail.js";
8
6
  import { isPathWithin, sanitizeSessionId, scoreSignalsAgainstWorktrees, sessionIdFromFileName, shortHash, } from "./attribution-core.js";
9
- import { extractClaudeSessionSignals, isSchemaDriftSuspected, streamMainSignals, } from "./claude-attribution-signals.js";
7
+ import { isSchemaDriftSuspected, streamMainSignals } from "./claude-attribution-signals.js";
8
+ import { readMain, readSidecar } from "./claude-attribution-memoized-reads.js";
10
9
  import { CLAUDE_SESSION_MAX_FILE_BYTES, } from "./claude-attribution-types.js";
11
10
  /**
12
11
  * Turns one discovered session into an attribution verdict: read (or, for an
@@ -18,7 +17,7 @@ import { CLAUDE_SESSION_MAX_FILE_BYTES, } from "./claude-attribution-types.js";
18
17
  * are counted: a sidecar rejected for a cwd mismatch is another worktree's
19
18
  * work and must not inflate this session's numbers.
20
19
  */
21
- export async function attributeOneSession(session, worktrees, collectionRoots) {
20
+ export async function attributeOneSession(session, worktrees, collectionRoots, readMemo) {
22
21
  const factsAcc = newSessionFactsAccumulator("claude_code");
23
22
  const base = createSessionBase(session);
24
23
  if (session.mainByteSize === 0) {
@@ -26,7 +25,7 @@ export async function attributeOneSession(session, worktrees, collectionRoots) {
26
25
  }
27
26
  const read = session.mainByteSize > CLAUDE_SESSION_MAX_FILE_BYTES
28
27
  ? await streamOversizedMain(session, base, factsAcc)
29
- : await readMainSignals(session, base, factsAcc);
28
+ : await readMainSignals(session, base, factsAcc, readMemo);
30
29
  if (typeof read === "string")
31
30
  return skippedResult(base, read);
32
31
  const signals = read;
@@ -54,7 +53,7 @@ export async function attributeOneSession(session, worktrees, collectionRoots) {
54
53
  extraSignals.push("main_file_oversized");
55
54
  if (isSchemaDriftSuspected(signals))
56
55
  extraSignals.push("schema_drift_suspected");
57
- const sidecarFiles = await collectSessionSidecars(session, outcome, factsAcc);
56
+ const sidecarFiles = await collectSessionSidecars(session, outcome, factsAcc, readMemo);
58
57
  return {
59
58
  ...base,
60
59
  session_facts: finalizeSessionFacts(factsAcc),
@@ -116,10 +115,10 @@ async function streamOversizedMain(session, base, factsAcc) {
116
115
  base.oversized_lines_skipped = streamed.oversizedLinesSkipped;
117
116
  return streamed.signals;
118
117
  }
119
- async function readMainSignals(session, base, factsAcc) {
120
- let raw;
118
+ async function readMainSignals(session, base, factsAcc, readMemo) {
119
+ let read;
121
120
  try {
122
- raw = await fs.readFile(session.mainFile);
121
+ read = await readMain(readMemo, session.mainFile);
123
122
  }
124
123
  catch (error) {
125
124
  console.error("[claude-attribution] session file unreadable, skipping it", JSON.stringify({
@@ -130,18 +129,12 @@ async function readMainSignals(session, base, factsAcc) {
130
129
  }));
131
130
  return "file_read_failed";
132
131
  }
133
- const content = raw.toString("utf8");
134
- base.content_hash_sha256 = sha256(raw);
135
- base.byte_size = raw.byteLength;
136
- if (content.trim() === "") {
132
+ base.content_hash_sha256 = read.sha256;
133
+ base.byte_size = read.byte_size;
134
+ if ("empty" in read)
137
135
  return "empty_file";
138
- }
139
- const signals = extractClaudeSessionSignals(content, factsAcc);
140
- observeSessionFactsFile(factsAcc, {
141
- bytes: raw.byteLength,
142
- lines: signals.line_count,
143
- });
144
- return signals;
136
+ mergeSessionFactsDelta(factsAcc, read.facts);
137
+ return read.signals;
145
138
  }
146
139
  function applySessionMetadata(base, signals) {
147
140
  const metaSessionId = sanitizeSessionId(signals.session_ids[0]);
@@ -155,9 +148,9 @@ function applySessionMetadata(base, signals) {
155
148
  base.cwd_hash = shortHash(primaryCwd);
156
149
  }
157
150
  }
158
- async function collectSessionSidecars(session, outcome, factsAcc) {
151
+ async function collectSessionSidecars(session, outcome, factsAcc, readMemo) {
159
152
  return isRawEvidenceUploadableAttributionState(outcome.state, outcome.worktree !== null) && outcome.worktree
160
- ? await collectSidecarDiagnostics(session.sidecars, outcome.worktree, factsAcc)
153
+ ? await collectSidecarDiagnostics(session.sidecars, outcome.worktree, factsAcc, readMemo)
161
154
  : session.sidecars.map((sidecar) => ({
162
155
  local_path: sidecar.local_path,
163
156
  file_name: sidecar.file_name,
@@ -174,7 +167,7 @@ async function collectSessionSidecars(session, outcome, factsAcc) {
174
167
  * oversized sidecar is recorded with a reason and the session stays
175
168
  * harvestable.
176
169
  */
177
- async function collectSidecarDiagnostics(sidecars, worktree, facts) {
170
+ async function collectSidecarDiagnostics(sidecars, worktree, facts, readMemo) {
178
171
  const out = [];
179
172
  for (const sidecar of sidecars) {
180
173
  const entry = {
@@ -188,9 +181,9 @@ async function collectSidecarDiagnostics(sidecars, worktree, facts) {
188
181
  out.push({ ...entry, skipped_reason: "file_too_large" });
189
182
  continue;
190
183
  }
191
- let raw;
184
+ let read;
192
185
  try {
193
- raw = await fs.readFile(sidecar.local_path);
186
+ read = await readSidecar(readMemo, sidecar.local_path);
194
187
  }
195
188
  catch (error) {
196
189
  // `file_read_failed` stays on the sidecar row. Beside it: a subagent
@@ -205,8 +198,7 @@ async function collectSidecarDiagnostics(sidecars, worktree, facts) {
205
198
  out.push({ ...entry, skipped_reason: "file_read_failed" });
206
199
  continue;
207
200
  }
208
- const content = raw.toString("utf8");
209
- const sidecarCwds = extractClaudeSessionSignals(content).cwds;
201
+ const sidecarCwds = read.cwds;
210
202
  if (sidecarCwds.length > 0 &&
211
203
  !sidecarCwds.some((cwd) => isPathWithin(cwd, worktree.repo_root))) {
212
204
  out.push({ ...entry, skipped_reason: "sidecar_cwd_mismatch" });
@@ -214,11 +206,11 @@ async function collectSidecarDiagnostics(sidecars, worktree, facts) {
214
206
  }
215
207
  // Accepted: this subagent's turns are part of this session, so its tokens
216
208
  // are too. Counted only now, after the cwd check has cleared it.
217
- observeSessionFactsContent(facts, content);
209
+ mergeSessionFactsDelta(facts, read.facts);
218
210
  out.push({
219
211
  ...entry,
220
- byte_size: raw.byteLength,
221
- content_hash_sha256: sha256(raw),
212
+ byte_size: read.byte_size,
213
+ content_hash_sha256: read.sha256,
222
214
  });
223
215
  }
224
216
  return out;
@@ -233,7 +225,4 @@ function skippedResult(base, reason) {
233
225
  path_score: 0,
234
226
  worktree: null,
235
227
  };
236
- }
237
- function sha256(value) {
238
- return crypto.createHash("sha256").update(value).digest("hex");
239
228
  }
@@ -37,7 +37,7 @@ export async function scanAndAttributeClaudeSessions(options) {
37
37
  .slice(0, limit);
38
38
  const results = [];
39
39
  for (const session of sessions) {
40
- results.push(await attributeOneSession(session, options.worktrees, options.collectionRoots ?? []));
40
+ results.push(await attributeOneSession(session, options.worktrees, options.collectionRoots ?? [], options.readMemo));
41
41
  }
42
42
  const sumBy = (pick) => results.reduce((total, result) => total + pick(result), 0);
43
43
  const countState = (state) => results.filter((result) => result.state === state).length;
@@ -24,6 +24,7 @@
24
24
  * collection is done.
25
25
  */
26
26
  import { openPrepareMemo } from "./raw-evidence-prepare-memo.js";
27
+ import { openTranscriptReadMemo } from "./transcript-read-memo.js";
27
28
  import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
28
29
  import crypto from "node:crypto";
29
30
  import fs from "node:fs/promises";
@@ -65,6 +66,7 @@ export async function openCollection(context, options, places) {
65
66
  budget.images ??= options.images;
66
67
  const stagingAllowance = (budget.staging ??= await openStagingAllowance(options.stateDir, options.env ?? process.env, { statfs: options.statfs }));
67
68
  budget.prepareMemo ??= await openPrepareMemo(options.stateDir);
69
+ budget.readMemo ??= await openTranscriptReadMemo(options.stateDir, "image_scan").catch(() => undefined);
68
70
  return {
69
71
  context,
70
72
  filesDir: path.join(places.stagingDir, "files"),
@@ -9,18 +9,42 @@
9
9
  */
10
10
  import path from "node:path";
11
11
  import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
12
+ import { statSourceForMemo } from "./transcript-read-memo.js";
13
+ import { AGENT_IMAGE_MAX_BYTES } from "./agent-image-validation.js";
12
14
  import { evidenceSourceKey } from "../raw-evidence-staging.js";
13
15
  import { sha256 } from "./raw-evidence-keys.js";
14
16
  import { markBudgetCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
15
17
  import { evidenceEntry } from "./raw-evidence-manifest.js";
16
18
  import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
19
+ /**
20
+ * BLI-22797: a transcript with no image candidate is remembered by its stamp, so the next tick does not open it.
21
+ * Only that answer is remembered: an image whose bytes live in another file can change while the transcript does
22
+ * not, so a transcript that offered images is always read again. Bump the version when
23
+ * `agent-image-records.ts` changes what counts as a candidate.
24
+ */
25
+ const IMAGE_SCAN_RECIPE = ["image-scan.v1", String(AGENT_IMAGE_MAX_BYTES)].join("|");
17
26
  export async function collectAgentImagesFromTranscript(collection, options) {
18
- const result = await collectAgentImageEvidenceFromJsonlFile({
19
- filePath: options.filePath,
20
- source: options.source,
21
- sessionId: options.sessionId,
22
- sidecarId: options.sidecarId,
23
- });
27
+ const memo = collection.budget.readMemo;
28
+ const stamp = memo ? await statSourceForMemo(options.filePath) : null;
29
+ const remembered = memo && stamp
30
+ ? memo.recall(options.filePath, stamp, IMAGE_SCAN_RECIPE, stamp.size, (data) => (data === "no_image_candidates" ? true : null))
31
+ : null;
32
+ if (!remembered && memo && !stamp)
33
+ memo.note("stamp_missing");
34
+ const result = remembered
35
+ ? { images: [], skipped: [] }
36
+ : await collectAgentImageEvidenceFromJsonlFile({
37
+ filePath: options.filePath,
38
+ source: options.source,
39
+ sessionId: options.sessionId,
40
+ sidecarId: options.sidecarId,
41
+ });
42
+ if (!remembered && memo && stamp) {
43
+ memo.note("fresh");
44
+ if (result.images.length === 0 && result.skipped.length === 0) {
45
+ memo.remember(options.filePath, stamp, IMAGE_SCAN_RECIPE, "no_image_candidates");
46
+ }
47
+ }
24
48
  recordScanned(collection, options.kind, result.images.length + result.skipped.length);
25
49
  for (const skipped of result.skipped) {
26
50
  collection.skipped.push({
@@ -13,11 +13,14 @@
13
13
  *
14
14
  * - the file's size, inode and mtime/ctime (nanoseconds) are exactly what they were when the
15
15
  * entry was written, so the file has not been appended to, rewritten or replaced;
16
- * - the recipe is the same: collector version (slim rules and secret rules ship with it), the
17
- * machine's image-handling mode, and whether an image step ran at all;
18
- * - the entry is not `volatile`: a transcript that still held an image with no stored
19
- * description (or whose slim step failed) can change shape on the next tick without the file
20
- * changing, so it is always rebuilt.
16
+ * - the recipe is the same: a fingerprint of the rules that decide the bytes (the slim rules, the
17
+ * secret rules, the image marker format; NOT the CLI version, BLI-22794), the machine's
18
+ * image-handling mode, and whether an image step ran at all;
19
+ * - the entry is not `volatile`, or it is volatile ONLY because of images with no stored
20
+ * description (`pending_images`, BLI-22794). Such an entry is handed back to the caller, who
21
+ * trusts it while none of those images has been described since; a described one rebuilds. A
22
+ * volatile entry without `pending_images` (a failed slim step, or any older entry) is never
23
+ * recalled: it can change shape on the next tick without the file changing.
21
24
  *
22
25
  * The caller then reuses the entry ONLY when the remembered content hash is already durable
23
26
  * remotely (`skipContentHashes`). Anything else, including every file that is not yet delivered,
@@ -25,19 +28,21 @@
25
28
  * known. A memo that is lost, corrupt, or wrong in the safe direction costs one slow tick, never
26
29
  * a missed upload. It holds ids, sizes, hashes and redaction metadata, never a byte of content.
27
30
  */
31
+ import { createHash } from "node:crypto";
28
32
  import path from "node:path";
29
33
  import fs from "node:fs/promises";
30
34
  import { isMissingFileFailure } from "../health-detail.js";
31
35
  import { readJsonFile, writeJsonFileAtomic } from "../local-state-files.js";
32
- import { LOCAL_COLLECTOR_VERSION } from "../local-state-config.js";
36
+ import { evidenceRulesFingerprint } from "@bli-cockpit/telemetry-core";
37
+ import { SLIM_RULES_VERSION } from "../slim/slim-transcript.js";
33
38
  export const PREPARE_MEMO_FILENAME = "transcript-prepare-memo.json";
34
39
  const MEMO_FORMAT = "prepare-memo.v1";
35
40
  /** About 700 bytes an entry, so this is at most a few MB; the oldest leave first. */
36
41
  export const PREPARE_MEMO_MAX_ENTRIES = 20_000;
37
42
  /** A source not seen for this long is a deleted or rotated session. */
38
- const FORGET_AFTER_MS = 14 * 24 * 3_600_000;
43
+ export const FORGET_AFTER_MS = 14 * 24 * 3_600_000;
39
44
  /** Refresh `seen_ms` at most this often, so an unchanged tick does not rewrite the file. */
40
- const SEEN_REFRESH_MS = 24 * 3_600_000;
45
+ export const SEEN_REFRESH_MS = 24 * 3_600_000;
41
46
  const TAG = "[raw-evidence-memo]";
42
47
  export function sourceStampOf(stat) {
43
48
  return {
@@ -54,29 +59,38 @@ export async function statSourceForMemo(filePath) {
54
59
  return null;
55
60
  }
56
61
  }
62
+ /**
63
+ * What the prepared bytes depend on besides the file: the slim rules and the secret and image rules.
64
+ * Computed once per process. NOT the CLI version (BLI-22794): an upgrade that changes none of these
65
+ * keeps the memo warm. Changing a rule changes this, which rebuilds the memo once; that is intended
66
+ * (see `SLIM_RULES_VERSION` and telemetry-core's `evidenceRulesFingerprint`).
67
+ */
68
+ let rulesFingerprint;
69
+ export function rulesFingerprintOf(slimRulesVersion, evidenceRules) {
70
+ return createHash("sha256").update(`${slimRulesVersion}|${evidenceRules}`).digest("hex").slice(0, 32);
71
+ }
57
72
  /** Everything besides the file that decides what its prepared bytes are. */
58
73
  export function prepareRecipe(options) {
59
- return [MEMO_FORMAT, LOCAL_COLLECTOR_VERSION, options.kind, options.imageMode ?? "no_image_step"].join("|");
74
+ rulesFingerprint ??= rulesFingerprintOf(SLIM_RULES_VERSION, evidenceRulesFingerprint());
75
+ return [MEMO_FORMAT, rulesFingerprint, options.kind, options.imageMode ?? "no_image_step"].join("|");
60
76
  }
61
77
  export function prepareMemoPath(stateDir) {
62
78
  return path.join(stateDir, PREPARE_MEMO_FILENAME);
63
79
  }
64
- const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0, volatile_reused_count: 0, volatile_reused_bytes: 0 };
80
+ const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0, volatile_reused_count: 0, volatile_reused_bytes: 0, rebuilt_image_now_described_count: 0 };
65
81
  export async function openPrepareMemo(stateDir) {
66
82
  const entries = await readMemoEntries(stateDir);
67
83
  let dirty = false;
68
84
  let counts = { ...EMPTY_COUNTS };
69
85
  /** Sources recalled since the last `settle`: only these have their `seen_ms` refreshed. */
70
86
  const recalled = new Set();
71
- /** In-memory only, never persisted: see `rememberForThisRun`. */
72
- const thisRun = new Map();
73
87
  return {
74
88
  recall(sourceKey, stamp, recipe) {
75
89
  const entry = entries.get(sourceKey);
76
90
  if (!entry)
77
- return recallThisRun(thisRun, sourceKey, stamp, recipe);
78
- if (entry.volatile)
79
- return recallThisRun(thisRun, sourceKey, stamp, recipe);
91
+ return null;
92
+ if (entry.volatile && !entry.pending_images)
93
+ return null;
80
94
  if (entry.recipe !== recipe)
81
95
  return null;
82
96
  if (entry.size !== stamp.size || entry.identity !== stamp.identity)
@@ -86,13 +100,13 @@ export async function openPrepareMemo(stateDir) {
86
100
  },
87
101
  remember(sourceKey, entry, now = Date.now()) {
88
102
  entries.set(sourceKey, { ...entry, seen_ms: now });
89
- thisRun.delete(sourceKey);
90
103
  dirty = true;
91
104
  },
92
- rememberForThisRun(sourceKey, entry) {
93
- thisRun.set(sourceKey, { ...entry, volatile: true, seen_ms: Date.now() });
94
- },
95
105
  count(outcome, bytes) {
106
+ if (outcome === "rebuilt_image_now_described") {
107
+ counts.rebuilt_image_now_described_count += 1;
108
+ return;
109
+ }
96
110
  if (outcome === "volatile_reused") {
97
111
  counts.volatile_reused_count += 1;
98
112
  counts.volatile_reused_bytes += bytes;
@@ -127,20 +141,12 @@ export async function openPrepareMemo(stateDir) {
127
141
  },
128
142
  };
129
143
  }
130
- function recallThisRun(thisRun, sourceKey, stamp, recipe) {
131
- const entry = thisRun.get(sourceKey);
132
- if (!entry || entry.recipe !== recipe)
133
- return null;
134
- if (entry.size !== stamp.size || entry.identity !== stamp.identity)
135
- return null;
136
- return entry;
137
- }
138
144
  /**
139
145
  * Entries recalled this pass keep their `seen_ms` fresh, lazily, so they are not forgotten while in
140
146
  * use. Only those: an entry nobody recalls (a deleted or rotated session) ages out after
141
147
  * FORGET_AFTER_MS.
142
148
  */
143
- function refreshSeen(entries, recalled, now) {
149
+ export function refreshSeen(entries, recalled, now) {
144
150
  let touched = false;
145
151
  for (const key of recalled) {
146
152
  const entry = entries.get(key);
@@ -151,13 +157,13 @@ function refreshSeen(entries, recalled, now) {
151
157
  }
152
158
  return touched;
153
159
  }
154
- function trimEntries(entries, now) {
160
+ export function trimEntries(entries, now, maxEntries = PREPARE_MEMO_MAX_ENTRIES) {
155
161
  for (const [key, entry] of entries) {
156
162
  if (now - entry.seen_ms > FORGET_AFTER_MS)
157
163
  entries.delete(key);
158
164
  }
159
165
  const newestFirst = [...entries.entries()].sort((a, b) => b[1].seen_ms - a[1].seen_ms);
160
- const kept = newestFirst.slice(0, PREPARE_MEMO_MAX_ENTRIES);
166
+ const kept = newestFirst.slice(0, maxEntries);
161
167
  if (kept.length < entries.size) {
162
168
  entries.clear();
163
169
  for (const [key, entry] of kept)
@@ -205,6 +211,9 @@ function parseEntry(value) {
205
211
  return null;
206
212
  if (typeof entry["volatile"] !== "boolean" || typeof entry["seen_ms"] !== "number")
207
213
  return null;
214
+ const pending = entry["pending_images"];
215
+ if (pending !== undefined && (!Array.isArray(pending) || !pending.every((item) => typeof item === "string")))
216
+ return null;
208
217
  const redacted = entry["redacted"];
209
218
  if (redacted !== null && (typeof redacted !== "object" || typeof redacted.completenessLabel !== "string"))
210
219
  return null;
@@ -26,6 +26,7 @@ import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
26
26
  import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
27
27
  import { evidenceEntry } from "./raw-evidence-manifest.js";
28
28
  import { prepareRecipe, statSourceForMemo, } from "./raw-evidence-prepare-memo.js";
29
+ import { logVolatileReuse, trustVolatileEntry } from "./raw-evidence-volatile-reuse.js";
29
30
  import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
30
31
  import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
31
32
  /**
@@ -63,9 +64,9 @@ export async function collectOneEvidenceFile(collection, options) {
63
64
  markCapApplied(collection, options.kind, "max_file_bytes");
64
65
  return skip("file_too_large");
65
66
  }
66
- const { sanitized, contentHash, stubClaim, volatile, deferredOnly, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
67
+ const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
67
68
  await rememberPreparedTranscript(collection, options, names, {
68
- stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, deferredOnly, redactedEntry,
69
+ stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
69
70
  });
70
71
  if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
71
72
  collection.reused.push({
@@ -121,7 +122,7 @@ async function prepareTranscriptForUpload(collection, options, names, bytes) {
121
122
  byteSize: sanitized.bytes.byteLength,
122
123
  alreadyDurable: collection.skipContentHashes.has(contentHash),
123
124
  });
124
- return { sanitized, contentHash, stubClaim, volatile: images.volatile, deferredOnly: images.deferredOnly, redactedEntry };
125
+ return { sanitized, contentHash, stubClaim, volatile: images.volatile, pendingImages: images.pendingImages, redactedEntry };
125
126
  }
126
127
  /**
127
128
  * The transcript bytes after the image step. Unchanged (the very same buffer) unless this sync has
@@ -133,10 +134,10 @@ async function prepareEmbeddedImages(collection, file) {
133
134
  const session = collection.budget.images;
134
135
  const source = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
135
136
  if (!session || !source)
136
- return { bytes: file.bytes, volatile: false, deferredOnly: false };
137
+ return { bytes: file.bytes, volatile: false, pendingImages: null };
137
138
  // Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
138
139
  if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.bytes))
139
- return { bytes: file.bytes, volatile: false, deferredOnly: false };
140
+ return { bytes: file.bytes, volatile: false, pendingImages: null };
140
141
  try {
141
142
  const prepared = await session.prepareTranscript({
142
143
  text: file.bytes.toString("utf8"),
@@ -151,15 +152,16 @@ async function prepareEmbeddedImages(collection, file) {
151
152
  : prepared.imageCount > 0;
152
153
  const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
153
154
  const volatile = imagesPending || slimFailed;
154
- // BLI-22785: volatile ONLY because the budget or a backoff held images back, and the slim step held up.
155
- const deferredOnly = volatile && prepared.deferredOnly && !slimFailed;
155
+ // BLI-22794: volatile ONLY because of images without a stored description (the slim step held
156
+ // up): the bytes change only when one of these gains one, so the memo may keep the entry.
157
+ const pendingImages = volatile && !slimFailed ? prepared.pendingImages : null;
156
158
  if (!prepared.changed)
157
- return { bytes: file.bytes, volatile, deferredOnly };
158
- return { bytes: Buffer.from(prepared.text, "utf8"), volatile, deferredOnly };
159
+ return { bytes: file.bytes, volatile, pendingImages };
160
+ return { bytes: Buffer.from(prepared.text, "utf8"), volatile, pendingImages };
159
161
  }
160
162
  catch (error) {
161
163
  console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
162
- return { bytes: file.bytes, volatile: true, deferredOnly: false };
164
+ return { bytes: file.bytes, volatile: true, pendingImages: null };
163
165
  }
164
166
  }
165
167
  /**
@@ -182,6 +184,8 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
182
184
  const known = memo.recall(names.sourceKey, stamp, recipe);
183
185
  if (!known)
184
186
  return false;
187
+ if (known.volatile && !trustVolatileEntry(collection, memo, known, options.kind, names.sourceKey, stamp.size))
188
+ return false;
185
189
  if (!collection.skipContentHashes.has(known.content_hash)) {
186
190
  if (known.volatile)
187
191
  logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "hash_not_durable_full_path");
@@ -202,24 +206,21 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
202
206
  codex_session_id: options.sessionId,
203
207
  });
204
208
  if (known.volatile) {
205
- // BLI-22785: images held back by this run's budget or a backoff; rebuilding gives the same bytes.
209
+ // BLI-22794: only images without a description held it back, and none has one now.
206
210
  memo.count("volatile_reused", stamp.size);
207
- logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "images_deferred_budget_cannot_refill_this_run");
211
+ logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "pending_images_still_undescribed");
208
212
  return true;
209
213
  }
210
214
  memo.count("skipped", stamp.size);
211
215
  return true;
212
216
  }
213
- function logVolatileReuse(kind, sourceKey, contentHash, reason) {
214
- console.error("[raw-evidence] volatile transcript " + (reason === "hash_not_durable_full_path" ? "rebuilt" : "reused this run"), JSON.stringify({ reason, kind, source_key: sourceKey, hash_prefix: contentHash.slice(0, 12) }));
215
- }
216
217
  /** Remember what this transcript became, but only if the file stayed put while it was built. */
217
218
  async function rememberPreparedTranscript(collection, options, names, built) {
218
219
  const memo = collection.budget.prepareMemo;
219
220
  if (!memo)
220
221
  return;
221
222
  memo.count(built.volatile ? "volatile" : "rebuilt", built.readBytes);
222
- if ((built.volatile && !built.deferredOnly) || !built.stamp)
223
+ if ((built.volatile && !built.pendingImages) || !built.stamp)
223
224
  return;
224
225
  const after = await statSourceForMemo(options.filePath);
225
226
  const stable = after !== null
@@ -237,15 +238,13 @@ async function rememberPreparedTranscript(collection, options, names, built) {
237
238
  ? { redaction: built.redactedEntry.redaction, completenessLabel: built.redactedEntry.completenessLabel }
238
239
  : null,
239
240
  };
240
- if (built.volatile) {
241
- // BLI-22785: in memory for this process only; the memo file keeps no volatile entry.
242
- memo.rememberForThisRun(names.sourceKey, entry);
243
- return;
244
- }
245
241
  const known = memo.recall(names.sourceKey, built.stamp, built.recipe);
246
- if (known && !known.volatile && known.content_hash === built.contentHash)
242
+ const pending = built.volatile ? built.pendingImages : undefined;
243
+ if (known && known.volatile === built.volatile && known.content_hash === built.contentHash
244
+ && (known.pending_images ?? []).join() === (pending ?? []).join())
247
245
  return;
248
- memo.remember(names.sourceKey, { ...entry, volatile: false });
246
+ // BLI-22794: a volatile entry is kept across ticks with the images it is waiting on.
247
+ memo.remember(names.sourceKey, { ...entry, volatile: built.volatile, ...(pending ? { pending_images: pending } : {}) });
249
248
  }
250
249
  function nameEvidenceFile(options) {
251
250
  const fileName = path.basename(options.filePath);
@@ -0,0 +1,27 @@
1
+ /**
2
+ * BLI-22794: may a remembered image-pending entry stand in for a rebuild? Not when one of its
3
+ * images has a stored description now (the bytes would differ, counted as
4
+ * `rebuilt_image_now_described_count`), and not while the describer could still take one of them
5
+ * this tick: a skipped transcript never reaches the describer, so reusing it would starve the
6
+ * images. Once the tick's budget is spent, the rest are reused until the next tick.
7
+ */
8
+ export function trustVolatileEntry(collection, memo, known, kind, sourceKey, bytes) {
9
+ const session = collection.budget.images;
10
+ const pending = known.pending_images ?? [];
11
+ if (!session)
12
+ return false;
13
+ if (pending.some((sha256) => session.isDescribed(sha256))) {
14
+ memo.count("rebuilt_image_now_described", bytes);
15
+ logVolatileReuse(kind, sourceKey, known.content_hash, "rebuilt_image_now_described");
16
+ return false;
17
+ }
18
+ if (session.hasDescribeRoom(pending)) {
19
+ logVolatileReuse(kind, sourceKey, known.content_hash, "rebuilt_describer_has_room");
20
+ return false;
21
+ }
22
+ return true;
23
+ }
24
+ export function logVolatileReuse(kind, sourceKey, contentHash, reason) {
25
+ const reused = reason === "pending_images_still_undescribed";
26
+ console.error("[raw-evidence] volatile transcript " + (reused ? "reused across ticks" : "rebuilt"), JSON.stringify({ reason, kind, source_key: sourceKey, hash_prefix: contentHash.slice(0, 12) }));
27
+ }
@@ -106,6 +106,7 @@ export async function collectRawEvidencePack(context, options) {
106
106
  * made it possible. Counts and bytes only. Never throws: the memo is a cache.
107
107
  */
108
108
  async function reportPrepareMemo(collection) {
109
+ await collection.budget.readMemo?.settle();
109
110
  const counts = await collection.budget.prepareMemo?.settle();
110
111
  if (!counts || counts.skipped_count + counts.rebuilt_count + counts.volatile_reused_count === 0)
111
112
  return;
@@ -118,6 +119,7 @@ async function reportPrepareMemo(collection) {
118
119
  rebuilt_volatile_count: counts.volatile_count,
119
120
  rebuilt_volatile_reused_count: counts.volatile_reused_count,
120
121
  rebuilt_volatile_reused_bytes: counts.volatile_reused_bytes,
122
+ rebuilt_image_now_described_count: counts.rebuilt_image_now_described_count,
121
123
  }));
122
124
  }
123
125
  /**
@@ -0,0 +1,109 @@
1
+ /**
2
+ * Remember what a transcript file said, so an unchanged file is not opened again (BLI-22797).
3
+ *
4
+ * A sync tick with nothing new still re-read every attributed session, every subagent sidecar and every
5
+ * transcript the image step scans: on the reference Mac 33 of the 37 seconds of main-thread JavaScript went to
6
+ * reading, parsing and hashing files whose size and mtime had not moved since the tick before. This is the
7
+ * sibling of the prepare memo (`raw-evidence-prepare-memo.ts`) and uses ITS stamp: size plus inode, mtime and
8
+ * ctime in nanoseconds, so an append, rewrite or replace is never mistaken for "unchanged". The entry cap, the
9
+ * `seen_ms` ageing and the atomic write are the prepare memo's own helpers.
10
+ *
11
+ * An entry is trusted only when the stamp AND the recipe match. The recipe names the rules that produced the
12
+ * remembered answer, so changing a rule rebuilds the memo once. What an entry holds is decided by its caller
13
+ * (`data`); this file never sees a byte of transcript content, and neither does the file on disk: signals,
14
+ * hashes, sizes and counts only. A memo that is lost, corrupt or stale in the safe direction costs one slow
15
+ * tick, never a wrong answer. The file path is hashed, so no path reaches the memo either.
16
+ */
17
+ import { createHash } from "node:crypto";
18
+ import path from "node:path";
19
+ import { isMissingFileFailure } from "../health-detail.js";
20
+ import { readJsonFile, writeJsonFileAtomic } from "../local-state-files.js";
21
+ import { PREPARE_MEMO_MAX_ENTRIES, refreshSeen, trimEntries, } from "./raw-evidence-prepare-memo.js";
22
+ export { sourceStampOf, statSourceForMemo } from "./raw-evidence-prepare-memo.js";
23
+ const MEMO_FORMAT = "transcript-read-memo.v1";
24
+ const TAG = "[transcript-reads]";
25
+ export function transcriptReadMemoPath(stateDir, name) {
26
+ return path.join(stateDir, `transcript-read-memo-${name}.json`);
27
+ }
28
+ const keyOf = (filePath) => createHash("sha256").update(filePath).digest("hex").slice(0, 32);
29
+ export async function openTranscriptReadMemo(stateDir, name) {
30
+ const entries = await readEntries(stateDir, name);
31
+ let dirty = false;
32
+ let counts = { reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0 };
33
+ const recalled = new Set();
34
+ return {
35
+ recall(filePath, stamp, recipe, bytes, parse) {
36
+ const key = keyOf(filePath);
37
+ const entry = entries.get(key);
38
+ if (!entry || entry.recipe !== recipe || entry.size !== stamp.size || entry.identity !== stamp.identity)
39
+ return null;
40
+ const data = parse(entry.data);
41
+ if (data === null)
42
+ return null;
43
+ recalled.add(key);
44
+ counts.reused += 1;
45
+ counts.bytes_not_read += bytes;
46
+ return data;
47
+ },
48
+ remember(filePath, stamp, recipe, data, now = Date.now()) {
49
+ entries.set(keyOf(filePath), { ...stamp, recipe, data, seen_ms: now });
50
+ dirty = true;
51
+ },
52
+ note(outcome) {
53
+ if (outcome === "fresh")
54
+ counts.fresh += 1;
55
+ else
56
+ counts.stamp_missing += 1;
57
+ },
58
+ async settle(now = Date.now()) {
59
+ const done = counts;
60
+ counts = { reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0 };
61
+ dirty = refreshSeen(entries, recalled, now) || dirty;
62
+ recalled.clear();
63
+ let written = false;
64
+ let reason = done.reused > 0 ? "stamp_unchanged" : done.fresh > 0 ? "no_reusable_entry" : "stamp_missing";
65
+ if (dirty) {
66
+ const before = entries.size;
67
+ const kept = trimEntries(entries, now, PREPARE_MEMO_MAX_ENTRIES);
68
+ try {
69
+ await writeJsonFileAtomic(transcriptReadMemoPath(stateDir, name), { schema_version: MEMO_FORMAT, entries: Object.fromEntries(kept) });
70
+ dirty = false;
71
+ written = true;
72
+ }
73
+ catch (error) {
74
+ reason = "memo_write_failed";
75
+ console.error(TAG + " memo not written; the next tick reads these files again", JSON.stringify({ memo: name, reason, entries: before, error_name: error instanceof Error ? error.name : typeof error }));
76
+ }
77
+ }
78
+ if (done.reused + done.fresh + done.stamp_missing === 0)
79
+ return;
80
+ console.error(TAG + " tick", JSON.stringify({ memo: name, reason, reused: done.reused, read_fresh: done.fresh, stamp_missing: done.stamp_missing, bytes_not_read: done.bytes_not_read, entries: entries.size, memo_written: written }));
81
+ },
82
+ };
83
+ }
84
+ async function readEntries(stateDir, name) {
85
+ const entries = new Map();
86
+ let raw;
87
+ try {
88
+ raw = await readJsonFile(transcriptReadMemoPath(stateDir, name));
89
+ }
90
+ catch (error) {
91
+ if (!isMissingFileFailure(error))
92
+ console.error(TAG + " memo unreadable; every file is read once", JSON.stringify({ memo: name, reason: "read_memo_unreadable" }));
93
+ return entries;
94
+ }
95
+ const stored = raw;
96
+ if (stored?.schema_version !== MEMO_FORMAT || !stored.entries || typeof stored.entries !== "object")
97
+ return entries;
98
+ let dropped = 0;
99
+ for (const [key, value] of Object.entries(stored.entries)) {
100
+ const e = value;
101
+ if (e && typeof e.size === "number" && typeof e.identity === "string" && typeof e.recipe === "string" && typeof e.seen_ms === "number" && e.data !== undefined)
102
+ entries.set(key, e);
103
+ else
104
+ dropped += 1;
105
+ }
106
+ if (dropped > 0)
107
+ console.error(TAG + " memo entries dropped as malformed", JSON.stringify({ memo: name, reason: "read_memo_entries_malformed", dropped, kept: entries.size }));
108
+ return entries;
109
+ }
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
21
21
  }
22
22
 
23
23
  if (command === "--version" || command === "-V" || command === "version") {
24
- writeLine(io?.stdout ?? process.stdout, "0.2.251");
24
+ writeLine(io?.stdout ?? process.stdout, "0.2.253");
25
25
  return 0;
26
26
  }
27
27
 
@@ -15,6 +15,7 @@ import { CLAUDE_CURSOR_FILENAME } from "../cursors/raw-evidence-cursor.js";
15
15
  import { clearSourceRetryFailure, recordSourceRetryFailure, } from "../spool/local-spool.js";
16
16
  import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
17
17
  import { widenForCatchUp } from "./session-sync-outside-roots.js";
18
+ import { openTranscriptReadMemo } from "../adapters/transcript-read-memo.js";
18
19
  const CLAUDE_FIRST_RUN_BACKFILL_MINUTES = 14 * 24 * 60;
19
20
  /** The Claude scan's own default window when nothing widens it: the last 24 hours. */
20
21
  const CLAUDE_DEFAULT_WINDOW_MINUTES = 24 * 60;
@@ -51,6 +52,7 @@ export async function scanAndAttributeBothSources(options) {
51
52
  retrySinceMinutes: plan.claudeRetrySinceMinutes,
52
53
  firstRunBackfill,
53
54
  outsideRootCatchUpPending: plan.outsideRootCatchUpPending,
55
+ stateDir: options.paths.state_dir,
54
56
  });
55
57
  if (plan.codexEnabled) {
56
58
  await reconcileSourceScanRetry({
@@ -125,24 +127,33 @@ async function scanClaudeSessionsForSync(options) {
125
127
  ? CLAUDE_FIRST_RUN_BACKFILL_MINUTES
126
128
  : undefined, options.outsideRootCatchUpPending, CLAUDE_DEFAULT_WINDOW_MINUTES);
127
129
  const projectsDir = path.join(options.homeDir, ".claude", "projects");
128
- let scan = await scanAndAttributeClaudeSessions({
129
- projectsDir,
130
- worktrees: options.worktrees,
131
- now: options.now,
132
- collectionRoots: options.collectionRoots,
133
- sinceMinutes,
134
- });
135
- if (scan.session_limit_applied) {
136
- scan = await scanAndAttributeClaudeSessions({
130
+ // BLI-22797: one memo for both passes, settled once, so an unchanged file is never opened.
131
+ const readMemo = await openTranscriptReadMemo(options.stateDir, "attribution").catch(() => undefined);
132
+ try {
133
+ let scan = await scanAndAttributeClaudeSessions({
137
134
  projectsDir,
138
135
  worktrees: options.worktrees,
139
136
  now: options.now,
140
137
  collectionRoots: options.collectionRoots,
141
138
  sinceMinutes,
142
- limit: scan.discovered_session_count,
139
+ readMemo,
143
140
  });
141
+ if (scan.session_limit_applied) {
142
+ scan = await scanAndAttributeClaudeSessions({
143
+ projectsDir,
144
+ worktrees: options.worktrees,
145
+ now: options.now,
146
+ collectionRoots: options.collectionRoots,
147
+ sinceMinutes,
148
+ limit: scan.discovered_session_count,
149
+ readMemo,
150
+ });
151
+ }
152
+ return scan;
153
+ }
154
+ finally {
155
+ await readMemo?.settle();
144
156
  }
145
- return scan;
146
157
  }
147
158
  export function sourceScanRetryReason(source, scan) {
148
159
  const reasons = new Set();
@@ -1,3 +1,4 @@
1
+ import { releaseHeldSyncLocksOnExit } from "./sync-lock.js";
1
2
  /**
2
3
  * A run that dies must not be able to say it worked (BLI-4110).
3
4
  *
@@ -125,6 +126,14 @@ export function installCrashGuard(options = {}) {
125
126
  }));
126
127
  });
127
128
  target.on("exit", () => {
129
+ // A lock still held at exit is an orphan, whatever the exit code: release
130
+ // it synchronously so the next tick is not blocked (BLI-22795).
131
+ try {
132
+ releaseHeldSyncLocksOnExit();
133
+ }
134
+ catch (error) {
135
+ console.error("[sync-lock] exit release threw", JSON.stringify({ reason: "exit_release_threw", ...describeCrash(error) }));
136
+ }
128
137
  if (commandReturned)
129
138
  return;
130
139
  // `EXIT_DIED_MID_RUN` has to be in this set, and leaving it out made this
@@ -0,0 +1,36 @@
1
+ /**
2
+ * A deadline that keeps the event loop alive (BLI-22795).
3
+ *
4
+ * `AbortSignal.timeout()` is documented to use an unref'd timer. A fetch whose
5
+ * socket has gone (network dropped mid-upload) then has nothing ref'd behind
6
+ * it: the loop drains, Node exits with the await still pending, and the
7
+ * crash guard logs `died_mid_run` with the collection lock still held. Here
8
+ * the timer is ref'd, and the race rejects even when an injected or broken
9
+ * fetch ignores the abort, so the caller always gets a named failure.
10
+ */
11
+ export const UPLOAD_DEADLINE_ERROR_NAME = "EvidenceUploadDeadlineError";
12
+ export function isUploadDeadlineError(error) {
13
+ return error instanceof Error && error.name === UPLOAD_DEADLINE_ERROR_NAME;
14
+ }
15
+ export async function fetchWithDeadline(fetchImpl, url, init, deadlineMs) {
16
+ const controller = new AbortController();
17
+ let timer;
18
+ const deadline = new Promise((_, reject) => {
19
+ timer = setTimeout(() => {
20
+ const error = new Error("upload request exceeded its deadline");
21
+ error.name = UPLOAD_DEADLINE_ERROR_NAME;
22
+ controller.abort();
23
+ reject(error);
24
+ }, deadlineMs);
25
+ });
26
+ try {
27
+ const pending = fetchImpl(url, { ...init, signal: controller.signal });
28
+ // If the deadline wins, the abandoned fetch may reject later; that must not
29
+ // become an unhandled rejection.
30
+ pending.catch(() => undefined);
31
+ return await Promise.race([pending, deadline]);
32
+ }
33
+ finally {
34
+ clearTimeout(timer);
35
+ }
36
+ }
@@ -1,4 +1,5 @@
1
1
  import { encodeEvidenceBytes } from "@bli-cockpit/telemetry-core/evidence-storage-encoding";
2
+ import { fetchWithDeadline, isUploadDeadlineError } from "./evidence-upload-deadline.js";
2
3
  import { noteSyncProgress } from "./sync-lock.js";
3
4
  /** The signed capability is the only credential sent to Storage. Never log it,
4
5
  * its query string, provider bodies, or thrown errors that might contain it. */
@@ -13,10 +14,10 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index)
13
14
  for (let attempt = 1; attempt <= attempts; attempt++) {
14
15
  let status = 0;
15
16
  try {
16
- const response = await options.fetchImpl(signedUrl, {
17
+ const response = await fetchWithDeadline(options.fetchImpl, signedUrl, {
17
18
  method: "PUT", headers: { "Content-Type": "application/octet-stream", "x-upsert": "false" },
18
- body: new Uint8Array(encoded), signal: AbortSignal.timeout(60_000), redirect: "error",
19
- });
19
+ body: new Uint8Array(encoded), redirect: "error",
20
+ }, 60_000);
20
21
  status = response.status;
21
22
  await response.body?.cancel();
22
23
  if (response.ok) {
@@ -28,8 +29,8 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index)
28
29
  if (status < 500 && status !== 429)
29
30
  return false;
30
31
  }
31
- catch {
32
- log("signed_upload_network_failure", uploadId, index, attempt, status);
32
+ catch (error) {
33
+ log(isUploadDeadlineError(error) ? "signed_upload_deadline_exceeded" : "signed_upload_network_failure", uploadId, index, attempt, status);
33
34
  }
34
35
  if (attempt < attempts)
35
36
  await (options.sleep ?? sleep)(250 * attempt);
@@ -1,5 +1,6 @@
1
1
  import crypto from "node:crypto";
2
2
  import { describeError } from "./health-detail.js";
3
+ import { fetchWithDeadline, isUploadDeadlineError } from "./evidence-upload-deadline.js";
3
4
  import { noteSyncProgress } from "./sync-lock.js";
4
5
  /**
5
6
  * The `send` stage: one retrying POST to an evidence-upload route, plus the
@@ -10,6 +11,7 @@ import { noteSyncProgress } from "./sync-lock.js";
10
11
  */
11
12
  const DEFAULT_MAX_ATTEMPTS = 3;
12
13
  const RETRY_DELAY_MS = 250;
14
+ const REQUEST_DEADLINE_MS = 120_000;
13
15
  export async function requestJson(options, routePath, body) {
14
16
  const maxAttempts = options.maxAttemptsPerRequest ?? DEFAULT_MAX_ATTEMPTS;
15
17
  const sleep = options.sleep ?? defaultSleep;
@@ -17,7 +19,7 @@ export async function requestJson(options, routePath, body) {
17
19
  let lastBody = null;
18
20
  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
19
21
  try {
20
- const response = await options.fetchImpl(`${options.dashboardUrl}${routePath}`, {
22
+ const response = await fetchWithDeadline(options.fetchImpl, `${options.dashboardUrl}${routePath}`, {
21
23
  method: "POST",
22
24
  headers: {
23
25
  "Authorization": `Bearer ${options.deviceToken}`,
@@ -26,7 +28,7 @@ export async function requestJson(options, routePath, body) {
26
28
  ? { "x-tower-evidence-direct": "staged-gzip-v1" } : {}),
27
29
  },
28
30
  body: JSON.stringify(body),
29
- });
31
+ }, REQUEST_DEADLINE_MS);
30
32
  lastStatus = response.status;
31
33
  lastBody = await readResponseJson(response);
32
34
  if (response.ok || (response.status < 500 && response.status !== 429)) {
@@ -42,7 +44,9 @@ export async function requestJson(options, routePath, body) {
42
44
  // the request left the machine. This is the exact silence BLI-2528 sat
43
45
  // behind for 57 days; the route is named, the body never is.
44
46
  console.error("[evidence-upload] request failed before a status came back", JSON.stringify({
45
- reason: "upload_request_transport_error",
47
+ reason: isUploadDeadlineError(error)
48
+ ? "upload_request_deadline_exceeded"
49
+ : "upload_request_transport_error",
46
50
  route: routePath,
47
51
  attempt,
48
52
  max_attempts: maxAttempts,
@@ -13,9 +13,10 @@ import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedIma
13
13
  /** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
14
14
  export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
15
15
  /**
16
- * After this long spent DESCRIBING (the clock starts at the first image sent, not when the session
17
- * opens: collection can reach image-bearing transcripts minutes into a tick), remaining images
18
- * wait for the next tick.
16
+ * After this long spent DESCRIBING, remaining images wait for the next tick. Only time inside the
17
+ * sends counts: collection reaches image-bearing transcripts minutes into a tick and keeps slimming
18
+ * other transcripts between sends, and a wall clock from the session open (or from the first send)
19
+ * spent the budget on that work, so almost nothing was described (BLI-22789: 1 of 1,874, then 10).
19
20
  */
20
21
  export const IMAGE_DESCRIBE_MAX_RUN_MS = 120_000;
21
22
  const CONCURRENCY = 3;
@@ -32,9 +33,11 @@ export async function openImageDescriptions(options) {
32
33
  const maxPerRun = options.maxPerRun ?? IMAGE_DESCRIBE_MAX_PER_RUN;
33
34
  const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
34
35
  const sentThisRun = 0;
35
- const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, firstSendAt: null, maxPerRun, maxRunMs, sentThisRun };
36
+ const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun };
36
37
  return {
37
38
  mode,
39
+ isDescribed: (sha256) => described.has(imageStateKey(sha256)),
40
+ hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
38
41
  async prepareTranscript(input) {
39
42
  const afterImages = await describeAndReplace(run, input);
40
43
  if (mode !== "describe_and_drop")
@@ -44,6 +47,19 @@ export async function openImageDescriptions(options) {
44
47
  },
45
48
  };
46
49
  }
50
+ function describeRoomLeft(run, sha256s) {
51
+ if (run.sentThisRun >= run.maxPerRun)
52
+ return false;
53
+ if (run.describingMs > run.maxRunMs)
54
+ return false;
55
+ return sha256s.some((sha256) => {
56
+ // Already sent (or found too large) this run: it will not be sent again before the next tick.
57
+ if (run.settledThisRun.has(sha256))
58
+ return false;
59
+ const failure = run.failedImages[imageStateKey(sha256)];
60
+ return !(failure && failure.retry_after > run.now());
61
+ });
62
+ }
47
63
  function markDescribed(run, sha256) {
48
64
  const { described, orderedKeys } = run;
49
65
  const key = imageStateKey(sha256);
@@ -55,7 +71,6 @@ function markDescribed(run, sha256) {
55
71
  async function sendOne(run, image, text, input) {
56
72
  const { options, fetchImpl, settledThisRun, failedImages } = run;
57
73
  run.sentThisRun += 1;
58
- run.firstSendAt ??= run.now();
59
74
  const outcome = await postImageToDescribeDoor({ fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, sleep: options.sleep }, {
60
75
  schema_version: "image-describe.v1",
61
76
  sha256: image.sha256,
@@ -96,7 +111,7 @@ async function describeAndReplace(run, input) {
96
111
  const { mode, described, options, orderedKeys, failedImages } = run;
97
112
  const images = findEmbeddedImages(input.text);
98
113
  if (images.length === 0)
99
- return { text: input.text, imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
114
+ return { text: input.text, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
100
115
  const unique = new Map();
101
116
  for (const image of images)
102
117
  if (!unique.has(image.sha256))
@@ -114,6 +129,7 @@ async function describeAndReplace(run, input) {
114
129
  const replaced = mode === "describe_and_drop"
115
130
  ? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
116
131
  : { text: input.text, replacedCount: 0, charsSaved: 0 };
132
+ const pendingImages = [...unique.keys()].filter((sha256) => !described.has(imageStateKey(sha256)));
117
133
  console.error(TAG + " transcript prepared", JSON.stringify({
118
134
  mode,
119
135
  source: input.source,
@@ -128,6 +144,7 @@ async function describeAndReplace(run, input) {
128
144
  }));
129
145
  return {
130
146
  text: replaced.text,
147
+ pendingImages,
131
148
  imageCount: images.length,
132
149
  uniqueCount: unique.size,
133
150
  replacedCount: replaced.replacedCount,
@@ -164,11 +181,12 @@ function selectImagesToDescribe(run, unique, counts) {
164
181
  }
165
182
  async function sendPendingImages(run, toSend, counts, input) {
166
183
  const { now, maxPerRun, maxRunMs } = run;
184
+ const callStartedAt = now();
167
185
  let next = 0;
168
186
  const worker = async () => {
169
187
  while (next < toSend.length) {
170
188
  const image = toSend[next++];
171
- const timeUp = run.firstSendAt !== null && now() - run.firstSendAt > maxRunMs;
189
+ const timeUp = run.describingMs + (now() - callStartedAt) > maxRunMs;
172
190
  if (run.sentThisRun >= maxPerRun || timeUp) {
173
191
  counts.over_budget += 1;
174
192
  continue;
@@ -177,6 +195,7 @@ async function sendPendingImages(run, toSend, counts, input) {
177
195
  }
178
196
  };
179
197
  await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));
198
+ run.describingMs += now() - callStartedAt;
180
199
  }
181
200
  function countImageOutcomes(run, toSend, counts) {
182
201
  const { settledThisRun } = run;
@@ -25,6 +25,17 @@
25
25
  import { newClaudeSlimState, slimClaudeRecord } from "./claude-rules.js";
26
26
  import { newCodexSlimState, slimCodexRecord } from "./codex-rules.js";
27
27
  import { LineEdits, SeenBlocks, isJsonObject } from "./slim-support.js";
28
+ /**
29
+ * The version of the slim rule set (claude-rules.ts, codex-rules.ts, claude-tool-results.ts,
30
+ * claude-attachments.ts, slim-support.ts and this file), BLI-22794. The local prepare memo keys what a
31
+ * transcript became on this value (plus the secret and image rules, see telemetry-core's
32
+ * `evidenceRulesFingerprint`) instead of on the CLI version.
33
+ *
34
+ * CHANGING ANY SLIM RULE MEANS BUMPING THIS CONSTANT. That rebuilds every collector's prepare memo
35
+ * once, which is intended: the stored bytes are no longer what the new rules would produce.
36
+ * `slim-rules-version.test.ts` fails when a rule changes without this moving.
37
+ */
38
+ export const SLIM_RULES_VERSION = "slim-rules.v1";
28
39
  /** The text an image marker starts with; a line carrying one is a modified line (an image became a marker). */
29
40
  const IMAGE_MARKER_START = "[image sha256:";
30
41
  export function createSlimmer(kind, rulesForTests) {
package/dist/sync-lock.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import crypto from "node:crypto";
2
+ import fsSync from "node:fs";
2
3
  import fs from "node:fs/promises";
3
4
  import path from "node:path";
4
5
  import { describeError } from "./health-detail.js";
@@ -44,23 +45,25 @@ export async function acquireSyncLock(paths, now = new Date(), options = {}) {
44
45
  marks: 0,
45
46
  lastMarkMs: now.getTime(),
46
47
  longestGapMs: 0,
48
+ lockPath,
49
+ token,
47
50
  };
48
51
  const existing = await readLock(lockPath);
49
- if (existing && now.getTime() - existing.heartbeat_ms <= STALE_TAKEOVER_MS) {
52
+ if (existing && isFreshAndAlive(existing, now)) {
50
53
  logHeldByLiveOwner(existing, now);
51
54
  return { acquired: false, held_since: existing.heartbeat_at };
52
55
  }
53
56
  // Free or stale: try an exclusive create first; if another process created it
54
57
  // in the gap, re-check staleness before taking over.
55
- let takeover = existing ? "stale" : null;
58
+ let takeover = existing ? staleReason(existing) : null;
56
59
  let owned = await tryExclusiveCreate(lockPath, token, now, held);
57
60
  if (!owned) {
58
61
  const current = await readLock(lockPath);
59
- if (current && now.getTime() - current.heartbeat_ms <= STALE_TAKEOVER_MS) {
62
+ if (current && isFreshAndAlive(current, now)) {
60
63
  logHeldByLiveOwner(current, now);
61
64
  return { acquired: false, held_since: current.heartbeat_at };
62
65
  }
63
- takeover = current ? "stale" : "unreadable";
66
+ takeover = current ? staleReason(current) : "unreadable";
64
67
  await writeLock(lockPath, token, now, held); // take over the stale lock
65
68
  owned = true;
66
69
  }
@@ -68,6 +71,63 @@ export async function acquireSyncLock(paths, now = new Date(), options = {}) {
68
71
  heldStates.add(held);
69
72
  return maintainOwnedLock(lockPath, token, held);
70
73
  }
74
+ /**
75
+ * Does a process with this pid exist? `kill(pid, 0)` sends nothing: ESRCH means
76
+ * no such process, EPERM means it exists but is not ours (alive). Same on
77
+ * Windows. An unknown pid (-1 from an older record) is reported alive, so only
78
+ * the heartbeat decides for it.
79
+ */
80
+ export function isProcessAlive(pid) {
81
+ if (!Number.isInteger(pid) || pid <= 0)
82
+ return true;
83
+ try {
84
+ process.kill(pid, 0);
85
+ return true;
86
+ }
87
+ catch (error) {
88
+ return error.code !== "ESRCH";
89
+ }
90
+ }
91
+ /** Blocks only when the heartbeat is fresh AND the owner process still exists. */
92
+ function isFreshAndAlive(record, now) {
93
+ return now.getTime() - record.heartbeat_ms <= STALE_TAKEOVER_MS && isProcessAlive(record.pid);
94
+ }
95
+ /** A fresh heartbeat reaching here means the owner pid is gone; otherwise it aged out. */
96
+ function staleReason(record) {
97
+ return isProcessAlive(record.pid) ? "stale" : "dead";
98
+ }
99
+ /**
100
+ * Synchronous release of every lock this process still holds, for the `exit`
101
+ * handler (BLI-22795): a process that ends without `release()` must not keep
102
+ * the next tick out until the heartbeat ages. Ownership-safe by token.
103
+ */
104
+ export function releaseHeldSyncLocksOnExit() {
105
+ let released = 0;
106
+ for (const held of [...heldStates]) {
107
+ heldStates.delete(held);
108
+ let outcome = "not_ours_or_gone";
109
+ try {
110
+ if (held.lockPath && held.token) {
111
+ const raw = JSON.parse(fsSync.readFileSync(held.lockPath, "utf8"));
112
+ if (raw.token === held.token) {
113
+ fsSync.rmSync(held.lockPath, { force: true });
114
+ outcome = "removed";
115
+ released += 1;
116
+ }
117
+ }
118
+ }
119
+ catch (error) {
120
+ outcome = `release_failed_${error.code ?? "unknown"}`;
121
+ }
122
+ console.error("[sync-lock] lock released on exit", JSON.stringify({
123
+ reason: "holder_exited_holding_lock",
124
+ outcome,
125
+ role: held.role,
126
+ held_ms: Math.max(0, Date.now() - held.startedMs),
127
+ }));
128
+ }
129
+ return released;
130
+ }
71
131
  function maintainOwnedLock(lockPath, token, held) {
72
132
  const timer = setInterval(() => {
73
133
  // Refresh only while the lock is still ours; if we were taken over, stop
@@ -132,7 +192,13 @@ function logAcquired(held, takeover, previous, now) {
132
192
  ? "[sync-lock] lock acquired"
133
193
  : "[sync-lock] took over a lock whose owner stopped answering", JSON.stringify({
134
194
  role: held.role,
135
- reason: takeover === "stale" ? "heartbeat_stale" : takeover === "unreadable" ? "lock_file_unreadable" : "lock_free",
195
+ reason: takeover === "stale"
196
+ ? "heartbeat_stale"
197
+ : takeover === "dead"
198
+ ? "previous_owner_dead"
199
+ : takeover === "unreadable"
200
+ ? "lock_file_unreadable"
201
+ : "lock_free",
136
202
  previous_pid: previous && previous.pid > 0 ? previous.pid : null,
137
203
  previous_heartbeat_age_ms: previous ? Math.max(0, now.getTime() - previous.heartbeat_ms) : null,
138
204
  }));
@@ -222,7 +288,10 @@ export async function inspectSyncLock(paths, now = Date.now()) {
222
288
  const record = await readLock(path.join(paths.state_dir, LOCK_FILENAME));
223
289
  if (!record)
224
290
  return null;
225
- return { ...record, held: now - record.heartbeat_ms <= STALE_TAKEOVER_MS };
291
+ return {
292
+ ...record,
293
+ held: now - record.heartbeat_ms <= STALE_TAKEOVER_MS && isProcessAlive(record.pid),
294
+ };
226
295
  }
227
296
  /**
228
297
  * Remove the lock file only if it still carries `token`: the holder this caller
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.251",
3
+ "version": "0.2.253",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -27,9 +27,9 @@
27
27
  "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
28
28
  },
29
29
  "dependencies": {
30
- "@bli-cockpit/memory-mcp": "0.1.73",
31
- "@bli-cockpit/mcp": "0.1.154",
32
- "@bli-cockpit/telemetry-core": "0.1.80"
30
+ "@bli-cockpit/memory-mcp": "0.1.75",
31
+ "@bli-cockpit/mcp": "0.1.156",
32
+ "@bli-cockpit/telemetry-core": "0.1.82"
33
33
  },
34
34
  "repository": {
35
35
  "type": "git",