@bli-cockpit/cli 0.2.272 → 0.2.274

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -36,7 +36,7 @@ function parseMainRead(data) {
36
36
  export async function readSidecar(memo, localPath) {
37
37
  const stamp = memo ? await statSourceForMemo(localPath) : null;
38
38
  if (memo && stamp) {
39
- const hit = memo.recall(localPath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseSidecarRead);
39
+ const hit = await memo.recall(localPath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseSidecarRead);
40
40
  if (hit)
41
41
  return hit;
42
42
  }
@@ -60,7 +60,7 @@ export async function readSidecar(memo, localPath) {
60
60
  export async function readMain(memo, filePath) {
61
61
  const stamp = memo ? await statSourceForMemo(filePath) : null;
62
62
  if (memo && stamp) {
63
- const hit = memo.recall(filePath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseMainRead);
63
+ const hit = await memo.recall(filePath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseMainRead);
64
64
  if (hit)
65
65
  return hit;
66
66
  }
@@ -0,0 +1,52 @@
1
+ import { SESSION_FACTS_VERSION, SKILL_USE_VERSION } from "@bli-cockpit/telemetry-core";
2
+ import { readCodexMetadataSignals } from "./codex-session-signals.js";
3
+ import { statSourceForMemo } from "./transcript-read-memo.js";
4
+ /**
5
+ * The read attribution does of one Codex session file, remembered by the file's stamp (BLI-23346), the sibling
6
+ * of Claude's `claude-attribution-memoized-reads.ts`. Before this a tick re-streamed and re-hashed every Codex
7
+ * session in the window (177 files, 490 MB on the reference Mac) although no byte had changed. A Codex session
8
+ * is one file, so what is remembered is the finished answer of the streaming pass: the metadata signals, the
9
+ * sha256 and size of the bytes, and the finalized facts. The stamp (size, inode, mtime and ctime in
10
+ * nanoseconds) is the prepare memo's, so an append, rewrite or replace is read fresh. Scoring against the
11
+ * worktrees and the cwd-exists check run after the read, on every tick, so a repo that appears or vanishes is
12
+ * seen at once. Bump SIGNALS_VERSION when `codex-session-signals.ts` changes what it extracts.
13
+ */
14
+ const SIGNALS_VERSION = "codex-signals.v1";
15
+ export const CODEX_READ_RECIPE = ["codex-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION].join("|");
16
+ const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
17
+ const isNum = (v) => typeof v === "number" && Number.isFinite(v);
18
+ const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
19
+ function parseCodexRead(data) {
20
+ if (!isObj(data) || !isNum(data["byteSize"]) || typeof data["contentHashSha256"] !== "string")
21
+ return null;
22
+ const s = data["signals"];
23
+ const facts = data["facts"];
24
+ if (!isObj(s) || !isObj(facts))
25
+ return null;
26
+ for (const key of ["session_ids", "cwds", "workspace_roots", "branches", "commit_hashes", "repository_urls"]) {
27
+ if (!isStrings(s[key]))
28
+ return null;
29
+ }
30
+ if (!isNum(s["line_count"]) || !isNum(s["parse_error_count"]))
31
+ return null;
32
+ if (typeof facts["extractor_version"] !== "string" || facts["source"] !== "codex" || !isObj(facts["usage_by_model"]) || !isNum(facts["transcript_bytes"]))
33
+ return null;
34
+ return data;
35
+ }
36
+ /** The session's signals, hash, size and facts: recalled when its stamp is unchanged, else streamed (a read failure throws). */
37
+ export async function readCodexMetadataMemoized(memo, filePath) {
38
+ const stamp = memo ? await statSourceForMemo(filePath) : null;
39
+ if (memo && stamp) {
40
+ const hit = await memo.recall(filePath, stamp, CODEX_READ_RECIPE, stamp.size, parseCodexRead);
41
+ if (hit)
42
+ return hit;
43
+ }
44
+ else
45
+ memo?.note("stamp_missing");
46
+ const read = await readCodexMetadataSignals(filePath);
47
+ if (memo && stamp) {
48
+ memo.note("fresh");
49
+ memo.remember(filePath, stamp, CODEX_READ_RECIPE, read);
50
+ }
51
+ return read;
52
+ }
@@ -2,7 +2,8 @@ import { existsSync } from "node:fs";
2
2
  import fs from "node:fs/promises";
3
3
  import path from "node:path";
4
4
  import { describeError } from "../health-detail.js";
5
- import { extractCodexSessionSignals, readCodexMetadataSignals, } from "./codex-session-signals.js";
5
+ import { extractCodexSessionSignals, } from "./codex-session-signals.js";
6
+ import { readCodexMetadataMemoized } from "./codex-attribution-memoized-reads.js";
6
7
  import { sanitizeSessionId, scoreSignalsAgainstWorktrees, sessionIdFromFileName, shortHash, } from "./attribution-core.js";
7
8
  /**
8
9
  * Deterministic Codex session JSONL -> repo/worktree attribution.
@@ -41,7 +42,7 @@ export async function scanAndAttributeCodexSessions(options) {
41
42
  const files = discovery.files.slice(0, limit);
42
43
  const results = [];
43
44
  for (const file of files) {
44
- results.push(await attributeOneSession(file, options.worktrees, options.collectionRoots ?? []));
45
+ results.push(await attributeOneSession(file, options.worktrees, options.collectionRoots ?? [], options.readMemo));
45
46
  }
46
47
  return {
47
48
  results,
@@ -164,11 +165,11 @@ function dedupePaths(values) {
164
165
  }
165
166
  return out;
166
167
  }
167
- async function attributeOneSession(file, worktrees, collectionRoots) {
168
+ async function attributeOneSession(file, worktrees, collectionRoots, readMemo) {
168
169
  const base = createSessionBase(file);
169
170
  let read;
170
171
  try {
171
- read = await readCodexMetadataSignals(file.file);
172
+ read = await readCodexMetadataMemoized(readMemo, file.file);
172
173
  }
173
174
  catch (error) {
174
175
  // One skipped session, with `file_read_failed` on its row. Logged per
@@ -27,7 +27,7 @@ export async function collectAgentImagesFromTranscript(collection, options) {
27
27
  const memo = collection.budget.readMemo;
28
28
  const stamp = memo ? await statSourceForMemo(options.filePath) : null;
29
29
  const remembered = memo && stamp
30
- ? memo.recall(options.filePath, stamp, IMAGE_SCAN_RECIPE, stamp.size, (data) => (data === "no_image_candidates" ? true : null))
30
+ ? await memo.recall(options.filePath, stamp, IMAGE_SCAN_RECIPE, stamp.size, (data) => (data === "no_image_candidates" ? true : null))
31
31
  : null;
32
32
  if (!remembered && memo && !stamp)
33
33
  memo.note("stamp_missing");
@@ -5,49 +5,77 @@
5
5
  * transcript the image step scans: on the reference Mac 33 of the 37 seconds of main-thread JavaScript went to
6
6
  * reading, parsing and hashing files whose size and mtime had not moved since the tick before. This is the
7
7
  * sibling of the prepare memo (`raw-evidence-prepare-memo.ts`) and uses ITS stamp: size plus inode, mtime and
8
- * ctime in nanoseconds, so an append, rewrite or replace is never mistaken for "unchanged". The entry cap, the
9
- * `seen_ms` ageing and the atomic write are the prepare memo's own helpers.
8
+ * ctime in nanoseconds, so an append, rewrite or replace is never mistaken for "unchanged". The entry cap and
9
+ * the 14-day forgetting are the prepare memo's own numbers.
10
10
  *
11
11
  * An entry is trusted only when the stamp AND the recipe match. The recipe names the rules that produced the
12
12
  * remembered answer, so changing a rule rebuilds the memo once. What an entry holds is decided by its caller
13
13
  * (`data`); this file never sees a byte of transcript content, and neither does the file on disk: signals,
14
14
  * hashes, sizes and counts only. A memo that is lost, corrupt or stale in the safe direction costs one slow
15
15
  * tick, never a wrong answer. The file path is hashed, so no path reaches the memo either.
16
+ *
17
+ * BLI-23346: one small file per source key under `transcript-read-memo-<name>/`, read on demand. The first
18
+ * layout was one JSON file; on the reference Mac that reached 18.2 MB (2,931 entries, 1.1 M timestamps), was
19
+ * parsed whole at open, held through both scan passes and rewritten whole whenever one transcript was fresh.
20
+ * Now an unchanged transcript costs one small read that is parsed and dropped, a changed one costs one small
21
+ * atomic write, and nothing else is touched. A hit older than a day gets its `seen_ms` rewritten (one small
22
+ * file), and a prune pass (at most daily, by file mtime) forgets what was not written or refreshed for 14 days
23
+ * and the oldest beyond the cap. Exact facts are unchanged: an entry holds the same `data` the caller always
24
+ * stored (request-id-ordered usage, delta-encoded timestamps), it is only filed apart.
16
25
  */
17
26
  import { createHash } from "node:crypto";
27
+ import fs from "node:fs/promises";
18
28
  import path from "node:path";
19
- import { isMissingFileFailure } from "../health-detail.js";
20
- import { readJsonFile, writeJsonFileAtomic } from "../local-state-files.js";
21
- import { PREPARE_MEMO_MAX_ENTRIES, refreshSeen, trimEntries, } from "./raw-evidence-prepare-memo.js";
29
+ import { describeError } from "../health-detail.js";
30
+ import { isMissingFileError, writeJsonFileAtomic } from "../local-state-files.js";
31
+ import { FORGET_AFTER_MS, PREPARE_MEMO_MAX_ENTRIES, SEEN_REFRESH_MS, } from "./raw-evidence-prepare-memo.js";
22
32
  export { sourceStampOf, statSourceForMemo } from "./raw-evidence-prepare-memo.js";
23
- const MEMO_FORMAT = "transcript-read-memo.v1";
33
+ const MEMO_FORMAT = "transcript-read-memo.v2";
24
34
  const TAG = "[transcript-reads]";
35
+ const PRUNE_EVERY_MS = 24 * 3_600_000;
36
+ const PRUNE_MARKER = ".pruned";
37
+ const WRITE_CONCURRENCY = 8;
25
38
  export function transcriptReadMemoPath(stateDir, name) {
26
- return path.join(stateDir, `transcript-read-memo-${name}.json`);
39
+ return path.join(stateDir, `transcript-read-memo-${name}`);
27
40
  }
28
41
  const keyOf = (filePath) => createHash("sha256").update(filePath).digest("hex").slice(0, 32);
42
+ const zeroCounts = () => ({ reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0, unreadable: 0, refreshed: 0 });
29
43
  export async function openTranscriptReadMemo(stateDir, name) {
30
- const entries = await readEntries(stateDir, name);
31
- let dirty = false;
32
- let counts = { reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0 };
33
- const recalled = new Set();
44
+ const dir = transcriptReadMemoPath(stateDir, name);
45
+ const entryPath = (key) => path.join(dir, `${key}.json`);
46
+ /** Entries remembered this pass, written at `settle`; also answers a second pass over the same file before then. */
47
+ const pending = new Map();
48
+ let counts = zeroCounts();
49
+ await removeLegacySingleFile(stateDir, name);
34
50
  return {
35
- recall(filePath, stamp, recipe, bytes, parse) {
51
+ async recall(filePath, stamp, recipe, bytes, parse) {
36
52
  const key = keyOf(filePath);
37
- const entry = entries.get(key);
53
+ const fromPending = pending.get(key);
54
+ let entry = fromPending;
55
+ if (!entry) {
56
+ const loaded = await readEntry(entryPath(key));
57
+ if (loaded === "unreadable")
58
+ counts.unreadable += 1;
59
+ else if (loaded)
60
+ entry = loaded;
61
+ }
38
62
  if (!entry || entry.recipe !== recipe || entry.size !== stamp.size || entry.identity !== stamp.identity)
39
63
  return null;
40
64
  const data = parse(entry.data);
41
65
  if (data === null)
42
66
  return null;
43
- recalled.add(key);
44
67
  counts.reused += 1;
45
68
  counts.bytes_not_read += bytes;
69
+ const now = Date.now();
70
+ if (!fromPending && now - entry.seen_ms > SEEN_REFRESH_MS) {
71
+ entry.seen_ms = now;
72
+ counts.refreshed += 1;
73
+ await writeJsonFileAtomic(entryPath(key), entry, { compact: true }).catch(() => undefined);
74
+ }
46
75
  return data;
47
76
  },
48
77
  remember(filePath, stamp, recipe, data, now = Date.now()) {
49
- entries.set(keyOf(filePath), { ...stamp, recipe, data, seen_ms: now });
50
- dirty = true;
78
+ pending.set(keyOf(filePath), { schema_version: MEMO_FORMAT, ...stamp, recipe, data, seen_ms: now });
51
79
  },
52
80
  note(outcome) {
53
81
  if (outcome === "fresh")
@@ -57,53 +85,102 @@ export async function openTranscriptReadMemo(stateDir, name) {
57
85
  },
58
86
  async settle(now = Date.now()) {
59
87
  const done = counts;
60
- counts = { reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0 };
61
- dirty = refreshSeen(entries, recalled, now) || dirty;
62
- recalled.clear();
63
- let written = false;
64
- let reason = done.reused > 0 ? "stamp_unchanged" : done.fresh > 0 ? "no_reusable_entry" : "stamp_missing";
65
- if (dirty) {
66
- const before = entries.size;
67
- const kept = trimEntries(entries, now, PREPARE_MEMO_MAX_ENTRIES);
68
- try {
69
- await writeJsonFileAtomic(transcriptReadMemoPath(stateDir, name), { schema_version: MEMO_FORMAT, entries: Object.fromEntries(kept) }, { compact: true });
70
- dirty = false;
71
- written = true;
72
- }
73
- catch (error) {
74
- reason = "memo_write_failed";
75
- console.error(TAG + " memo not written; the next tick reads these files again", JSON.stringify({ memo: name, reason, entries: before, error_name: error instanceof Error ? error.name : typeof error }));
88
+ counts = zeroCounts();
89
+ const toWrite = [...pending.entries()];
90
+ pending.clear();
91
+ let written = 0;
92
+ let failed = 0;
93
+ for (let i = 0; i < toWrite.length; i += WRITE_CONCURRENCY) {
94
+ const outcomes = await Promise.all(toWrite.slice(i, i + WRITE_CONCURRENCY).map(([key, entry]) => writeJsonFileAtomic(entryPath(key), entry, { compact: true }).then(() => true, (error) => {
95
+ if (failed === 0)
96
+ console.error(TAG + " memo entry not written; that file is read again next tick", JSON.stringify({ memo: name, reason: "memo_write_failed", ...describeError(error) }));
97
+ return false;
98
+ })));
99
+ for (const ok of outcomes) {
100
+ if (ok)
101
+ written += 1;
102
+ else
103
+ failed += 1;
76
104
  }
77
105
  }
78
- if (done.reused + done.fresh + done.stamp_missing === 0)
106
+ const pruned = await pruneIfDue(dir, name, now);
107
+ if (done.reused + done.fresh + done.stamp_missing === 0 && pruned === null)
79
108
  return;
80
- console.error(TAG + " tick", JSON.stringify({ memo: name, reason, reused: done.reused, read_fresh: done.fresh, stamp_missing: done.stamp_missing, bytes_not_read: done.bytes_not_read, entries: entries.size, memo_written: written }));
109
+ const reason = failed > 0 ? "memo_write_failed" : done.reused > 0 ? "stamp_unchanged" : done.fresh > 0 ? "no_reusable_entry" : "stamp_missing";
110
+ console.error(TAG + " tick", JSON.stringify({
111
+ memo: name,
112
+ reason,
113
+ reused: done.reused,
114
+ read_fresh: done.fresh,
115
+ stamp_missing: done.stamp_missing,
116
+ bytes_not_read: done.bytes_not_read,
117
+ entries_written: written,
118
+ entries_write_failed: failed,
119
+ entries_unreadable: done.unreadable,
120
+ seen_refreshed: done.refreshed,
121
+ ...(pruned ? { pruned_expired: pruned.expired, pruned_over_cap: pruned.overCap, kept: pruned.kept } : {}),
122
+ }));
81
123
  },
82
124
  };
83
125
  }
84
- async function readEntries(stateDir, name) {
85
- const entries = new Map();
86
- let raw;
126
+ /** One entry file; null when absent; "unreadable" when it exists but is not an entry (a miss, rewritten by the next fresh read). */
127
+ async function readEntry(file) {
128
+ let text;
129
+ try {
130
+ text = await fs.readFile(file, "utf8");
131
+ }
132
+ catch (error) {
133
+ return isMissingFileError(error) ? null : "unreadable";
134
+ }
135
+ try {
136
+ const e = JSON.parse(text);
137
+ if (e && e.schema_version === MEMO_FORMAT && typeof e.size === "number" && typeof e.identity === "string" && typeof e.recipe === "string" && typeof e.seen_ms === "number" && e.data !== undefined)
138
+ return e;
139
+ }
140
+ catch {
141
+ // A torn or foreign file is a miss; the caller counts it as unreadable.
142
+ }
143
+ return "unreadable";
144
+ }
145
+ /** The pre-BLI-23346 single file is dead weight (18 MB on the reference Mac): remove it once and say so. */
146
+ async function removeLegacySingleFile(stateDir, name) {
147
+ const legacy = path.join(stateDir, `transcript-read-memo-${name}.json`);
87
148
  try {
88
- raw = await readJsonFile(transcriptReadMemoPath(stateDir, name));
149
+ const { size } = await fs.stat(legacy);
150
+ await fs.rm(legacy, { force: true });
151
+ console.error(TAG + " legacy single-file memo removed", JSON.stringify({ memo: name, reason: "legacy_memo_replaced_by_entry_files", bytes: size }));
89
152
  }
90
153
  catch (error) {
91
- if (!isMissingFileFailure(error))
92
- console.error(TAG + " memo unreadable; every file is read once", JSON.stringify({ memo: name, reason: "read_memo_unreadable" }));
93
- return entries;
154
+ if (!isMissingFileError(error))
155
+ console.error(TAG + " legacy memo not removed", JSON.stringify({ memo: name, reason: "legacy_memo_remove_failed", ...describeError(error) }));
94
156
  }
95
- const stored = raw;
96
- if (stored?.schema_version !== MEMO_FORMAT || !stored.entries || typeof stored.entries !== "object")
97
- return entries;
98
- let dropped = 0;
99
- for (const [key, value] of Object.entries(stored.entries)) {
100
- const e = value;
101
- if (e && typeof e.size === "number" && typeof e.identity === "string" && typeof e.recipe === "string" && typeof e.seen_ms === "number" && e.data !== undefined)
102
- entries.set(key, e);
103
- else
104
- dropped += 1;
157
+ }
158
+ /** At most once a day: forget entries not written or refreshed for 14 days, then the oldest beyond the cap. Null when not due or nothing to prune. */
159
+ async function pruneIfDue(dir, name, now) {
160
+ const marker = path.join(dir, PRUNE_MARKER);
161
+ try {
162
+ const last = await fs.stat(marker).then((s) => s.mtimeMs, () => 0);
163
+ if (now - last < PRUNE_EVERY_MS)
164
+ return null;
165
+ const names = (await fs.readdir(dir)).filter((n) => n.endsWith(".json"));
166
+ const aged = [];
167
+ for (const n of names) {
168
+ const file = path.join(dir, n);
169
+ const stat = await fs.stat(file).catch(() => null);
170
+ if (stat)
171
+ aged.push({ file, mtimeMs: stat.mtimeMs });
172
+ }
173
+ const expired = aged.filter((a) => now - a.mtimeMs > FORGET_AFTER_MS);
174
+ const live = aged.filter((a) => now - a.mtimeMs <= FORGET_AFTER_MS).sort((a, b) => b.mtimeMs - a.mtimeMs);
175
+ const over = live.slice(PREPARE_MEMO_MAX_ENTRIES);
176
+ await Promise.all([...expired, ...over].map((a) => fs.rm(a.file, { force: true })));
177
+ await fs.writeFile(marker, `${new Date(now).toISOString()}\n`);
178
+ await fs.utimes(marker, new Date(now), new Date(now));
179
+ return { expired: expired.length, overCap: over.length, kept: live.length - over.length };
180
+ }
181
+ catch (error) {
182
+ if (!isMissingFileError(error))
183
+ console.error(TAG + " memo prune failed", JSON.stringify({ memo: name, reason: "memo_prune_failed", ...describeError(error) }));
184
+ return null;
105
185
  }
106
- if (dropped > 0)
107
- console.error(TAG + " memo entries dropped as malformed", JSON.stringify({ memo: name, reason: "read_memo_entries_malformed", dropped, kept: entries.size }));
108
- return entries;
109
186
  }
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Tick-scoped bookkeeping: load each big state file once, write it at checkpoints (BLI-23346).
3
+ *
4
+ * The worktree loop of one sync tick used to read the 7 MB raw-evidence cursor and the staging state from disk,
5
+ * mutate them and write them back (serialise, fsync, rename) once per worktree, about 45 times a tick on the
6
+ * reference Mac: ~650 MB of JSON churn and ~326 MB of fsync'd writes to say "nothing changed". Inside
7
+ * `withBookkeepingSession` the first read of a file loads it, later reads return the SAME in-memory object, and a
8
+ * write only marks it dirty. Dirty files are persisted by the SAME writer the caller named (so fsync, temp file,
9
+ * rename and pruning are unchanged) at three points: when `CHECKPOINT_MS` has passed since the last flush, and
10
+ * when the session ends (also on an exception or a yielded loop, before the sync lock is released). A process
11
+ * killed between checkpoints loses at most that window of bookkeeping, which only means objects are re-offered
12
+ * next tick (the server dedups by content hash); it never loses a delivery.
13
+ *
14
+ * Outside a session every call is the plain disk read / write it always was, so a lone `cockpit sync`, backfill
15
+ * or doctor behaves exactly as before. Only the worktree loop opens a session, and it holds the sync lock.
16
+ */
17
+ import { AsyncLocalStorage } from "node:async_hooks";
18
+ import { describeError } from "./health-detail.js";
19
+ const TAG = "[bookkeeping]";
20
+ export const CHECKPOINT_MS = 30_000;
21
+ const storage = new AsyncLocalStorage();
22
+ /** The value for `key`: loaded by `load` on first use inside a session, then served from memory. Outside a session, `load()`. */
23
+ export async function bookkeepingRead(key, load) {
24
+ const session = storage.getStore();
25
+ if (!session)
26
+ return load();
27
+ const held = session.entries.get(key);
28
+ if (held) {
29
+ session.memoryReads += 1;
30
+ return held.value;
31
+ }
32
+ const value = await load();
33
+ session.diskReads += 1;
34
+ // `persist` is filled in by the first write; a never-written entry is never dirty.
35
+ session.entries.set(key, { value, persist: async () => undefined, dirty: false });
36
+ return value;
37
+ }
38
+ /** Record `value` as the new state of `key`. Inside a session it is persisted at the next checkpoint; outside, now. */
39
+ export async function bookkeepingWrite(key, value, persist) {
40
+ const session = storage.getStore();
41
+ if (!session)
42
+ return persist(value);
43
+ session.writesRequested += 1;
44
+ session.entries.set(key, { value, persist: persist, dirty: true });
45
+ if (Date.now() - session.lastFlushMs >= CHECKPOINT_MS)
46
+ await flush(session, "checkpoint");
47
+ }
48
+ async function flush(session, why) {
49
+ session.lastFlushMs = Date.now();
50
+ const dirty = [...session.entries.values()].filter((entry) => entry.dirty);
51
+ if (dirty.length === 0)
52
+ return;
53
+ session.flushes += 1;
54
+ for (const entry of dirty) {
55
+ entry.dirty = false;
56
+ try {
57
+ await entry.persist(entry.value);
58
+ session.filesFlushed += 1;
59
+ }
60
+ catch (error) {
61
+ entry.dirty = true;
62
+ console.error(TAG + " state not written", JSON.stringify({ reason: why === "end" ? "end_flush_failed" : "checkpoint_flush_failed", ...describeError(error) }));
63
+ if (why === "checkpoint")
64
+ throw error;
65
+ }
66
+ }
67
+ }
68
+ /** Run `fn` with bookkeeping held in memory; flush whatever is dirty when it settles, however it settles. */
69
+ export async function withBookkeepingSession(fn) {
70
+ if (storage.getStore())
71
+ return fn();
72
+ const session = { entries: new Map(), lastFlushMs: Date.now(), diskReads: 0, memoryReads: 0, writesRequested: 0, flushes: 0, filesFlushed: 0 };
73
+ try {
74
+ return await storage.run(session, fn);
75
+ }
76
+ finally {
77
+ await flush(session, "end");
78
+ if (session.diskReads + session.memoryReads + session.writesRequested > 0) {
79
+ console.error(TAG + " session settled", JSON.stringify({ reason: "session_end", disk_reads: session.diskReads, memory_reads: session.memoryReads, writes_requested: session.writesRequested, flush_count: session.flushes, files_written: session.filesFlushed }));
80
+ }
81
+ }
82
+ }
@@ -1,78 +1,11 @@
1
- import { runOpsCommand } from "./local-run-ops.js";
2
- import { runGlossary } from "./glossary.js";
3
- import { runSpend } from "./spend.js";
4
- import { runSessionsForget } from "./sessions-forget.js";
5
- import { runPush } from "./push.js";
6
1
  import { defaultIo, writeLine } from "./cli-io.js";
7
2
  import { isLocalHelpRequest, localCommandHelp } from "./local-help.js";
8
3
  import { beginStage, endStage } from "../crash-guard.js";
9
4
  import { reportCommandFailure, refuseArguments, stripGlobalToggles, watchWrites } from "./local-run-guards.js";
10
- import { reportInstallEventsBestEffort } from "./install-receipts.js";
11
- import { persistOnboardingRootConfig, resolveOnboardingRootsForCommand, } from "./collection-roots.js";
12
5
  import { rememberDiscoveryLimits } from "./local-discovery.js";
13
- import { runInstall, runRelease, runSelfUpdate, runUpdate } from "./install-update.js";
14
- import { runOnboard } from "./onboard.js";
15
- import { runLogin } from "./login.js";
16
- import { runLogout } from "./logout.js";
17
- import { runStart } from "./start.js";
18
- import { runSync } from "./sync.js";
19
- import { runDoctorAfterSync } from "./sync-doctor.js";
20
- import { runAnalyze } from "./analyze.js";
21
- import { runBackfillCommand } from "./backfill.js";
22
- import { runDoctor } from "./doctor.js";
23
- import { runStatus } from "./status.js";
24
- import { runSessions } from "./sessions.js";
25
- import { runJarvis } from "./jarvis.js";
26
- import { runModels } from "./models.js";
27
- import { runSlack } from "./slack.js";
28
- import { runModel, runSettings } from "./settings.js";
29
- import { runTeam } from "./team.js";
30
- import { runWorkbook } from "./workbook.js";
31
- import { runBrief } from "./brief.js";
32
- import { runBriefEdit } from "./brief-edit.js";
33
- import { runBriefRewrite } from "./brief-rewrite.js";
34
- import { runBriefDone } from "./brief-done.js";
35
- import { runCorrect } from "./correct.js";
36
- import { runNotes } from "./notes.js";
37
- import { runServe } from "./serve.js";
38
- import { runAutostart } from "./autostart-command.js";
39
- import { runAgentRules } from "./agent-rules-command.js";
40
- import { runMemoryLog } from "./memory-log.js";
41
- import { runMemoryInstall } from "./memory-install.js";
42
- import { runMemoryStore, runMemoryVerbList } from "./memory-store.js";
43
- import { runClean } from "./clean.js";
44
- import { runSourcesCheck } from "./sources-check.js";
45
- import { runDocs } from "./docs.js";
46
- import { runMsg } from "./msg.js";
47
- import { runIssue } from "./issue.js";
48
- import { runCal } from "./cal.js";
49
- import { runMail } from "./mail.js";
50
- import { runProject } from "./project.js";
51
- import { runMilestone } from "./milestone.js";
52
- import { runGithub } from "./github.js";
53
- import { runSearch } from "./search.js";
54
- import { runCareers } from "./careers.js";
55
- import { runRename } from "./rename-scope.js";
56
- import { runSetup } from "./setup.js";
57
- import { runUsage } from "./usage.js";
58
- import { runSpeed } from "./speed.js";
59
- import { runLogs } from "./logs.js";
60
- import { runApprovals, runDrills } from "./approvals.js";
61
- import { runManager } from "./manager.js";
62
- import { runImage } from "./image.js";
63
- import { runBrain } from "./brain.js";
64
- import { runHits } from "./hits.js";
65
- import { runIntentions } from "./intentions.js";
66
- import { runWho } from "./who.js";
67
- import { runPeople } from "./people.js";
68
6
  import { parseLocalArgs } from "./local-args.js";
69
- // `./local.js` is the published entry point (the public CLI root, root.ts, doctor.ts, tests): a split must not move a name off it.
7
+ // `./local.js` is the published entry point (the public CLI root, root.ts): it keeps the help registry names and loads each command module only when that command runs (BLI-23349).
70
8
  export { rootCommandNames, localCommandHelp, SURFACE_ALIASES, unknownCommandJson, unknownCommandLine, } from "./local-help.js";
71
- export { buildCollectorHeartbeat, collectionRootLabel, collectionRootLabels, sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
72
- export { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, SYNC_ERROR_DETAIL_MAX_CHARS, } from "./install-receipts.js";
73
- export { assertCollectionRootPersisted } from "./collection-roots.js";
74
- export { runSelfUpdate, SelfUpdateError, } from "./install-update.js";
75
- import { runRoots } from "./roots.js";
76
9
  /** Commands that act on THIS machine; the rest talk to Tower. */
77
10
  const MACHINE_COMMAND_KINDS = [
78
11
  "roots",
@@ -101,7 +34,9 @@ const MACHINE_COMMAND_KINDS = [
101
34
  function isMachineCommand(command) {
102
35
  return MACHINE_COMMAND_KINDS.includes(command.kind);
103
36
  }
104
- export async function runLocalCockpitCli(argv, io = { ...defaultIo(), runDoctorAfterSync }) {
37
+ /** `sync` is the only caller of this hook; the module loads when it is first called, not at startup. */
38
+ const lazyRunDoctorAfterSync = async (...args) => (await import("./sync-doctor.js")).runDoctorAfterSync(...args);
39
+ export async function runLocalCockpitCli(argv, io = { ...defaultIo(), runDoctorAfterSync: lazyRunDoctorAfterSync }) {
105
40
  argv = stripGlobalToggles(argv);
106
41
  if (isLocalHelpRequest(argv)) {
107
42
  writeLine(io.stdout, localCommandHelp(argv[0]));
@@ -131,7 +66,7 @@ export async function runLocalCockpitCli(argv, io = { ...defaultIo(), runDoctorA
131
66
  try {
132
67
  // Not awaited, as it never was: a `cal` rejection bypasses the net below.
133
68
  if (command.kind === "cal")
134
- return runCal(command, io);
69
+ return (await import("./cal.js")).runCal(command, io);
135
70
  if (isMachineCommand(command))
136
71
  return await runMachineCommand(command, io);
137
72
  return await runTowerCommand(command, io);
@@ -145,86 +80,86 @@ export async function runLocalCockpitCli(argv, io = { ...defaultIo(), runDoctorA
145
80
  }
146
81
  async function runMachineCommand(command, io) {
147
82
  switch (command.kind) {
148
- case "roots": return await runRoots(command, io);
149
- case "install": return await runInstall(command, io);
150
- case "onboard": return await runOnboard(command, io);
151
- case "update": return await runUpdate(command, io);
83
+ case "roots": return await (await import("./roots.js")).runRoots(command, io);
84
+ case "install": return await (await import("./install-update.js")).runInstall(command, io);
85
+ case "onboard": return await (await import("./onboard.js")).runOnboard(command, io);
86
+ case "update": return await (await import("./install-update.js")).runUpdate(command, io);
152
87
  case "doctor":
153
- return await runDoctor(command, io, {
154
- reportInstallEvents: reportInstallEventsBestEffort,
155
- selfUpdate: runSelfUpdate,
88
+ return await (await import("./doctor.js")).runDoctor(command, io, {
89
+ reportInstallEvents: (await import("./install-receipts.js")).reportInstallEventsBestEffort,
90
+ selfUpdate: (await import("./install-update.js")).runSelfUpdate,
156
91
  runLogin: runDoctorLogin,
157
92
  resolveAndSaveRoots: resolveAndSaveDoctorRoots,
158
93
  });
159
- case "login": return await runLogin(command, io);
160
- case "logout": return await runLogout(command, io);
161
- case "start": return await runStart(command, io);
162
- case "sync": return await runSync(command, io);
163
- case "analyze": return await runAnalyze(command, io);
164
- case "backfill": return await runBackfillCommand(command, io);
165
- case "status": return await runStatus(command, io);
166
- case "sessions": return await runSessions(command, io);
167
- case "serve": return await runServe(command, io);
168
- case "autostart": return await runAutostart(command, io);
169
- case "agent-rules": return await runAgentRules(command, io);
170
- case "memory-log": return await runMemoryLog(command, io);
94
+ case "login": return await (await import("./login.js")).runLogin(command, io);
95
+ case "logout": return await (await import("./logout.js")).runLogout(command, io);
96
+ case "start": return await (await import("./start.js")).runStart(command, io);
97
+ case "sync": return await (await import("./sync.js")).runSync(command, io);
98
+ case "analyze": return await (await import("./analyze.js")).runAnalyze(command, io);
99
+ case "backfill": return await (await import("./backfill.js")).runBackfillCommand(command, io);
100
+ case "status": return await (await import("./status.js")).runStatus(command, io);
101
+ case "sessions": return await (await import("./sessions.js")).runSessions(command, io);
102
+ case "serve": return await (await import("./serve.js")).runServe(command, io);
103
+ case "autostart": return await (await import("./autostart-command.js")).runAutostart(command, io);
104
+ case "agent-rules": return await (await import("./agent-rules-command.js")).runAgentRules(command, io);
105
+ case "memory-log": return await (await import("./memory-log.js")).runMemoryLog(command, io);
171
106
  case "memory":
172
107
  // BLI-4047: a bare `cockpit memory` says what the noun does. It used
173
108
  // to install; `install` is still one word away and every unattended
174
109
  // caller asks for it by name in code.
175
110
  if (command.action === "help")
176
- return runMemoryVerbList(io, command.json);
177
- return await runMemoryInstall(command, io);
178
- case "memory-store": return await runMemoryStore(command, io);
179
- case "clean": return await runClean(command, io);
180
- case "sources": return await runSourcesCheck(command, io);
181
- case "release": return await runRelease(command, io);
111
+ return (await import("./memory-store.js")).runMemoryVerbList(io, command.json);
112
+ return await (await import("./memory-install.js")).runMemoryInstall(command, io);
113
+ case "memory-store": return await (await import("./memory-store.js")).runMemoryStore(command, io);
114
+ case "clean": return await (await import("./clean.js")).runClean(command, io);
115
+ case "sources": return await (await import("./sources-check.js")).runSourcesCheck(command, io);
116
+ case "release": return await (await import("./install-update.js")).runRelease(command, io);
182
117
  }
183
118
  }
184
119
  async function runTowerCommand(command, io) {
185
120
  switch (command.kind) {
186
- case "jarvis": return await runJarvis(command, io);
187
- case "model": return await runModel(command, io);
188
- case "models": return await runModels(command, io);
121
+ case "jarvis": return await (await import("./jarvis.js")).runJarvis(command, io);
122
+ case "model": return await (await import("./settings.js")).runModel(command, io);
123
+ case "models": return await (await import("./models.js")).runModels(command, io);
189
124
  case "scout":
190
125
  case "ci":
191
126
  case "delivery":
192
127
  case "ops":
193
- return await runOpsCommand(command, io);
194
- case "slack": return await runSlack(command, io);
195
- case "settings": return await runSettings(command, io);
196
- case "team": return await runTeam(command, io);
197
- case "workbook": return await runWorkbook(command, io);
128
+ return await (await import("./local-run-ops.js")).runOpsCommand(command, io);
129
+ case "slack": return await (await import("./slack.js")).runSlack(command, io);
130
+ case "settings": return await (await import("./settings.js")).runSettings(command, io);
131
+ case "team": return await (await import("./team.js")).runTeam(command, io);
132
+ case "workbook": return await (await import("./workbook.js")).runWorkbook(command, io);
198
133
  case "brief": return await runBriefVerb(command, io);
199
- case "correct": return await runCorrect(command, io);
200
- case "notes": return await runNotes(command, io);
201
- case "glossary": return await runGlossary(command, io);
202
- case "sessions-forget": return await runSessionsForget(command, io);
203
- case "spend": return await runSpend(command, io);
204
- case "push": return await runPush(command, io);
205
- case "docs": return await runDocs(command, io);
206
- case "msg": return await runMsg(command, io);
207
- case "issue": return await runIssue(command, io);
208
- case "mail": return await runMail(command, io);
209
- case "project": return await runProject(command, io);
210
- case "milestone": return await runMilestone(command, io);
211
- case "search": return await runSearch(command, io);
212
- case "github": return await runGithub(command, io);
213
- case "careers": return await runCareers(command, io);
214
- case "rename": return await runRename(command, io);
215
- case "setup": return await runSetup(command, io);
216
- case "usage": return await runUsage(command, io);
217
- case "speed": return await runSpeed(command, io);
218
- case "logs": return await runLogs(command, io);
219
- case "brain": return await runBrain(command, io);
220
- case "hits": return await runHits(command, io);
221
- case "intentions": return await runIntentions(command, io);
222
- case "who": return await runWho(command, io);
223
- case "people": return await runPeople(command, io);
224
- case "approvals": return await runApprovals(command, io);
225
- case "drills": return await runDrills(command, io);
226
- case "manager": return await runManager(command, io);
227
- case "image": return await runImage(command, io);
134
+ case "correct": return await (await import("./correct.js")).runCorrect(command, io);
135
+ case "notes": return await (await import("./notes.js")).runNotes(command, io);
136
+ case "glossary": return await (await import("./glossary.js")).runGlossary(command, io);
137
+ case "sessions-forget": return await (await import("./sessions-forget.js")).runSessionsForget(command, io);
138
+ case "spend": return await (await import("./spend.js")).runSpend(command, io);
139
+ case "push": return await (await import("./push.js")).runPush(command, io);
140
+ case "docs": return await (await import("./docs.js")).runDocs(command, io);
141
+ case "msg": return await (await import("./msg.js")).runMsg(command, io);
142
+ case "issue": return await (await import("./issue.js")).runIssue(command, io);
143
+ case "mail": return await (await import("./mail.js")).runMail(command, io);
144
+ case "project": return await (await import("./project.js")).runProject(command, io);
145
+ case "milestone": return await (await import("./milestone.js")).runMilestone(command, io);
146
+ case "search": return await (await import("./search.js")).runSearch(command, io);
147
+ case "github": return await (await import("./github.js")).runGithub(command, io);
148
+ case "careers": return await (await import("./careers.js")).runCareers(command, io);
149
+ case "rename": return await (await import("./rename-scope.js")).runRename(command, io);
150
+ case "setup": return await (await import("./setup.js")).runSetup(command, io);
151
+ case "usage": return await (await import("./usage.js")).runUsage(command, io);
152
+ case "speed": return await (await import("./speed.js")).runSpeed(command, io);
153
+ case "logs": return await (await import("./logs.js")).runLogs(command, io);
154
+ case "brain": return await (await import("./brain.js")).runBrain(command, io);
155
+ case "hits": return await (await import("./hits.js")).runHits(command, io);
156
+ case "intentions": return await (await import("./intentions.js")).runIntentions(command, io);
157
+ case "who": return await (await import("./who.js")).runWho(command, io);
158
+ case "people": return await (await import("./people.js")).runPeople(command, io);
159
+ case "approvals": return await (await import("./approvals.js")).runApprovals(command, io);
160
+ case "drills": return await (await import("./approvals.js")).runDrills(command, io);
161
+ case "manager": return await (await import("./manager.js")).runManager(command, io);
162
+ case "image": return await (await import("./image.js")).runImage(command, io);
228
163
  }
229
164
  }
230
165
  /**
@@ -235,17 +170,17 @@ async function runTowerCommand(command, io) {
235
170
  */
236
171
  async function runBriefVerb(command, io) {
237
172
  if (command.action === "edit")
238
- return await runBriefEdit(command, io);
173
+ return await (await import("./brief-edit.js")).runBriefEdit(command, io);
239
174
  if (command.action === "rewrite")
240
- return await runBriefRewrite(command, io);
175
+ return await (await import("./brief-rewrite.js")).runBriefRewrite(command, io);
241
176
  // BLI-3605: the board's done checkbox, at a terminal.
242
177
  if (command.action === "done" || command.action === "undone") {
243
- return await runBriefDone(command, io);
178
+ return await (await import("./brief-done.js")).runBriefDone(command, io);
244
179
  }
245
- return await runBrief(command, io);
180
+ return await (await import("./brief.js")).runBrief(command, io);
246
181
  }
247
182
  async function runDoctorLogin(command, io) {
248
- return runLogin({
183
+ return (await import("./login.js")).runLogin({
249
184
  kind: "login",
250
185
  homeDir: command.homeDir,
251
186
  dashboardUrl: command.dashboardUrl,
@@ -265,8 +200,9 @@ async function resolveAndSaveDoctorRoots(command, io) {
265
200
  // headless equivalent for scripted and scheduled runs.
266
201
  allowHomeRoot: command.allowHomeRoot,
267
202
  };
268
- const resolution = await resolveOnboardingRootsForCommand(rootCommand, io);
269
- await persistOnboardingRootConfig(rootCommand, resolution);
203
+ const roots = await import("./collection-roots.js");
204
+ const resolution = await roots.resolveOnboardingRootsForCommand(rootCommand, io);
205
+ await roots.persistOnboardingRootConfig(rootCommand, resolution);
270
206
  if (!command.json) {
271
207
  writeLine(io.stdout, `Saved collection roots: ${resolution.collectionRoots.join(", ")}`);
272
208
  }
@@ -31,7 +31,7 @@ export async function resolveMemoryMcpBin(options) {
31
31
  export async function resolveMemoryConfig(command, io, platform, deps) {
32
32
  const dashboardUrl = await resolveDashboardUrl(command, deps);
33
33
  const found = await resolveMemoryMcpBin({
34
- env: envWithNodeRuntimeOnPath(io.env ?? process.env),
34
+ env: envWithNodeRuntimeOnPath(io.env ?? process.env, undefined, platform),
35
35
  platform,
36
36
  fileExists: deps.fileExists,
37
37
  cliEntryPoint: deps.cliEntryPoint,
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
21
21
  }
22
22
 
23
23
  if (command === "--version" || command === "-V" || command === "version") {
24
- writeLine(io?.stdout ?? process.stdout, "0.2.272");
24
+ writeLine(io?.stdout ?? process.stdout, "0.2.274");
25
25
  return 0;
26
26
  }
27
27
 
@@ -12,6 +12,7 @@ import { saveRootWorkScan } from "../root-work.js";
12
12
  import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
13
13
  import { scanAndAttributeClaudeSessions, } from "../adapters/claude-attribution.js";
14
14
  import { CLAUDE_CURSOR_FILENAME } from "../cursors/raw-evidence-cursor.js";
15
+ import { isRedactionStubObjectKey } from "../cursors/raw-evidence-stub-entry.js";
15
16
  import { clearSourceRetryFailure, recordSourceRetryFailure, } from "../spool/local-spool.js";
16
17
  import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
17
18
  import { widenForCatchUp } from "./session-sync-outside-roots.js";
@@ -37,6 +38,7 @@ export async function scanAndAttributeBothSources(options) {
37
38
  retryPending: plan.codexRetryPending,
38
39
  retrySinceMinutes: plan.codexRetrySinceMinutes,
39
40
  outsideRootCatchUpPending: plan.outsideRootCatchUpPending,
41
+ stateDir: options.paths.state_dir,
40
42
  });
41
43
  // First run (D B.4 §8): without a Claude cursor yet, widen the window to 14
42
44
  // days so the first sync captures retroactive history instead of only 24h.
@@ -60,7 +62,7 @@ export async function scanAndAttributeBothSources(options) {
60
62
  source: "codex",
61
63
  attemptedAt: options.now.toISOString(),
62
64
  pendingBefore: plan.codexSourceRetryPending,
63
- reason: sourceScanRetryReason("codex", codexAttribution),
65
+ reason: sourceScanRetryReason("codex", codexAttribution, plan.codexCursorBefore),
64
66
  });
65
67
  }
66
68
  if (plan.claudeEnabled) {
@@ -69,7 +71,7 @@ export async function scanAndAttributeBothSources(options) {
69
71
  source: "claude_code",
70
72
  attemptedAt: options.now.toISOString(),
71
73
  pendingBefore: plan.claudeSourceRetryPending,
72
- reason: sourceScanRetryReason("claude_code", claudeAttribution),
74
+ reason: sourceScanRetryReason("claude_code", claudeAttribution, plan.claudeCursorBefore),
73
75
  });
74
76
  }
75
77
  await saveRootWorkScan(options.paths, [...codexAttribution.results, ...claudeAttribution.results], options.now);
@@ -89,28 +91,37 @@ async function scanCodexSessionsForSync(options) {
89
91
  const sinceMinutes = widenForCatchUp(options.retryPending
90
92
  ? options.retrySinceMinutes
91
93
  : CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, options.outsideRootCatchUpPending, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES);
92
- let scan = await scanAndAttributeCodexSessions({
93
- sessionsDirs: defaultCodexSessionDirs(options.homeDir),
94
- worktrees: options.worktrees,
95
- now: options.now,
96
- sinceMinutes,
97
- limit: CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT,
98
- collectionRoots: options.collectionRoots,
99
- });
100
- if (scan.session_limit_applied) {
101
- // Re-run once, bounded by exactly what the first pass discovered, so a
102
- // capped scan still returns every session it found instead of silently
103
- // truncating at the window's default limit.
104
- scan = await scanAndAttributeCodexSessions({
94
+ // BLI-23346: one memo for both passes, settled once, so an unchanged session file is never streamed again.
95
+ const readMemo = await openTranscriptReadMemo(options.stateDir, "codex_attribution").catch(() => undefined);
96
+ try {
97
+ let scan = await scanAndAttributeCodexSessions({
105
98
  sessionsDirs: defaultCodexSessionDirs(options.homeDir),
106
99
  worktrees: options.worktrees,
107
100
  now: options.now,
108
101
  sinceMinutes,
109
- limit: scan.discovered_file_count,
102
+ limit: CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT,
110
103
  collectionRoots: options.collectionRoots,
104
+ readMemo,
111
105
  });
106
+ if (scan.session_limit_applied) {
107
+ // Re-run once, bounded by exactly what the first pass discovered, so a
108
+ // capped scan still returns every session it found instead of silently
109
+ // truncating at the window's default limit.
110
+ scan = await scanAndAttributeCodexSessions({
111
+ sessionsDirs: defaultCodexSessionDirs(options.homeDir),
112
+ worktrees: options.worktrees,
113
+ now: options.now,
114
+ sinceMinutes,
115
+ limit: scan.discovered_file_count,
116
+ collectionRoots: options.collectionRoots,
117
+ readMemo,
118
+ });
119
+ }
120
+ return scan;
121
+ }
122
+ finally {
123
+ await readMemo?.settle();
112
124
  }
113
- return scan;
114
125
  }
115
126
  /**
116
127
  * Scan and attribute Claude sessions for this sync — the Claude counterpart to
@@ -155,7 +166,7 @@ async function scanClaudeSessionsForSync(options) {
155
166
  await readMemo?.settle();
156
167
  }
157
168
  }
158
- export function sourceScanRetryReason(source, scan) {
169
+ export function sourceScanRetryReason(source, scan, cursorBefore) {
159
170
  const reasons = new Set();
160
171
  const failure = sourceScanFailureReason(source, scan);
161
172
  if (failure)
@@ -164,11 +175,30 @@ export function sourceScanRetryReason(source, scan) {
164
175
  // reason to widen the next scan window, because the transcript fallback can
165
176
  // still attribute it. It is not a reason to call this sync failed — see
166
177
  // `sourceScanFailureReason`.
167
- if (scan.results.some((result) => result.reason === "repo_not_on_disk")) {
178
+ //
179
+ // BLI-23346: only while that session still owes its upload. A `repo_not_on_disk` session the cursor already
180
+ // holds a durable pointer for owes nothing, and widening for it re-scanned 30 days every tick on the reference
181
+ // Mac (the widened window found the same already-delivered sessions again, which re-armed the flag).
182
+ const owing = scan.results.filter((result) => result.reason === "repo_not_on_disk" && sessionStillOwesUpload(source, result, cursorBefore));
183
+ if (owing.length > 0) {
168
184
  reasons.add("repo_not_on_disk");
169
185
  }
186
+ else if (cursorBefore) {
187
+ const delivered = scan.results.filter((result) => result.reason === "repo_not_on_disk").length;
188
+ if (delivered > 0)
189
+ console.error("[session-sync] repo_not_on_disk sessions are already delivered, no scan retry kept", JSON.stringify({ reason: "repo_not_on_disk_already_durable", source, delivered_count: delivered }));
190
+ }
170
191
  return reasons.size > 0 ? [...reasons].sort().join(",") : null;
171
192
  }
193
+ function sessionStillOwesUpload(source, result, cursorBefore) {
194
+ if (!cursorBefore)
195
+ return true;
196
+ const sessionId = source === "codex"
197
+ ? result.codex_session_id
198
+ : result.claude_session_id;
199
+ const key = cursorBefore.sessions[sessionId]?.uploaded_object_key;
200
+ return !key || isRedactionStubObjectKey(key);
201
+ }
172
202
  /**
173
203
  * The part of the scan outcome that is a genuine FAILURE: the session store
174
204
  * itself could not be read, so sessions that exist were not seen.
@@ -6,6 +6,7 @@
6
6
  * and the Claude growth damper lives here because it is the only step that can
7
7
  * see both what the cursor already made durable and what is about to be sent.
8
8
  */
9
+ import { withBookkeepingSession } from "../bookkeeping-session.js";
9
10
  import { startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from "../local-state.js";
10
11
  import { LocalUploadBlockedError, syncLocalAmbientEnvelope, } from "../upload.js";
11
12
  import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET, } from "../adapters/raw-evidence.js";
@@ -41,12 +42,14 @@ export async function syncEveryTargetWorktree(options) {
41
42
  };
42
43
  // BLI-11112: a waiting backfill takes the lock at the next worktree
43
44
  // boundary; the worktrees not reached stay pending for it or the next tick.
44
- const loop = await runYieldingAtBoundaries({
45
+ // BLI-23346: the cursor and the staging state are loaded once for the whole loop and written at checkpoints and
46
+ // when the loop ends (before the lock is released), not read and rewritten per worktree.
47
+ const loop = await withBookkeepingSession(() => runYieldingAtBoundaries({
45
48
  items: syncWorktrees,
46
49
  shouldYield: run.yieldCheck,
47
50
  describe: (outcome) => ({ status: outcome.sync.status }),
48
51
  runOne: (worktree) => syncTargetWithAllowance(options, worktree, syncWorktrees, rawEvidenceBudget, discoveredWorktreeKeys, dampedClaudePointers),
49
- });
52
+ }));
50
53
  const outcomes = loop.results;
51
54
  const everyWorktreeUploaded = outcomes.every((outcome) => outcome.sync.status === "uploaded");
52
55
  return {
@@ -26,7 +26,7 @@ import { envWithNodeRuntimeOnPath } from "../scheduled-self-update.js";
26
26
  async function resolveTowerConfig(deps) {
27
27
  const found = await resolveMcpBin({
28
28
  binName: TOWER_MCP_BIN,
29
- env: envWithNodeRuntimeOnPath(deps.env),
29
+ env: envWithNodeRuntimeOnPath(deps.env, undefined, deps.platform),
30
30
  platform: deps.platform,
31
31
  fileExists: deps.fileExists,
32
32
  cliEntryPoint: deps.cliEntryPoint,
@@ -1,6 +1,7 @@
1
1
  import crypto from "node:crypto";
2
2
  import fs from "node:fs/promises";
3
3
  import path from "node:path";
4
+ import { bookkeepingRead, bookkeepingWrite } from "../bookkeeping-session.js";
4
5
  import { describeError, isMissingFileFailure } from "../health-detail.js";
5
6
  /**
6
7
  * Durable local cursor for raw-evidence harvesting.
@@ -37,7 +38,22 @@ export function emptyRawEvidenceCursorState() {
37
38
  sessions: {},
38
39
  };
39
40
  }
41
+ /**
42
+ * Inside a worktree-loop bookkeeping session the cursor is loaded once and handed out as one shared object
43
+ * (`bookkeeping-session.ts`, BLI-23346); anywhere else this is the plain disk read.
44
+ */
40
45
  export async function readRawEvidenceCursor(paths, options = {}) {
46
+ let key;
47
+ try {
48
+ key = rawEvidenceCursorPath(paths, options.filename);
49
+ }
50
+ catch {
51
+ // Paths without a cursors dir: the disk reader below reports it exactly as it always did.
52
+ return readRawEvidenceCursorFromDisk(paths, options);
53
+ }
54
+ return bookkeepingRead(key, () => readRawEvidenceCursorFromDisk(paths, options));
55
+ }
56
+ async function readRawEvidenceCursorFromDisk(paths, options) {
41
57
  try {
42
58
  const raw = JSON.parse(await fs.readFile(rawEvidenceCursorPath(paths, options.filename), "utf8"));
43
59
  return parseCursorState(raw);
@@ -59,6 +75,9 @@ export async function readRawEvidenceCursor(paths, options = {}) {
59
75
  }
60
76
  }
61
77
  export async function writeRawEvidenceCursor(paths, state, options = {}) {
78
+ await bookkeepingWrite(rawEvidenceCursorPath(paths, options.filename), state, (held) => writeRawEvidenceCursorToDisk(paths, held, options));
79
+ }
80
+ async function writeRawEvidenceCursorToDisk(paths, state, options) {
62
81
  const filePath = rawEvidenceCursorPath(paths, options.filename);
63
82
  await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
64
83
  const pruned = pruneCursorState(state);
@@ -1,185 +1,2 @@
1
- /**
2
- * Find the embedded images of one transcript line without ever building the base64 as a JS string
3
- * (BLI-23317, stage D).
4
- *
5
- * `findEmbeddedImages` (telemetry-core/transcript-images.ts) is the RULE and stays the test oracle: it
6
- * feeds `evidenceRulesFingerprint`, so its regex and its results do not change. This module is the same
7
- * rule run on the line's BYTES, which is what the streamed prepare path calls:
8
- *
9
- * - a whole-string payload: a base64 run that starts right after a `"`, starts with one of the four
10
- * image starts (`iVBOR`, `/9j/`, `R0lGO`, `UklGR`), is at least 495 characters past that start
11
- * (so 499 for a JPEG, 500 for the others, exactly as the regex counts) and is closed by `"` or `\"`;
12
- * - a data URL: `data:image/<[a-z0-9.+-]+>;base64,` and then a run of at least 500 characters, no closing
13
- * quote needed. The whole URL is the replaced range.
14
- *
15
- * A match is accepted only when the decoded bytes sniff as PNG, JPEG, GIF or WebP; either way the scan
16
- * resumes after the run, as `matchAll` does. All matched characters are ASCII, so a byte offset here is
17
- * the character offset there on ASCII text, and the replaced ranges are the same everywhere else.
18
- *
19
- * Decoding is incremental: the run is decoded in blocks of 48 Ki characters (a multiple of 4, so each
20
- * block decodes on its own) straight into a sha256, with the first block sniffed. At no point does a
21
- * string of the run, or a Buffer of the whole decoded image, exist. The one exception is a run whose
22
- * `=` characters are not plain trailing padding: Node's decoder rules for those are version specific,
23
- * so such a run takes the whole-run decode the oracle takes (never seen in real transcripts).
24
- *
25
- * `byteSize` is the length Node's `Buffer.from(run, "base64")` gives, because every block, including a
26
- * short last one, is decoded by that very function.
27
- */
28
- import { createHash } from "node:crypto";
29
- import { IMAGE_MIN_BASE64_CHARS, sniffImageMediaType, } from "@bli-cockpit/telemetry-core";
30
- const QUOTE = 0x22;
31
- const BACKSLASH = 0x5c;
32
- const EQUALS = 0x3d;
33
- /** Characters per decoded block. A multiple of 4: each block is a whole number of base64 quanta. */
34
- const BLOCK_CHARS = 48 * 1024;
35
- const BASE64 = new Uint8Array(256);
36
- for (const range of ["AZ", "az", "09"]) {
37
- for (let code = range.charCodeAt(0); code <= range.charCodeAt(1); code += 1)
38
- BASE64[code] = 1;
39
- }
40
- for (const char of "+/=")
41
- BASE64[char.charCodeAt(0)] = 1;
42
- const DATA_URL_LOWER = new Uint8Array(256);
43
- for (const range of ["az", "09"]) {
44
- for (let code = range.charCodeAt(0); code <= range.charCodeAt(1); code += 1)
45
- DATA_URL_LOWER[code] = 1;
46
- }
47
- for (const char of ".+-")
48
- DATA_URL_LOWER[char.charCodeAt(0)] = 1;
49
- /** The four image starts of a whole-string payload, and the data URL anchor. */
50
- const STARTS = ["iVBOR", "/9j/", "R0lGO", "UklGR"].map((text) => Buffer.from(text, "latin1"));
51
- const DATA_URL = Buffer.from("data:image/", "latin1");
52
- const BASE64_TAG = Buffer.from(";base64,", "latin1");
53
- /** Is there any chance this line holds an image? Same anchors as the string check, on bytes. */
54
- export function bytesMayHoldImage(line) {
55
- return line.includes(DATA_URL) || STARTS.some((start) => line.includes(start));
56
- }
57
- /** The end (exclusive) of the base64-alphabet run that starts at `from`. */
58
- function runEnd(line, from) {
59
- let at = from;
60
- while (at < line.length && BASE64[line[at]] === 1)
61
- at += 1;
62
- return at;
63
- }
64
- /**
65
- * sha256, size and type of the image `line[from, to)` encodes, or null when it is not a PNG, JPEG, GIF
66
- * or WebP. Blocks of the run are decoded one at a time.
67
- */
68
- export function digestBase64Run(line, from, to) {
69
- const length = to - from;
70
- const inRun = line.subarray(from, to).indexOf(EQUALS);
71
- const firstPad = inRun === -1 ? -1 : from + inRun;
72
- const padCount = firstPad === -1 ? 0 : to - firstPad;
73
- const plainPadding = padCount === 0
74
- || (padCount <= 2 && length % 4 === 0 && onlyPaddingFrom(line, firstPad, to));
75
- if (!plainPadding) {
76
- const bytes = Buffer.from(line.toString("latin1", from, to), "base64");
77
- const mediaType = sniffImageMediaType(bytes);
78
- return mediaType ? { sha256: createHash("sha256").update(bytes).digest("hex"), mediaType, byteSize: bytes.byteLength } : null;
79
- }
80
- const hash = createHash("sha256");
81
- let byteSize = 0;
82
- let mediaType = null;
83
- for (let at = from; at < to; at += BLOCK_CHARS) {
84
- const block = Buffer.from(line.toString("latin1", at, Math.min(to, at + BLOCK_CHARS)), "base64");
85
- if (at === from) {
86
- mediaType = sniffImageMediaType(block);
87
- if (!mediaType)
88
- return null;
89
- }
90
- hash.update(block);
91
- byteSize += block.byteLength;
92
- }
93
- return mediaType ? { sha256: hash.digest("hex"), mediaType, byteSize } : null;
94
- }
95
- /** True when everything from `firstPad` to `to` is `=` (padding at the very end of the run, nothing after it). */
96
- function onlyPaddingFrom(line, firstPad, to) {
97
- for (let at = firstPad; at < to; at += 1)
98
- if (line[at] !== EQUALS)
99
- return false;
100
- return true;
101
- }
102
- /** The sha256 of an image's decoded bytes, or null when `line[from, to)` does not decode to an image (used to re-check a payload before it is sent). */
103
- export function sha256OfBase64Run(line, from, to) {
104
- return digestBase64Run(line, from, to)?.sha256 ?? null;
105
- }
106
- /**
107
- * The next index at or after `from` where `anchor` sits right after a `"`, or -1. `current` is the answer
108
- * of the last call: -2 not looked for yet, -1 none left, else still good while it is at or after `from`.
109
- */
110
- function nextQuotedStart(line, anchor, from, current) {
111
- if (current === -1 || current >= from)
112
- return current;
113
- let at = line.indexOf(anchor, from);
114
- while (at !== -1 && !(at > 0 && line[at - 1] === QUOTE))
115
- at = line.indexOf(anchor, at + 1);
116
- return at;
117
- }
118
- /**
119
- * Every embedded image in `line` (one physical line, no `\n` inside), in order, with byte offsets inside
120
- * the line. Equal to `findEmbeddedImages` on the decoded text in everything but the unit of the offsets.
121
- */
122
- export function scanImagesInLine(line) {
123
- const found = [];
124
- if (line.length < IMAGE_MIN_BASE64_CHARS - 1)
125
- return found;
126
- // For each whole-string start, the next quoted occurrence (-2: not looked for yet, -1: none left).
127
- const nextStart = STARTS.map(() => -2);
128
- let nextData = -2;
129
- let position = 0;
130
- while (position < line.length) {
131
- let candidate = -1;
132
- let kind = -1;
133
- for (let index = 0; index < STARTS.length; index += 1) {
134
- nextStart[index] = nextQuotedStart(line, STARTS[index], position, nextStart[index]);
135
- const at = nextStart[index];
136
- if (at !== -1 && (candidate === -1 || at < candidate)) {
137
- candidate = at;
138
- kind = index;
139
- }
140
- }
141
- if (nextData !== -1) {
142
- nextData = nextData >= position ? nextData : line.indexOf(DATA_URL, position);
143
- if (nextData !== -1 && (candidate === -1 || nextData < candidate)) {
144
- candidate = nextData;
145
- kind = STARTS.length;
146
- }
147
- }
148
- if (candidate === -1)
149
- break;
150
- if (kind < STARTS.length) {
151
- const startLength = STARTS[kind].length;
152
- const end = runEnd(line, candidate);
153
- const closed = end < line.length && (line[end] === QUOTE || (line[end] === BACKSLASH && line[end + 1] === QUOTE));
154
- if (!closed || end - candidate < startLength + (IMAGE_MIN_BASE64_CHARS - 5)) {
155
- position = candidate + 1;
156
- continue;
157
- }
158
- position = end;
159
- const digest = digestBase64Run(line, candidate, end);
160
- if (digest)
161
- found.push({ replaceStart: candidate, replaceEnd: end, base64Start: candidate, base64End: end, ...digest });
162
- continue;
163
- }
164
- // data:image/<type>;base64,<run of at least 500>
165
- let cursor = candidate + DATA_URL.length;
166
- const typeStart = cursor;
167
- while (cursor < line.length && DATA_URL_LOWER[line[cursor]] === 1)
168
- cursor += 1;
169
- if (cursor === typeStart || line.compare(BASE64_TAG, 0, BASE64_TAG.length, cursor, cursor + BASE64_TAG.length) !== 0) {
170
- position = candidate + 1;
171
- continue;
172
- }
173
- const base64Start = cursor + BASE64_TAG.length;
174
- const end = runEnd(line, base64Start);
175
- if (end - base64Start < IMAGE_MIN_BASE64_CHARS) {
176
- position = candidate + 1;
177
- continue;
178
- }
179
- position = end;
180
- const digest = digestBase64Run(line, base64Start, end);
181
- if (digest)
182
- found.push({ replaceStart: candidate, replaceEnd: end, base64Start, base64End: end, ...digest });
183
- }
184
- return found;
185
- }
1
+ /** The byte scanner lives in telemetry-core (BLI-23349) so the memory hook shares it; this keeps the collector's import path. */
2
+ export { bytesMayHoldImage, digestBase64Run, scanImagesInLine, sha256OfBase64Run, } from "@bli-cockpit/telemetry-core";
@@ -2,6 +2,7 @@ import { DELIVERY_BACKOFF_HOLDING } from "@bli-cockpit/telemetry-core";
2
2
  import crypto from "node:crypto";
3
3
  import fs from "node:fs/promises";
4
4
  import path from "node:path";
5
+ import { bookkeepingRead, bookkeepingWrite } from "./bookkeeping-session.js";
5
6
  import { describeError, isMissingFileFailure } from "./health-detail.js";
6
7
  /**
7
8
  * Durable local state for raw-evidence STAGING and DELIVERY ATTEMPTS.
@@ -67,7 +68,9 @@ export function emptyRawEvidenceStagingState() {
67
68
  export function rawEvidenceStagingStatePath(stateDir) {
68
69
  return path.join(stateDir, RAW_EVIDENCE_STAGING_FILENAME);
69
70
  }
70
- export async function readRawEvidenceStagingState(stateDir) {
71
+ export const readRawEvidenceStagingState = (stateDir) => // BLI-23346: once per loop
72
+ bookkeepingRead(rawEvidenceStagingStatePath(stateDir), () => readStagingFromDisk(stateDir));
73
+ async function readStagingFromDisk(stateDir) {
71
74
  try {
72
75
  const raw = JSON.parse(await fs.readFile(rawEvidenceStagingStatePath(stateDir), "utf8"));
73
76
  const state = parseStagingState(raw);
@@ -94,11 +97,12 @@ export async function readRawEvidenceStagingState(stateDir) {
94
97
  * lose every attempt count at once — which is exactly the thing that must not
95
98
  * be losable.
96
99
  */
97
- export async function writeRawEvidenceStagingState(stateDir, state) {
100
+ export const writeRawEvidenceStagingState = (stateDir, state) => bookkeepingWrite(rawEvidenceStagingStatePath(stateDir), state, (held) => writeStagingToDisk(stateDir, held));
101
+ async function writeStagingToDisk(stateDir, state) {
98
102
  const filePath = rawEvidenceStagingStatePath(stateDir);
99
103
  await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
100
104
  const pruned = pruneStagingState(state);
101
- const serialized = `${JSON.stringify(pruned, null, 2)}\n`;
105
+ const serialized = `${JSON.stringify(pruned)}\n`; // compact (BLI-23346); readers accept either
102
106
  const tempPath = `${filePath}.tmp-${process.pid}-${crypto.randomUUID()}`;
103
107
  let handle = null;
104
108
  try {
@@ -7,14 +7,24 @@ export const SELF_UPDATE_THROTTLE_MARKER = ".last-self-update-check";
7
7
  * Put the running node binary's directory on PATH for scheduled npm commands.
8
8
  * Schedulers use a bare PATH; npm's shim ships beside node on both host families.
9
9
  */
10
- export function envWithNodeRuntimeOnPath(env, nodeExecutable = process.execPath) {
11
- const nodeBinDir = path.dirname(nodeExecutable);
12
- const currentPath = env["PATH"] ?? "";
13
- if (currentPath.split(path.delimiter).includes(nodeBinDir))
10
+ export function envWithNodeRuntimeOnPath(env, nodeExecutable = process.execPath,
11
+ /** Which platform's rules the PATH follows. Omitted: this host's own. */
12
+ platform) {
13
+ const api = platform === undefined ? path : platform === "win32" ? path.win32 : path.posix;
14
+ // The running node's folder means nothing on a platform it is not from (an install
15
+ // simulated for another platform): leave that PATH exactly as the caller wrote it.
16
+ if (!api.isAbsolute(nodeExecutable))
17
+ return env;
18
+ const nodeBinDir = api.dirname(nodeExecutable);
19
+ // Windows spells the variable `Path`, and `{ ...process.env }` keeps that spelling: reading
20
+ // only `PATH` saw an empty PATH and left two spellings in the child's environment (BLI-23322).
21
+ const pathKey = api.sep === "\\" ? Object.keys(env).find((key) => key.toUpperCase() === "PATH") ?? "PATH" : "PATH";
22
+ const currentPath = env[pathKey] ?? "";
23
+ if (currentPath.split(api.delimiter).includes(nodeBinDir))
14
24
  return env;
15
25
  return {
16
26
  ...env,
17
- PATH: [nodeBinDir, currentPath].filter(Boolean).join(path.delimiter),
27
+ [pathKey]: [nodeBinDir, currentPath].filter(Boolean).join(api.delimiter),
18
28
  };
19
29
  }
20
30
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.272",
3
+ "version": "0.2.274",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -27,9 +27,9 @@
27
27
  "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
28
28
  },
29
29
  "dependencies": {
30
- "@bli-cockpit/memory-mcp": "0.1.82",
31
- "@bli-cockpit/mcp": "0.1.166",
32
- "@bli-cockpit/telemetry-core": "0.1.89",
30
+ "@bli-cockpit/memory-mcp": "0.1.83",
31
+ "@bli-cockpit/mcp": "0.1.167",
32
+ "@bli-cockpit/telemetry-core": "0.1.90",
33
33
  "unpdf": "1.8.1"
34
34
  },
35
35
  "repository": {