@bli-cockpit/cli 0.2.271 → 0.2.273

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -36,7 +36,7 @@ function parseMainRead(data) {
36
36
  export async function readSidecar(memo, localPath) {
37
37
  const stamp = memo ? await statSourceForMemo(localPath) : null;
38
38
  if (memo && stamp) {
39
- const hit = memo.recall(localPath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseSidecarRead);
39
+ const hit = await memo.recall(localPath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseSidecarRead);
40
40
  if (hit)
41
41
  return hit;
42
42
  }
@@ -60,7 +60,7 @@ export async function readSidecar(memo, localPath) {
60
60
  export async function readMain(memo, filePath) {
61
61
  const stamp = memo ? await statSourceForMemo(filePath) : null;
62
62
  if (memo && stamp) {
63
- const hit = memo.recall(filePath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseMainRead);
63
+ const hit = await memo.recall(filePath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseMainRead);
64
64
  if (hit)
65
65
  return hit;
66
66
  }
@@ -0,0 +1,52 @@
1
+ import { SESSION_FACTS_VERSION, SKILL_USE_VERSION } from "@bli-cockpit/telemetry-core";
2
+ import { readCodexMetadataSignals } from "./codex-session-signals.js";
3
+ import { statSourceForMemo } from "./transcript-read-memo.js";
4
+ /**
5
+ * The read attribution does of one Codex session file, remembered by the file's stamp (BLI-23346), the sibling
6
+ * of Claude's `claude-attribution-memoized-reads.ts`. Before this a tick re-streamed and re-hashed every Codex
7
+ * session in the window (177 files, 490 MB on the reference Mac) although no byte had changed. A Codex session
8
+ * is one file, so what is remembered is the finished answer of the streaming pass: the metadata signals, the
9
+ * sha256 and size of the bytes, and the finalized facts. The stamp (size, inode, mtime and ctime in
10
+ * nanoseconds) is the prepare memo's, so an append, rewrite or replace is read fresh. Scoring against the
11
+ * worktrees and the cwd-exists check run after the read, on every tick, so a repo that appears or vanishes is
12
+ * seen at once. Bump SIGNALS_VERSION when `codex-session-signals.ts` changes what it extracts.
13
+ */
14
+ const SIGNALS_VERSION = "codex-signals.v1";
15
+ export const CODEX_READ_RECIPE = ["codex-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION].join("|");
16
+ const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
17
+ const isNum = (v) => typeof v === "number" && Number.isFinite(v);
18
+ const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
19
+ function parseCodexRead(data) {
20
+ if (!isObj(data) || !isNum(data["byteSize"]) || typeof data["contentHashSha256"] !== "string")
21
+ return null;
22
+ const s = data["signals"];
23
+ const facts = data["facts"];
24
+ if (!isObj(s) || !isObj(facts))
25
+ return null;
26
+ for (const key of ["session_ids", "cwds", "workspace_roots", "branches", "commit_hashes", "repository_urls"]) {
27
+ if (!isStrings(s[key]))
28
+ return null;
29
+ }
30
+ if (!isNum(s["line_count"]) || !isNum(s["parse_error_count"]))
31
+ return null;
32
+ if (typeof facts["extractor_version"] !== "string" || facts["source"] !== "codex" || !isObj(facts["usage_by_model"]) || !isNum(facts["transcript_bytes"]))
33
+ return null;
34
+ return data;
35
+ }
36
+ /** The session's signals, hash, size and facts: recalled when its stamp is unchanged, else streamed (a read failure throws). */
37
+ export async function readCodexMetadataMemoized(memo, filePath) {
38
+ const stamp = memo ? await statSourceForMemo(filePath) : null;
39
+ if (memo && stamp) {
40
+ const hit = await memo.recall(filePath, stamp, CODEX_READ_RECIPE, stamp.size, parseCodexRead);
41
+ if (hit)
42
+ return hit;
43
+ }
44
+ else
45
+ memo?.note("stamp_missing");
46
+ const read = await readCodexMetadataSignals(filePath);
47
+ if (memo && stamp) {
48
+ memo.note("fresh");
49
+ memo.remember(filePath, stamp, CODEX_READ_RECIPE, read);
50
+ }
51
+ return read;
52
+ }
@@ -2,7 +2,8 @@ import { existsSync } from "node:fs";
2
2
  import fs from "node:fs/promises";
3
3
  import path from "node:path";
4
4
  import { describeError } from "../health-detail.js";
5
- import { extractCodexSessionSignals, readCodexMetadataSignals, } from "./codex-session-signals.js";
5
+ import { extractCodexSessionSignals, } from "./codex-session-signals.js";
6
+ import { readCodexMetadataMemoized } from "./codex-attribution-memoized-reads.js";
6
7
  import { sanitizeSessionId, scoreSignalsAgainstWorktrees, sessionIdFromFileName, shortHash, } from "./attribution-core.js";
7
8
  /**
8
9
  * Deterministic Codex session JSONL -> repo/worktree attribution.
@@ -41,7 +42,7 @@ export async function scanAndAttributeCodexSessions(options) {
41
42
  const files = discovery.files.slice(0, limit);
42
43
  const results = [];
43
44
  for (const file of files) {
44
- results.push(await attributeOneSession(file, options.worktrees, options.collectionRoots ?? []));
45
+ results.push(await attributeOneSession(file, options.worktrees, options.collectionRoots ?? [], options.readMemo));
45
46
  }
46
47
  return {
47
48
  results,
@@ -164,11 +165,11 @@ function dedupePaths(values) {
164
165
  }
165
166
  return out;
166
167
  }
167
- async function attributeOneSession(file, worktrees, collectionRoots) {
168
+ async function attributeOneSession(file, worktrees, collectionRoots, readMemo) {
168
169
  const base = createSessionBase(file);
169
170
  let read;
170
171
  try {
171
- read = await readCodexMetadataSignals(file.file);
172
+ read = await readCodexMetadataMemoized(readMemo, file.file);
172
173
  }
173
174
  catch (error) {
174
175
  // One skipped session, with `file_read_failed` on its row. Logged per
@@ -27,7 +27,7 @@ export async function collectAgentImagesFromTranscript(collection, options) {
27
27
  const memo = collection.budget.readMemo;
28
28
  const stamp = memo ? await statSourceForMemo(options.filePath) : null;
29
29
  const remembered = memo && stamp
30
- ? memo.recall(options.filePath, stamp, IMAGE_SCAN_RECIPE, stamp.size, (data) => (data === "no_image_candidates" ? true : null))
30
+ ? await memo.recall(options.filePath, stamp, IMAGE_SCAN_RECIPE, stamp.size, (data) => (data === "no_image_candidates" ? true : null))
31
31
  : null;
32
32
  if (!remembered && memo && !stamp)
33
33
  memo.note("stamp_missing");
@@ -5,49 +5,77 @@
5
5
  * transcript the image step scans: on the reference Mac 33 of the 37 seconds of main-thread JavaScript went to
6
6
  * reading, parsing and hashing files whose size and mtime had not moved since the tick before. This is the
7
7
  * sibling of the prepare memo (`raw-evidence-prepare-memo.ts`) and uses ITS stamp: size plus inode, mtime and
8
- * ctime in nanoseconds, so an append, rewrite or replace is never mistaken for "unchanged". The entry cap, the
9
- * `seen_ms` ageing and the atomic write are the prepare memo's own helpers.
8
+ * ctime in nanoseconds, so an append, rewrite or replace is never mistaken for "unchanged". The entry cap and
9
+ * the 14-day forgetting are the prepare memo's own numbers.
10
10
  *
11
11
  * An entry is trusted only when the stamp AND the recipe match. The recipe names the rules that produced the
12
12
  * remembered answer, so changing a rule rebuilds the memo once. What an entry holds is decided by its caller
13
13
  * (`data`); this file never sees a byte of transcript content, and neither does the file on disk: signals,
14
14
  * hashes, sizes and counts only. A memo that is lost, corrupt or stale in the safe direction costs one slow
15
15
  * tick, never a wrong answer. The file path is hashed, so no path reaches the memo either.
16
+ *
17
+ * BLI-23346: one small file per source key under `transcript-read-memo-<name>/`, read on demand. The first
18
+ * layout was one JSON file; on the reference Mac that reached 18.2 MB (2,931 entries, 1.1 M timestamps), was
19
+ * parsed whole at open, held through both scan passes and rewritten whole whenever one transcript was fresh.
20
+ * Now an unchanged transcript costs one small read that is parsed and dropped, a changed one costs one small
21
+ * atomic write, and nothing else is touched. A hit older than a day gets its `seen_ms` rewritten (one small
22
+ * file), and a prune pass (at most daily, by file mtime) forgets what was not written or refreshed for 14 days
23
+ * and the oldest beyond the cap. Exact facts are unchanged: an entry holds the same `data` the caller always
24
+ * stored (request-id-ordered usage, delta-encoded timestamps), it is only filed apart.
16
25
  */
17
26
  import { createHash } from "node:crypto";
27
+ import fs from "node:fs/promises";
18
28
  import path from "node:path";
19
- import { isMissingFileFailure } from "../health-detail.js";
20
- import { readJsonFile, writeJsonFileAtomic } from "../local-state-files.js";
21
- import { PREPARE_MEMO_MAX_ENTRIES, refreshSeen, trimEntries, } from "./raw-evidence-prepare-memo.js";
29
+ import { describeError } from "../health-detail.js";
30
+ import { isMissingFileError, writeJsonFileAtomic } from "../local-state-files.js";
31
+ import { FORGET_AFTER_MS, PREPARE_MEMO_MAX_ENTRIES, SEEN_REFRESH_MS, } from "./raw-evidence-prepare-memo.js";
22
32
  export { sourceStampOf, statSourceForMemo } from "./raw-evidence-prepare-memo.js";
23
- const MEMO_FORMAT = "transcript-read-memo.v1";
33
+ const MEMO_FORMAT = "transcript-read-memo.v2";
24
34
  const TAG = "[transcript-reads]";
35
+ const PRUNE_EVERY_MS = 24 * 3_600_000;
36
+ const PRUNE_MARKER = ".pruned";
37
+ const WRITE_CONCURRENCY = 8;
25
38
  export function transcriptReadMemoPath(stateDir, name) {
26
- return path.join(stateDir, `transcript-read-memo-${name}.json`);
39
+ return path.join(stateDir, `transcript-read-memo-${name}`);
27
40
  }
28
41
  const keyOf = (filePath) => createHash("sha256").update(filePath).digest("hex").slice(0, 32);
42
+ const zeroCounts = () => ({ reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0, unreadable: 0, refreshed: 0 });
29
43
  export async function openTranscriptReadMemo(stateDir, name) {
30
- const entries = await readEntries(stateDir, name);
31
- let dirty = false;
32
- let counts = { reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0 };
33
- const recalled = new Set();
44
+ const dir = transcriptReadMemoPath(stateDir, name);
45
+ const entryPath = (key) => path.join(dir, `${key}.json`);
46
+ /** Entries remembered this pass, written at `settle`; also answers a second pass over the same file before then. */
47
+ const pending = new Map();
48
+ let counts = zeroCounts();
49
+ await removeLegacySingleFile(stateDir, name);
34
50
  return {
35
- recall(filePath, stamp, recipe, bytes, parse) {
51
+ async recall(filePath, stamp, recipe, bytes, parse) {
36
52
  const key = keyOf(filePath);
37
- const entry = entries.get(key);
53
+ const fromPending = pending.get(key);
54
+ let entry = fromPending;
55
+ if (!entry) {
56
+ const loaded = await readEntry(entryPath(key));
57
+ if (loaded === "unreadable")
58
+ counts.unreadable += 1;
59
+ else if (loaded)
60
+ entry = loaded;
61
+ }
38
62
  if (!entry || entry.recipe !== recipe || entry.size !== stamp.size || entry.identity !== stamp.identity)
39
63
  return null;
40
64
  const data = parse(entry.data);
41
65
  if (data === null)
42
66
  return null;
43
- recalled.add(key);
44
67
  counts.reused += 1;
45
68
  counts.bytes_not_read += bytes;
69
+ const now = Date.now();
70
+ if (!fromPending && now - entry.seen_ms > SEEN_REFRESH_MS) {
71
+ entry.seen_ms = now;
72
+ counts.refreshed += 1;
73
+ await writeJsonFileAtomic(entryPath(key), entry, { compact: true }).catch(() => undefined);
74
+ }
46
75
  return data;
47
76
  },
48
77
  remember(filePath, stamp, recipe, data, now = Date.now()) {
49
- entries.set(keyOf(filePath), { ...stamp, recipe, data, seen_ms: now });
50
- dirty = true;
78
+ pending.set(keyOf(filePath), { schema_version: MEMO_FORMAT, ...stamp, recipe, data, seen_ms: now });
51
79
  },
52
80
  note(outcome) {
53
81
  if (outcome === "fresh")
@@ -57,53 +85,102 @@ export async function openTranscriptReadMemo(stateDir, name) {
57
85
  },
58
86
  async settle(now = Date.now()) {
59
87
  const done = counts;
60
- counts = { reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0 };
61
- dirty = refreshSeen(entries, recalled, now) || dirty;
62
- recalled.clear();
63
- let written = false;
64
- let reason = done.reused > 0 ? "stamp_unchanged" : done.fresh > 0 ? "no_reusable_entry" : "stamp_missing";
65
- if (dirty) {
66
- const before = entries.size;
67
- const kept = trimEntries(entries, now, PREPARE_MEMO_MAX_ENTRIES);
68
- try {
69
- await writeJsonFileAtomic(transcriptReadMemoPath(stateDir, name), { schema_version: MEMO_FORMAT, entries: Object.fromEntries(kept) }, { compact: true });
70
- dirty = false;
71
- written = true;
72
- }
73
- catch (error) {
74
- reason = "memo_write_failed";
75
- console.error(TAG + " memo not written; the next tick reads these files again", JSON.stringify({ memo: name, reason, entries: before, error_name: error instanceof Error ? error.name : typeof error }));
88
+ counts = zeroCounts();
89
+ const toWrite = [...pending.entries()];
90
+ pending.clear();
91
+ let written = 0;
92
+ let failed = 0;
93
+ for (let i = 0; i < toWrite.length; i += WRITE_CONCURRENCY) {
94
+ const outcomes = await Promise.all(toWrite.slice(i, i + WRITE_CONCURRENCY).map(([key, entry]) => writeJsonFileAtomic(entryPath(key), entry, { compact: true }).then(() => true, (error) => {
95
+ if (failed === 0)
96
+ console.error(TAG + " memo entry not written; that file is read again next tick", JSON.stringify({ memo: name, reason: "memo_write_failed", ...describeError(error) }));
97
+ return false;
98
+ })));
99
+ for (const ok of outcomes) {
100
+ if (ok)
101
+ written += 1;
102
+ else
103
+ failed += 1;
76
104
  }
77
105
  }
78
- if (done.reused + done.fresh + done.stamp_missing === 0)
106
+ const pruned = await pruneIfDue(dir, name, now);
107
+ if (done.reused + done.fresh + done.stamp_missing === 0 && pruned === null)
79
108
  return;
80
- console.error(TAG + " tick", JSON.stringify({ memo: name, reason, reused: done.reused, read_fresh: done.fresh, stamp_missing: done.stamp_missing, bytes_not_read: done.bytes_not_read, entries: entries.size, memo_written: written }));
109
+ const reason = failed > 0 ? "memo_write_failed" : done.reused > 0 ? "stamp_unchanged" : done.fresh > 0 ? "no_reusable_entry" : "stamp_missing";
110
+ console.error(TAG + " tick", JSON.stringify({
111
+ memo: name,
112
+ reason,
113
+ reused: done.reused,
114
+ read_fresh: done.fresh,
115
+ stamp_missing: done.stamp_missing,
116
+ bytes_not_read: done.bytes_not_read,
117
+ entries_written: written,
118
+ entries_write_failed: failed,
119
+ entries_unreadable: done.unreadable,
120
+ seen_refreshed: done.refreshed,
121
+ ...(pruned ? { pruned_expired: pruned.expired, pruned_over_cap: pruned.overCap, kept: pruned.kept } : {}),
122
+ }));
81
123
  },
82
124
  };
83
125
  }
84
- async function readEntries(stateDir, name) {
85
- const entries = new Map();
86
- let raw;
126
+ /** One entry file; null when absent; "unreadable" when it exists but is not an entry (a miss, rewritten by the next fresh read). */
127
+ async function readEntry(file) {
128
+ let text;
129
+ try {
130
+ text = await fs.readFile(file, "utf8");
131
+ }
132
+ catch (error) {
133
+ return isMissingFileError(error) ? null : "unreadable";
134
+ }
135
+ try {
136
+ const e = JSON.parse(text);
137
+ if (e && e.schema_version === MEMO_FORMAT && typeof e.size === "number" && typeof e.identity === "string" && typeof e.recipe === "string" && typeof e.seen_ms === "number" && e.data !== undefined)
138
+ return e;
139
+ }
140
+ catch {
141
+ // A torn or foreign file is a miss; the caller counts it as unreadable.
142
+ }
143
+ return "unreadable";
144
+ }
145
+ /** The pre-BLI-23346 single file is dead weight (18 MB on the reference Mac): remove it once and say so. */
146
+ async function removeLegacySingleFile(stateDir, name) {
147
+ const legacy = path.join(stateDir, `transcript-read-memo-${name}.json`);
87
148
  try {
88
- raw = await readJsonFile(transcriptReadMemoPath(stateDir, name));
149
+ const { size } = await fs.stat(legacy);
150
+ await fs.rm(legacy, { force: true });
151
+ console.error(TAG + " legacy single-file memo removed", JSON.stringify({ memo: name, reason: "legacy_memo_replaced_by_entry_files", bytes: size }));
89
152
  }
90
153
  catch (error) {
91
- if (!isMissingFileFailure(error))
92
- console.error(TAG + " memo unreadable; every file is read once", JSON.stringify({ memo: name, reason: "read_memo_unreadable" }));
93
- return entries;
154
+ if (!isMissingFileError(error))
155
+ console.error(TAG + " legacy memo not removed", JSON.stringify({ memo: name, reason: "legacy_memo_remove_failed", ...describeError(error) }));
94
156
  }
95
- const stored = raw;
96
- if (stored?.schema_version !== MEMO_FORMAT || !stored.entries || typeof stored.entries !== "object")
97
- return entries;
98
- let dropped = 0;
99
- for (const [key, value] of Object.entries(stored.entries)) {
100
- const e = value;
101
- if (e && typeof e.size === "number" && typeof e.identity === "string" && typeof e.recipe === "string" && typeof e.seen_ms === "number" && e.data !== undefined)
102
- entries.set(key, e);
103
- else
104
- dropped += 1;
157
+ }
158
+ /** At most once a day: forget entries not written or refreshed for 14 days, then the oldest beyond the cap. Null when not due or nothing to prune. */
159
+ async function pruneIfDue(dir, name, now) {
160
+ const marker = path.join(dir, PRUNE_MARKER);
161
+ try {
162
+ const last = await fs.stat(marker).then((s) => s.mtimeMs, () => 0);
163
+ if (now - last < PRUNE_EVERY_MS)
164
+ return null;
165
+ const names = (await fs.readdir(dir)).filter((n) => n.endsWith(".json"));
166
+ const aged = [];
167
+ for (const n of names) {
168
+ const file = path.join(dir, n);
169
+ const stat = await fs.stat(file).catch(() => null);
170
+ if (stat)
171
+ aged.push({ file, mtimeMs: stat.mtimeMs });
172
+ }
173
+ const expired = aged.filter((a) => now - a.mtimeMs > FORGET_AFTER_MS);
174
+ const live = aged.filter((a) => now - a.mtimeMs <= FORGET_AFTER_MS).sort((a, b) => b.mtimeMs - a.mtimeMs);
175
+ const over = live.slice(PREPARE_MEMO_MAX_ENTRIES);
176
+ await Promise.all([...expired, ...over].map((a) => fs.rm(a.file, { force: true })));
177
+ await fs.writeFile(marker, `${new Date(now).toISOString()}\n`);
178
+ await fs.utimes(marker, new Date(now), new Date(now));
179
+ return { expired: expired.length, overCap: over.length, kept: live.length - over.length };
180
+ }
181
+ catch (error) {
182
+ if (!isMissingFileError(error))
183
+ console.error(TAG + " memo prune failed", JSON.stringify({ memo: name, reason: "memo_prune_failed", ...describeError(error) }));
184
+ return null;
105
185
  }
106
- if (dropped > 0)
107
- console.error(TAG + " memo entries dropped as malformed", JSON.stringify({ memo: name, reason: "read_memo_entries_malformed", dropped, kept: entries.size }));
108
- return entries;
109
186
  }
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Tick-scoped bookkeeping: load each big state file once, write it at checkpoints (BLI-23346).
3
+ *
4
+ * The worktree loop of one sync tick used to read the 7 MB raw-evidence cursor and the staging state from disk,
5
+ * mutate them and write them back (serialise, fsync, rename) once per worktree, about 45 times a tick on the
6
+ * reference Mac: ~650 MB of JSON churn and ~326 MB of fsync'd writes to say "nothing changed". Inside
7
+ * `withBookkeepingSession` the first read of a file loads it, later reads return the SAME in-memory object, and a
8
+ * write only marks it dirty. Dirty files are persisted by the SAME writer the caller named (so fsync, temp file,
9
+ * rename and pruning are unchanged) at three points: when `CHECKPOINT_MS` has passed since the last flush, and
10
+ * when the session ends (also on an exception or a yielded loop, before the sync lock is released). A process
11
+ * killed between checkpoints loses at most that window of bookkeeping, which only means objects are re-offered
12
+ * next tick (the server dedups by content hash); it never loses a delivery.
13
+ *
14
+ * Outside a session every call is the plain disk read / write it always was, so a lone `cockpit sync`, backfill
15
+ * or doctor behaves exactly as before. Only the worktree loop opens a session, and it holds the sync lock.
16
+ */
17
+ import { AsyncLocalStorage } from "node:async_hooks";
18
+ import { describeError } from "./health-detail.js";
19
+ const TAG = "[bookkeeping]";
20
+ export const CHECKPOINT_MS = 30_000;
21
+ const storage = new AsyncLocalStorage();
22
+ /** The value for `key`: loaded by `load` on first use inside a session, then served from memory. Outside a session, `load()`. */
23
+ export async function bookkeepingRead(key, load) {
24
+ const session = storage.getStore();
25
+ if (!session)
26
+ return load();
27
+ const held = session.entries.get(key);
28
+ if (held) {
29
+ session.memoryReads += 1;
30
+ return held.value;
31
+ }
32
+ const value = await load();
33
+ session.diskReads += 1;
34
+ // `persist` is filled in by the first write; a never-written entry is never dirty.
35
+ session.entries.set(key, { value, persist: async () => undefined, dirty: false });
36
+ return value;
37
+ }
38
+ /** Record `value` as the new state of `key`. Inside a session it is persisted at the next checkpoint; outside, now. */
39
+ export async function bookkeepingWrite(key, value, persist) {
40
+ const session = storage.getStore();
41
+ if (!session)
42
+ return persist(value);
43
+ session.writesRequested += 1;
44
+ session.entries.set(key, { value, persist: persist, dirty: true });
45
+ if (Date.now() - session.lastFlushMs >= CHECKPOINT_MS)
46
+ await flush(session, "checkpoint");
47
+ }
48
+ async function flush(session, why) {
49
+ session.lastFlushMs = Date.now();
50
+ const dirty = [...session.entries.values()].filter((entry) => entry.dirty);
51
+ if (dirty.length === 0)
52
+ return;
53
+ session.flushes += 1;
54
+ for (const entry of dirty) {
55
+ entry.dirty = false;
56
+ try {
57
+ await entry.persist(entry.value);
58
+ session.filesFlushed += 1;
59
+ }
60
+ catch (error) {
61
+ entry.dirty = true;
62
+ console.error(TAG + " state not written", JSON.stringify({ reason: why === "end" ? "end_flush_failed" : "checkpoint_flush_failed", ...describeError(error) }));
63
+ if (why === "checkpoint")
64
+ throw error;
65
+ }
66
+ }
67
+ }
68
+ /** Run `fn` with bookkeeping held in memory; flush whatever is dirty when it settles, however it settles. */
69
+ export async function withBookkeepingSession(fn) {
70
+ if (storage.getStore())
71
+ return fn();
72
+ const session = { entries: new Map(), lastFlushMs: Date.now(), diskReads: 0, memoryReads: 0, writesRequested: 0, flushes: 0, filesFlushed: 0 };
73
+ try {
74
+ return await storage.run(session, fn);
75
+ }
76
+ finally {
77
+ await flush(session, "end");
78
+ if (session.diskReads + session.memoryReads + session.writesRequested > 0) {
79
+ console.error(TAG + " session settled", JSON.stringify({ reason: "session_end", disk_reads: session.diskReads, memory_reads: session.memoryReads, writes_requested: session.writesRequested, flush_count: session.flushes, files_written: session.filesFlushed }));
80
+ }
81
+ }
82
+ }
@@ -31,7 +31,7 @@ export async function resolveMemoryMcpBin(options) {
31
31
  export async function resolveMemoryConfig(command, io, platform, deps) {
32
32
  const dashboardUrl = await resolveDashboardUrl(command, deps);
33
33
  const found = await resolveMemoryMcpBin({
34
- env: envWithNodeRuntimeOnPath(io.env ?? process.env),
34
+ env: envWithNodeRuntimeOnPath(io.env ?? process.env, undefined, platform),
35
35
  platform,
36
36
  fileExists: deps.fileExists,
37
37
  cliEntryPoint: deps.cliEntryPoint,
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
21
21
  }
22
22
 
23
23
  if (command === "--version" || command === "-V" || command === "version") {
24
- writeLine(io?.stdout ?? process.stdout, "0.2.271");
24
+ writeLine(io?.stdout ?? process.stdout, "0.2.273");
25
25
  return 0;
26
26
  }
27
27
 
@@ -12,6 +12,7 @@ import { saveRootWorkScan } from "../root-work.js";
12
12
  import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
13
13
  import { scanAndAttributeClaudeSessions, } from "../adapters/claude-attribution.js";
14
14
  import { CLAUDE_CURSOR_FILENAME } from "../cursors/raw-evidence-cursor.js";
15
+ import { isRedactionStubObjectKey } from "../cursors/raw-evidence-stub-entry.js";
15
16
  import { clearSourceRetryFailure, recordSourceRetryFailure, } from "../spool/local-spool.js";
16
17
  import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
17
18
  import { widenForCatchUp } from "./session-sync-outside-roots.js";
@@ -37,6 +38,7 @@ export async function scanAndAttributeBothSources(options) {
37
38
  retryPending: plan.codexRetryPending,
38
39
  retrySinceMinutes: plan.codexRetrySinceMinutes,
39
40
  outsideRootCatchUpPending: plan.outsideRootCatchUpPending,
41
+ stateDir: options.paths.state_dir,
40
42
  });
41
43
  // First run (D B.4 §8): without a Claude cursor yet, widen the window to 14
42
44
  // days so the first sync captures retroactive history instead of only 24h.
@@ -60,7 +62,7 @@ export async function scanAndAttributeBothSources(options) {
60
62
  source: "codex",
61
63
  attemptedAt: options.now.toISOString(),
62
64
  pendingBefore: plan.codexSourceRetryPending,
63
- reason: sourceScanRetryReason("codex", codexAttribution),
65
+ reason: sourceScanRetryReason("codex", codexAttribution, plan.codexCursorBefore),
64
66
  });
65
67
  }
66
68
  if (plan.claudeEnabled) {
@@ -69,7 +71,7 @@ export async function scanAndAttributeBothSources(options) {
69
71
  source: "claude_code",
70
72
  attemptedAt: options.now.toISOString(),
71
73
  pendingBefore: plan.claudeSourceRetryPending,
72
- reason: sourceScanRetryReason("claude_code", claudeAttribution),
74
+ reason: sourceScanRetryReason("claude_code", claudeAttribution, plan.claudeCursorBefore),
73
75
  });
74
76
  }
75
77
  await saveRootWorkScan(options.paths, [...codexAttribution.results, ...claudeAttribution.results], options.now);
@@ -89,28 +91,37 @@ async function scanCodexSessionsForSync(options) {
89
91
  const sinceMinutes = widenForCatchUp(options.retryPending
90
92
  ? options.retrySinceMinutes
91
93
  : CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, options.outsideRootCatchUpPending, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES);
92
- let scan = await scanAndAttributeCodexSessions({
93
- sessionsDirs: defaultCodexSessionDirs(options.homeDir),
94
- worktrees: options.worktrees,
95
- now: options.now,
96
- sinceMinutes,
97
- limit: CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT,
98
- collectionRoots: options.collectionRoots,
99
- });
100
- if (scan.session_limit_applied) {
101
- // Re-run once, bounded by exactly what the first pass discovered, so a
102
- // capped scan still returns every session it found instead of silently
103
- // truncating at the window's default limit.
104
- scan = await scanAndAttributeCodexSessions({
94
+ // BLI-23346: one memo for both passes, settled once, so an unchanged session file is never streamed again.
95
+ const readMemo = await openTranscriptReadMemo(options.stateDir, "codex_attribution").catch(() => undefined);
96
+ try {
97
+ let scan = await scanAndAttributeCodexSessions({
105
98
  sessionsDirs: defaultCodexSessionDirs(options.homeDir),
106
99
  worktrees: options.worktrees,
107
100
  now: options.now,
108
101
  sinceMinutes,
109
- limit: scan.discovered_file_count,
102
+ limit: CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT,
110
103
  collectionRoots: options.collectionRoots,
104
+ readMemo,
111
105
  });
106
+ if (scan.session_limit_applied) {
107
+ // Re-run once, bounded by exactly what the first pass discovered, so a
108
+ // capped scan still returns every session it found instead of silently
109
+ // truncating at the window's default limit.
110
+ scan = await scanAndAttributeCodexSessions({
111
+ sessionsDirs: defaultCodexSessionDirs(options.homeDir),
112
+ worktrees: options.worktrees,
113
+ now: options.now,
114
+ sinceMinutes,
115
+ limit: scan.discovered_file_count,
116
+ collectionRoots: options.collectionRoots,
117
+ readMemo,
118
+ });
119
+ }
120
+ return scan;
121
+ }
122
+ finally {
123
+ await readMemo?.settle();
112
124
  }
113
- return scan;
114
125
  }
115
126
  /**
116
127
  * Scan and attribute Claude sessions for this sync — the Claude counterpart to
@@ -155,7 +166,7 @@ async function scanClaudeSessionsForSync(options) {
155
166
  await readMemo?.settle();
156
167
  }
157
168
  }
158
- export function sourceScanRetryReason(source, scan) {
169
+ export function sourceScanRetryReason(source, scan, cursorBefore) {
159
170
  const reasons = new Set();
160
171
  const failure = sourceScanFailureReason(source, scan);
161
172
  if (failure)
@@ -164,11 +175,30 @@ export function sourceScanRetryReason(source, scan) {
164
175
  // reason to widen the next scan window, because the transcript fallback can
165
176
  // still attribute it. It is not a reason to call this sync failed — see
166
177
  // `sourceScanFailureReason`.
167
- if (scan.results.some((result) => result.reason === "repo_not_on_disk")) {
178
+ //
179
+ // BLI-23346: only while that session still owes its upload. A `repo_not_on_disk` session the cursor already
180
+ // holds a durable pointer for owes nothing, and widening for it re-scanned 30 days every tick on the reference
181
+ // Mac (the widened window found the same already-delivered sessions again, which re-armed the flag).
182
+ const owing = scan.results.filter((result) => result.reason === "repo_not_on_disk" && sessionStillOwesUpload(source, result, cursorBefore));
183
+ if (owing.length > 0) {
168
184
  reasons.add("repo_not_on_disk");
169
185
  }
186
+ else if (cursorBefore) {
187
+ const delivered = scan.results.filter((result) => result.reason === "repo_not_on_disk").length;
188
+ if (delivered > 0)
189
+ console.error("[session-sync] repo_not_on_disk sessions are already delivered, no scan retry kept", JSON.stringify({ reason: "repo_not_on_disk_already_durable", source, delivered_count: delivered }));
190
+ }
170
191
  return reasons.size > 0 ? [...reasons].sort().join(",") : null;
171
192
  }
193
+ function sessionStillOwesUpload(source, result, cursorBefore) {
194
+ if (!cursorBefore)
195
+ return true;
196
+ const sessionId = source === "codex"
197
+ ? result.codex_session_id
198
+ : result.claude_session_id;
199
+ const key = cursorBefore.sessions[sessionId]?.uploaded_object_key;
200
+ return !key || isRedactionStubObjectKey(key);
201
+ }
172
202
  /**
173
203
  * The part of the scan outcome that is a genuine FAILURE: the session store
174
204
  * itself could not be read, so sessions that exist were not seen.
@@ -6,6 +6,7 @@
6
6
  * and the Claude growth damper lives here because it is the only step that can
7
7
  * see both what the cursor already made durable and what is about to be sent.
8
8
  */
9
+ import { withBookkeepingSession } from "../bookkeeping-session.js";
9
10
  import { startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from "../local-state.js";
10
11
  import { LocalUploadBlockedError, syncLocalAmbientEnvelope, } from "../upload.js";
11
12
  import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET, } from "../adapters/raw-evidence.js";
@@ -41,12 +42,14 @@ export async function syncEveryTargetWorktree(options) {
41
42
  };
42
43
  // BLI-11112: a waiting backfill takes the lock at the next worktree
43
44
  // boundary; the worktrees not reached stay pending for it or the next tick.
44
- const loop = await runYieldingAtBoundaries({
45
+ // BLI-23346: the cursor and the staging state are loaded once for the whole loop and written at checkpoints and
46
+ // when the loop ends (before the lock is released), not read and rewritten per worktree.
47
+ const loop = await withBookkeepingSession(() => runYieldingAtBoundaries({
45
48
  items: syncWorktrees,
46
49
  shouldYield: run.yieldCheck,
47
50
  describe: (outcome) => ({ status: outcome.sync.status }),
48
51
  runOne: (worktree) => syncTargetWithAllowance(options, worktree, syncWorktrees, rawEvidenceBudget, discoveredWorktreeKeys, dampedClaudePointers),
49
- });
52
+ }));
50
53
  const outcomes = loop.results;
51
54
  const everyWorktreeUploaded = outcomes.every((outcome) => outcome.sync.status === "uploaded");
52
55
  return {
@@ -26,7 +26,7 @@ import { envWithNodeRuntimeOnPath } from "../scheduled-self-update.js";
26
26
  async function resolveTowerConfig(deps) {
27
27
  const found = await resolveMcpBin({
28
28
  binName: TOWER_MCP_BIN,
29
- env: envWithNodeRuntimeOnPath(deps.env),
29
+ env: envWithNodeRuntimeOnPath(deps.env, undefined, deps.platform),
30
30
  platform: deps.platform,
31
31
  fileExists: deps.fileExists,
32
32
  cliEntryPoint: deps.cliEntryPoint,