@bli-cockpit/cli 0.2.251 → 0.2.253
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-attribution-memoized-reads.js +85 -0
- package/dist/adapters/claude-attribution-score.js +23 -34
- package/dist/adapters/claude-attribution.js +1 -1
- package/dist/adapters/raw-evidence-collection-state.js +2 -0
- package/dist/adapters/raw-evidence-image-reader.js +30 -6
- package/dist/adapters/raw-evidence-prepare-memo.js +39 -30
- package/dist/adapters/raw-evidence-transcript-file.js +22 -23
- package/dist/adapters/raw-evidence-volatile-reuse.js +27 -0
- package/dist/adapters/raw-evidence.js +2 -0
- package/dist/adapters/transcript-read-memo.js +109 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-scan.js +22 -11
- package/dist/crash-guard.js +9 -0
- package/dist/evidence-upload-deadline.js +36 -0
- package/dist/evidence-upload-direct.js +6 -5
- package/dist/evidence-upload-transport.js +7 -3
- package/dist/image-description-session.js +26 -7
- package/dist/slim/slim-transcript.js +11 -0
- package/dist/sync-lock.js +75 -6
- package/package.json +4 -4
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { captureSessionFactsDelta, observeSessionFactsContent, observeSessionFactsFile, parseSessionFactsDelta, SESSION_FACTS_VERSION, SKILL_USE_VERSION, } from "@bli-cockpit/telemetry-core";
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import { extractClaudeSessionSignals } from "./claude-attribution-signals.js";
|
|
5
|
+
import { statSourceForMemo } from "./transcript-read-memo.js";
|
|
6
|
+
/**
|
|
7
|
+
* The reads attribution does of one Claude file, remembered by the file's stamp (BLI-22797;
|
|
8
|
+
* `transcript-read-memo.ts`). Every read goes through the same capture-then-merge path whether the file was
|
|
9
|
+
* opened or recalled, so a recalled file leaves the session's facts exactly as a fresh read does by construction
|
|
10
|
+
* (`session-facts-delta.ts`). Bump SIGNALS_VERSION when `claude-attribution-signals.ts` changes what it extracts.
|
|
11
|
+
*/
|
|
12
|
+
const SIGNALS_VERSION = "signals.v1";
|
|
13
|
+
export const ATTRIBUTION_READ_RECIPE = ["attribution-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION].join("|");
|
|
14
|
+
const sha256 = (value) => crypto.createHash("sha256").update(value).digest("hex");
|
|
15
|
+
const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
|
|
16
|
+
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
17
|
+
const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
18
|
+
function parseSidecarRead(data) {
|
|
19
|
+
if (!isObj(data) || !isStrings(data["cwds"]) || !isNum(data["byte_size"]) || typeof data["sha256"] !== "string")
|
|
20
|
+
return null;
|
|
21
|
+
const facts = parseSessionFactsDelta(data["facts"]);
|
|
22
|
+
return facts ? { cwds: data["cwds"], byte_size: data["byte_size"], sha256: data["sha256"], facts } : null;
|
|
23
|
+
}
|
|
24
|
+
function parseMainRead(data) {
|
|
25
|
+
if (!isObj(data) || !isNum(data["byte_size"]) || typeof data["sha256"] !== "string" || !isObj(data["signals"]))
|
|
26
|
+
return null;
|
|
27
|
+
const s = data["signals"];
|
|
28
|
+
if (!isStrings(s["session_ids"]) || !isStrings(s["cwds"]) || !isStrings(s["branches"]) || !isStrings(s["repository_urls"]))
|
|
29
|
+
return null;
|
|
30
|
+
if (!isNum(s["line_count"]) || !isNum(s["parse_error_count"]) || !isNum(s["content_record_count"]) || !isNum(s["envelope_field_hit_rate"]))
|
|
31
|
+
return null;
|
|
32
|
+
const facts = parseSessionFactsDelta(data["facts"]);
|
|
33
|
+
return facts ? { signals: s, byte_size: data["byte_size"], sha256: data["sha256"], facts } : null;
|
|
34
|
+
}
|
|
35
|
+
/** The sidecar's cwds, size, hash and facts contribution: recalled when its stamp is unchanged, else read (a read failure throws). */
|
|
36
|
+
export async function readSidecar(memo, localPath) {
|
|
37
|
+
const stamp = memo ? await statSourceForMemo(localPath) : null;
|
|
38
|
+
if (memo && stamp) {
|
|
39
|
+
const hit = memo.recall(localPath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseSidecarRead);
|
|
40
|
+
if (hit)
|
|
41
|
+
return hit;
|
|
42
|
+
}
|
|
43
|
+
else
|
|
44
|
+
memo?.note("stamp_missing");
|
|
45
|
+
const raw = await fs.readFile(localPath);
|
|
46
|
+
const content = raw.toString("utf8");
|
|
47
|
+
const read = {
|
|
48
|
+
cwds: extractClaudeSessionSignals(content).cwds,
|
|
49
|
+
byte_size: raw.byteLength,
|
|
50
|
+
sha256: sha256(raw),
|
|
51
|
+
facts: captureSessionFactsDelta("claude_code", (acc) => void observeSessionFactsContent(acc, content)),
|
|
52
|
+
};
|
|
53
|
+
if (memo && stamp) {
|
|
54
|
+
memo.note("fresh");
|
|
55
|
+
memo.remember(localPath, stamp, ATTRIBUTION_READ_RECIPE, read);
|
|
56
|
+
}
|
|
57
|
+
return read;
|
|
58
|
+
}
|
|
59
|
+
/** The main transcript's signals, size, hash and facts contribution. `{ empty: true }` when it holds only whitespace (never remembered). */
|
|
60
|
+
export async function readMain(memo, filePath) {
|
|
61
|
+
const stamp = memo ? await statSourceForMemo(filePath) : null;
|
|
62
|
+
if (memo && stamp) {
|
|
63
|
+
const hit = memo.recall(filePath, stamp, ATTRIBUTION_READ_RECIPE, stamp.size, parseMainRead);
|
|
64
|
+
if (hit)
|
|
65
|
+
return hit;
|
|
66
|
+
}
|
|
67
|
+
else
|
|
68
|
+
memo?.note("stamp_missing");
|
|
69
|
+
const raw = await fs.readFile(filePath);
|
|
70
|
+
const content = raw.toString("utf8");
|
|
71
|
+
const hash = sha256(raw);
|
|
72
|
+
if (content.trim() === "")
|
|
73
|
+
return { empty: true, byte_size: raw.byteLength, sha256: hash };
|
|
74
|
+
let signals;
|
|
75
|
+
const facts = captureSessionFactsDelta("claude_code", (acc) => {
|
|
76
|
+
signals = extractClaudeSessionSignals(content, acc);
|
|
77
|
+
observeSessionFactsFile(acc, { bytes: raw.byteLength, lines: signals.line_count });
|
|
78
|
+
});
|
|
79
|
+
const read = { signals: signals, byte_size: raw.byteLength, sha256: hash, facts };
|
|
80
|
+
if (memo && stamp) {
|
|
81
|
+
memo.note("fresh");
|
|
82
|
+
memo.remember(filePath, stamp, ATTRIBUTION_READ_RECIPE, read);
|
|
83
|
+
}
|
|
84
|
+
return read;
|
|
85
|
+
}
|
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
import { finalizeSessionFacts,
|
|
2
|
-
import crypto from "node:crypto";
|
|
1
|
+
import { finalizeSessionFacts, mergeSessionFactsDelta, newSessionFactsAccumulator, observeSessionFactsFile, } from "@bli-cockpit/telemetry-core";
|
|
3
2
|
import { existsSync } from "node:fs";
|
|
4
|
-
import fs from "node:fs/promises";
|
|
5
3
|
import path from "node:path";
|
|
6
4
|
import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
|
|
7
5
|
import { describeError } from "../health-detail.js";
|
|
8
6
|
import { isPathWithin, sanitizeSessionId, scoreSignalsAgainstWorktrees, sessionIdFromFileName, shortHash, } from "./attribution-core.js";
|
|
9
|
-
import {
|
|
7
|
+
import { isSchemaDriftSuspected, streamMainSignals } from "./claude-attribution-signals.js";
|
|
8
|
+
import { readMain, readSidecar } from "./claude-attribution-memoized-reads.js";
|
|
10
9
|
import { CLAUDE_SESSION_MAX_FILE_BYTES, } from "./claude-attribution-types.js";
|
|
11
10
|
/**
|
|
12
11
|
* Turns one discovered session into an attribution verdict: read (or, for an
|
|
@@ -18,7 +17,7 @@ import { CLAUDE_SESSION_MAX_FILE_BYTES, } from "./claude-attribution-types.js";
|
|
|
18
17
|
* are counted: a sidecar rejected for a cwd mismatch is another worktree's
|
|
19
18
|
* work and must not inflate this session's numbers.
|
|
20
19
|
*/
|
|
21
|
-
export async function attributeOneSession(session, worktrees, collectionRoots) {
|
|
20
|
+
export async function attributeOneSession(session, worktrees, collectionRoots, readMemo) {
|
|
22
21
|
const factsAcc = newSessionFactsAccumulator("claude_code");
|
|
23
22
|
const base = createSessionBase(session);
|
|
24
23
|
if (session.mainByteSize === 0) {
|
|
@@ -26,7 +25,7 @@ export async function attributeOneSession(session, worktrees, collectionRoots) {
|
|
|
26
25
|
}
|
|
27
26
|
const read = session.mainByteSize > CLAUDE_SESSION_MAX_FILE_BYTES
|
|
28
27
|
? await streamOversizedMain(session, base, factsAcc)
|
|
29
|
-
: await readMainSignals(session, base, factsAcc);
|
|
28
|
+
: await readMainSignals(session, base, factsAcc, readMemo);
|
|
30
29
|
if (typeof read === "string")
|
|
31
30
|
return skippedResult(base, read);
|
|
32
31
|
const signals = read;
|
|
@@ -54,7 +53,7 @@ export async function attributeOneSession(session, worktrees, collectionRoots) {
|
|
|
54
53
|
extraSignals.push("main_file_oversized");
|
|
55
54
|
if (isSchemaDriftSuspected(signals))
|
|
56
55
|
extraSignals.push("schema_drift_suspected");
|
|
57
|
-
const sidecarFiles = await collectSessionSidecars(session, outcome, factsAcc);
|
|
56
|
+
const sidecarFiles = await collectSessionSidecars(session, outcome, factsAcc, readMemo);
|
|
58
57
|
return {
|
|
59
58
|
...base,
|
|
60
59
|
session_facts: finalizeSessionFacts(factsAcc),
|
|
@@ -116,10 +115,10 @@ async function streamOversizedMain(session, base, factsAcc) {
|
|
|
116
115
|
base.oversized_lines_skipped = streamed.oversizedLinesSkipped;
|
|
117
116
|
return streamed.signals;
|
|
118
117
|
}
|
|
119
|
-
async function readMainSignals(session, base, factsAcc) {
|
|
120
|
-
let
|
|
118
|
+
async function readMainSignals(session, base, factsAcc, readMemo) {
|
|
119
|
+
let read;
|
|
121
120
|
try {
|
|
122
|
-
|
|
121
|
+
read = await readMain(readMemo, session.mainFile);
|
|
123
122
|
}
|
|
124
123
|
catch (error) {
|
|
125
124
|
console.error("[claude-attribution] session file unreadable, skipping it", JSON.stringify({
|
|
@@ -130,18 +129,12 @@ async function readMainSignals(session, base, factsAcc) {
|
|
|
130
129
|
}));
|
|
131
130
|
return "file_read_failed";
|
|
132
131
|
}
|
|
133
|
-
|
|
134
|
-
base.
|
|
135
|
-
|
|
136
|
-
if (content.trim() === "") {
|
|
132
|
+
base.content_hash_sha256 = read.sha256;
|
|
133
|
+
base.byte_size = read.byte_size;
|
|
134
|
+
if ("empty" in read)
|
|
137
135
|
return "empty_file";
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
observeSessionFactsFile(factsAcc, {
|
|
141
|
-
bytes: raw.byteLength,
|
|
142
|
-
lines: signals.line_count,
|
|
143
|
-
});
|
|
144
|
-
return signals;
|
|
136
|
+
mergeSessionFactsDelta(factsAcc, read.facts);
|
|
137
|
+
return read.signals;
|
|
145
138
|
}
|
|
146
139
|
function applySessionMetadata(base, signals) {
|
|
147
140
|
const metaSessionId = sanitizeSessionId(signals.session_ids[0]);
|
|
@@ -155,9 +148,9 @@ function applySessionMetadata(base, signals) {
|
|
|
155
148
|
base.cwd_hash = shortHash(primaryCwd);
|
|
156
149
|
}
|
|
157
150
|
}
|
|
158
|
-
async function collectSessionSidecars(session, outcome, factsAcc) {
|
|
151
|
+
async function collectSessionSidecars(session, outcome, factsAcc, readMemo) {
|
|
159
152
|
return isRawEvidenceUploadableAttributionState(outcome.state, outcome.worktree !== null) && outcome.worktree
|
|
160
|
-
? await collectSidecarDiagnostics(session.sidecars, outcome.worktree, factsAcc)
|
|
153
|
+
? await collectSidecarDiagnostics(session.sidecars, outcome.worktree, factsAcc, readMemo)
|
|
161
154
|
: session.sidecars.map((sidecar) => ({
|
|
162
155
|
local_path: sidecar.local_path,
|
|
163
156
|
file_name: sidecar.file_name,
|
|
@@ -174,7 +167,7 @@ async function collectSessionSidecars(session, outcome, factsAcc) {
|
|
|
174
167
|
* oversized sidecar is recorded with a reason and the session stays
|
|
175
168
|
* harvestable.
|
|
176
169
|
*/
|
|
177
|
-
async function collectSidecarDiagnostics(sidecars, worktree, facts) {
|
|
170
|
+
async function collectSidecarDiagnostics(sidecars, worktree, facts, readMemo) {
|
|
178
171
|
const out = [];
|
|
179
172
|
for (const sidecar of sidecars) {
|
|
180
173
|
const entry = {
|
|
@@ -188,9 +181,9 @@ async function collectSidecarDiagnostics(sidecars, worktree, facts) {
|
|
|
188
181
|
out.push({ ...entry, skipped_reason: "file_too_large" });
|
|
189
182
|
continue;
|
|
190
183
|
}
|
|
191
|
-
let
|
|
184
|
+
let read;
|
|
192
185
|
try {
|
|
193
|
-
|
|
186
|
+
read = await readSidecar(readMemo, sidecar.local_path);
|
|
194
187
|
}
|
|
195
188
|
catch (error) {
|
|
196
189
|
// `file_read_failed` stays on the sidecar row. Beside it: a subagent
|
|
@@ -205,8 +198,7 @@ async function collectSidecarDiagnostics(sidecars, worktree, facts) {
|
|
|
205
198
|
out.push({ ...entry, skipped_reason: "file_read_failed" });
|
|
206
199
|
continue;
|
|
207
200
|
}
|
|
208
|
-
const
|
|
209
|
-
const sidecarCwds = extractClaudeSessionSignals(content).cwds;
|
|
201
|
+
const sidecarCwds = read.cwds;
|
|
210
202
|
if (sidecarCwds.length > 0 &&
|
|
211
203
|
!sidecarCwds.some((cwd) => isPathWithin(cwd, worktree.repo_root))) {
|
|
212
204
|
out.push({ ...entry, skipped_reason: "sidecar_cwd_mismatch" });
|
|
@@ -214,11 +206,11 @@ async function collectSidecarDiagnostics(sidecars, worktree, facts) {
|
|
|
214
206
|
}
|
|
215
207
|
// Accepted: this subagent's turns are part of this session, so its tokens
|
|
216
208
|
// are too. Counted only now, after the cwd check has cleared it.
|
|
217
|
-
|
|
209
|
+
mergeSessionFactsDelta(facts, read.facts);
|
|
218
210
|
out.push({
|
|
219
211
|
...entry,
|
|
220
|
-
byte_size:
|
|
221
|
-
content_hash_sha256: sha256
|
|
212
|
+
byte_size: read.byte_size,
|
|
213
|
+
content_hash_sha256: read.sha256,
|
|
222
214
|
});
|
|
223
215
|
}
|
|
224
216
|
return out;
|
|
@@ -233,7 +225,4 @@ function skippedResult(base, reason) {
|
|
|
233
225
|
path_score: 0,
|
|
234
226
|
worktree: null,
|
|
235
227
|
};
|
|
236
|
-
}
|
|
237
|
-
function sha256(value) {
|
|
238
|
-
return crypto.createHash("sha256").update(value).digest("hex");
|
|
239
228
|
}
|
|
@@ -37,7 +37,7 @@ export async function scanAndAttributeClaudeSessions(options) {
|
|
|
37
37
|
.slice(0, limit);
|
|
38
38
|
const results = [];
|
|
39
39
|
for (const session of sessions) {
|
|
40
|
-
results.push(await attributeOneSession(session, options.worktrees, options.collectionRoots ?? []));
|
|
40
|
+
results.push(await attributeOneSession(session, options.worktrees, options.collectionRoots ?? [], options.readMemo));
|
|
41
41
|
}
|
|
42
42
|
const sumBy = (pick) => results.reduce((total, result) => total + pick(result), 0);
|
|
43
43
|
const countState = (state) => results.filter((result) => result.state === state).length;
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
* collection is done.
|
|
25
25
|
*/
|
|
26
26
|
import { openPrepareMemo } from "./raw-evidence-prepare-memo.js";
|
|
27
|
+
import { openTranscriptReadMemo } from "./transcript-read-memo.js";
|
|
27
28
|
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
28
29
|
import crypto from "node:crypto";
|
|
29
30
|
import fs from "node:fs/promises";
|
|
@@ -65,6 +66,7 @@ export async function openCollection(context, options, places) {
|
|
|
65
66
|
budget.images ??= options.images;
|
|
66
67
|
const stagingAllowance = (budget.staging ??= await openStagingAllowance(options.stateDir, options.env ?? process.env, { statfs: options.statfs }));
|
|
67
68
|
budget.prepareMemo ??= await openPrepareMemo(options.stateDir);
|
|
69
|
+
budget.readMemo ??= await openTranscriptReadMemo(options.stateDir, "image_scan").catch(() => undefined);
|
|
68
70
|
return {
|
|
69
71
|
context,
|
|
70
72
|
filesDir: path.join(places.stagingDir, "files"),
|
|
@@ -9,18 +9,42 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import path from "node:path";
|
|
11
11
|
import { collectAgentImageEvidenceFromJsonlFile, } from "./agent-image-evidence.js";
|
|
12
|
+
import { statSourceForMemo } from "./transcript-read-memo.js";
|
|
13
|
+
import { AGENT_IMAGE_MAX_BYTES } from "./agent-image-validation.js";
|
|
12
14
|
import { evidenceSourceKey } from "../raw-evidence-staging.js";
|
|
13
15
|
import { sha256 } from "./raw-evidence-keys.js";
|
|
14
16
|
import { markBudgetCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
|
|
15
17
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
16
18
|
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
19
|
+
/**
|
|
20
|
+
* BLI-22797: a transcript with no image candidate is remembered by its stamp, so the next tick does not open it.
|
|
21
|
+
* Only that answer is remembered: an image whose bytes live in another file can change while the transcript does
|
|
22
|
+
* not, so a transcript that offered images is always read again. Bump the version when
|
|
23
|
+
* `agent-image-records.ts` changes what counts as a candidate.
|
|
24
|
+
*/
|
|
25
|
+
const IMAGE_SCAN_RECIPE = ["image-scan.v1", String(AGENT_IMAGE_MAX_BYTES)].join("|");
|
|
17
26
|
export async function collectAgentImagesFromTranscript(collection, options) {
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
27
|
+
const memo = collection.budget.readMemo;
|
|
28
|
+
const stamp = memo ? await statSourceForMemo(options.filePath) : null;
|
|
29
|
+
const remembered = memo && stamp
|
|
30
|
+
? memo.recall(options.filePath, stamp, IMAGE_SCAN_RECIPE, stamp.size, (data) => (data === "no_image_candidates" ? true : null))
|
|
31
|
+
: null;
|
|
32
|
+
if (!remembered && memo && !stamp)
|
|
33
|
+
memo.note("stamp_missing");
|
|
34
|
+
const result = remembered
|
|
35
|
+
? { images: [], skipped: [] }
|
|
36
|
+
: await collectAgentImageEvidenceFromJsonlFile({
|
|
37
|
+
filePath: options.filePath,
|
|
38
|
+
source: options.source,
|
|
39
|
+
sessionId: options.sessionId,
|
|
40
|
+
sidecarId: options.sidecarId,
|
|
41
|
+
});
|
|
42
|
+
if (!remembered && memo && stamp) {
|
|
43
|
+
memo.note("fresh");
|
|
44
|
+
if (result.images.length === 0 && result.skipped.length === 0) {
|
|
45
|
+
memo.remember(options.filePath, stamp, IMAGE_SCAN_RECIPE, "no_image_candidates");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
24
48
|
recordScanned(collection, options.kind, result.images.length + result.skipped.length);
|
|
25
49
|
for (const skipped of result.skipped) {
|
|
26
50
|
collection.skipped.push({
|
|
@@ -13,11 +13,14 @@
|
|
|
13
13
|
*
|
|
14
14
|
* - the file's size, inode and mtime/ctime (nanoseconds) are exactly what they were when the
|
|
15
15
|
* entry was written, so the file has not been appended to, rewritten or replaced;
|
|
16
|
-
* - the recipe is the same:
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
16
|
+
* - the recipe is the same: a fingerprint of the rules that decide the bytes (the slim rules, the
|
|
17
|
+
* secret rules, the image marker format; NOT the CLI version, BLI-22794), the machine's
|
|
18
|
+
* image-handling mode, and whether an image step ran at all;
|
|
19
|
+
* - the entry is not `volatile`, or it is volatile ONLY because of images with no stored
|
|
20
|
+
* description (`pending_images`, BLI-22794). Such an entry is handed back to the caller, who
|
|
21
|
+
* trusts it while none of those images has been described since; a described one rebuilds. A
|
|
22
|
+
* volatile entry without `pending_images` (a failed slim step, or any older entry) is never
|
|
23
|
+
* recalled: it can change shape on the next tick without the file changing.
|
|
21
24
|
*
|
|
22
25
|
* The caller then reuses the entry ONLY when the remembered content hash is already durable
|
|
23
26
|
* remotely (`skipContentHashes`). Anything else, including every file that is not yet delivered,
|
|
@@ -25,19 +28,21 @@
|
|
|
25
28
|
* known. A memo that is lost, corrupt, or wrong in the safe direction costs one slow tick, never
|
|
26
29
|
* a missed upload. It holds ids, sizes, hashes and redaction metadata, never a byte of content.
|
|
27
30
|
*/
|
|
31
|
+
import { createHash } from "node:crypto";
|
|
28
32
|
import path from "node:path";
|
|
29
33
|
import fs from "node:fs/promises";
|
|
30
34
|
import { isMissingFileFailure } from "../health-detail.js";
|
|
31
35
|
import { readJsonFile, writeJsonFileAtomic } from "../local-state-files.js";
|
|
32
|
-
import {
|
|
36
|
+
import { evidenceRulesFingerprint } from "@bli-cockpit/telemetry-core";
|
|
37
|
+
import { SLIM_RULES_VERSION } from "../slim/slim-transcript.js";
|
|
33
38
|
export const PREPARE_MEMO_FILENAME = "transcript-prepare-memo.json";
|
|
34
39
|
const MEMO_FORMAT = "prepare-memo.v1";
|
|
35
40
|
/** About 700 bytes an entry, so this is at most a few MB; the oldest leave first. */
|
|
36
41
|
export const PREPARE_MEMO_MAX_ENTRIES = 20_000;
|
|
37
42
|
/** A source not seen for this long is a deleted or rotated session. */
|
|
38
|
-
const FORGET_AFTER_MS = 14 * 24 * 3_600_000;
|
|
43
|
+
export const FORGET_AFTER_MS = 14 * 24 * 3_600_000;
|
|
39
44
|
/** Refresh `seen_ms` at most this often, so an unchanged tick does not rewrite the file. */
|
|
40
|
-
const SEEN_REFRESH_MS = 24 * 3_600_000;
|
|
45
|
+
export const SEEN_REFRESH_MS = 24 * 3_600_000;
|
|
41
46
|
const TAG = "[raw-evidence-memo]";
|
|
42
47
|
export function sourceStampOf(stat) {
|
|
43
48
|
return {
|
|
@@ -54,29 +59,38 @@ export async function statSourceForMemo(filePath) {
|
|
|
54
59
|
return null;
|
|
55
60
|
}
|
|
56
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* What the prepared bytes depend on besides the file: the slim rules and the secret and image rules.
|
|
64
|
+
* Computed once per process. NOT the CLI version (BLI-22794): an upgrade that changes none of these
|
|
65
|
+
* keeps the memo warm. Changing a rule changes this, which rebuilds the memo once; that is intended
|
|
66
|
+
* (see `SLIM_RULES_VERSION` and telemetry-core's `evidenceRulesFingerprint`).
|
|
67
|
+
*/
|
|
68
|
+
let rulesFingerprint;
|
|
69
|
+
export function rulesFingerprintOf(slimRulesVersion, evidenceRules) {
|
|
70
|
+
return createHash("sha256").update(`${slimRulesVersion}|${evidenceRules}`).digest("hex").slice(0, 32);
|
|
71
|
+
}
|
|
57
72
|
/** Everything besides the file that decides what its prepared bytes are. */
|
|
58
73
|
export function prepareRecipe(options) {
|
|
59
|
-
|
|
74
|
+
rulesFingerprint ??= rulesFingerprintOf(SLIM_RULES_VERSION, evidenceRulesFingerprint());
|
|
75
|
+
return [MEMO_FORMAT, rulesFingerprint, options.kind, options.imageMode ?? "no_image_step"].join("|");
|
|
60
76
|
}
|
|
61
77
|
export function prepareMemoPath(stateDir) {
|
|
62
78
|
return path.join(stateDir, PREPARE_MEMO_FILENAME);
|
|
63
79
|
}
|
|
64
|
-
const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0, volatile_reused_count: 0, volatile_reused_bytes: 0 };
|
|
80
|
+
const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0, volatile_reused_count: 0, volatile_reused_bytes: 0, rebuilt_image_now_described_count: 0 };
|
|
65
81
|
export async function openPrepareMemo(stateDir) {
|
|
66
82
|
const entries = await readMemoEntries(stateDir);
|
|
67
83
|
let dirty = false;
|
|
68
84
|
let counts = { ...EMPTY_COUNTS };
|
|
69
85
|
/** Sources recalled since the last `settle`: only these have their `seen_ms` refreshed. */
|
|
70
86
|
const recalled = new Set();
|
|
71
|
-
/** In-memory only, never persisted: see `rememberForThisRun`. */
|
|
72
|
-
const thisRun = new Map();
|
|
73
87
|
return {
|
|
74
88
|
recall(sourceKey, stamp, recipe) {
|
|
75
89
|
const entry = entries.get(sourceKey);
|
|
76
90
|
if (!entry)
|
|
77
|
-
return
|
|
78
|
-
if (entry.volatile)
|
|
79
|
-
return
|
|
91
|
+
return null;
|
|
92
|
+
if (entry.volatile && !entry.pending_images)
|
|
93
|
+
return null;
|
|
80
94
|
if (entry.recipe !== recipe)
|
|
81
95
|
return null;
|
|
82
96
|
if (entry.size !== stamp.size || entry.identity !== stamp.identity)
|
|
@@ -86,13 +100,13 @@ export async function openPrepareMemo(stateDir) {
|
|
|
86
100
|
},
|
|
87
101
|
remember(sourceKey, entry, now = Date.now()) {
|
|
88
102
|
entries.set(sourceKey, { ...entry, seen_ms: now });
|
|
89
|
-
thisRun.delete(sourceKey);
|
|
90
103
|
dirty = true;
|
|
91
104
|
},
|
|
92
|
-
rememberForThisRun(sourceKey, entry) {
|
|
93
|
-
thisRun.set(sourceKey, { ...entry, volatile: true, seen_ms: Date.now() });
|
|
94
|
-
},
|
|
95
105
|
count(outcome, bytes) {
|
|
106
|
+
if (outcome === "rebuilt_image_now_described") {
|
|
107
|
+
counts.rebuilt_image_now_described_count += 1;
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
96
110
|
if (outcome === "volatile_reused") {
|
|
97
111
|
counts.volatile_reused_count += 1;
|
|
98
112
|
counts.volatile_reused_bytes += bytes;
|
|
@@ -127,20 +141,12 @@ export async function openPrepareMemo(stateDir) {
|
|
|
127
141
|
},
|
|
128
142
|
};
|
|
129
143
|
}
|
|
130
|
-
function recallThisRun(thisRun, sourceKey, stamp, recipe) {
|
|
131
|
-
const entry = thisRun.get(sourceKey);
|
|
132
|
-
if (!entry || entry.recipe !== recipe)
|
|
133
|
-
return null;
|
|
134
|
-
if (entry.size !== stamp.size || entry.identity !== stamp.identity)
|
|
135
|
-
return null;
|
|
136
|
-
return entry;
|
|
137
|
-
}
|
|
138
144
|
/**
|
|
139
145
|
* Entries recalled this pass keep their `seen_ms` fresh, lazily, so they are not forgotten while in
|
|
140
146
|
* use. Only those: an entry nobody recalls (a deleted or rotated session) ages out after
|
|
141
147
|
* FORGET_AFTER_MS.
|
|
142
148
|
*/
|
|
143
|
-
function refreshSeen(entries, recalled, now) {
|
|
149
|
+
export function refreshSeen(entries, recalled, now) {
|
|
144
150
|
let touched = false;
|
|
145
151
|
for (const key of recalled) {
|
|
146
152
|
const entry = entries.get(key);
|
|
@@ -151,13 +157,13 @@ function refreshSeen(entries, recalled, now) {
|
|
|
151
157
|
}
|
|
152
158
|
return touched;
|
|
153
159
|
}
|
|
154
|
-
function trimEntries(entries, now) {
|
|
160
|
+
export function trimEntries(entries, now, maxEntries = PREPARE_MEMO_MAX_ENTRIES) {
|
|
155
161
|
for (const [key, entry] of entries) {
|
|
156
162
|
if (now - entry.seen_ms > FORGET_AFTER_MS)
|
|
157
163
|
entries.delete(key);
|
|
158
164
|
}
|
|
159
165
|
const newestFirst = [...entries.entries()].sort((a, b) => b[1].seen_ms - a[1].seen_ms);
|
|
160
|
-
const kept = newestFirst.slice(0,
|
|
166
|
+
const kept = newestFirst.slice(0, maxEntries);
|
|
161
167
|
if (kept.length < entries.size) {
|
|
162
168
|
entries.clear();
|
|
163
169
|
for (const [key, entry] of kept)
|
|
@@ -205,6 +211,9 @@ function parseEntry(value) {
|
|
|
205
211
|
return null;
|
|
206
212
|
if (typeof entry["volatile"] !== "boolean" || typeof entry["seen_ms"] !== "number")
|
|
207
213
|
return null;
|
|
214
|
+
const pending = entry["pending_images"];
|
|
215
|
+
if (pending !== undefined && (!Array.isArray(pending) || !pending.every((item) => typeof item === "string")))
|
|
216
|
+
return null;
|
|
208
217
|
const redacted = entry["redacted"];
|
|
209
218
|
if (redacted !== null && (typeof redacted !== "object" || typeof redacted.completenessLabel !== "string"))
|
|
210
219
|
return null;
|
|
@@ -26,6 +26,7 @@ import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
|
|
|
26
26
|
import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
|
|
27
27
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
28
28
|
import { prepareRecipe, statSourceForMemo, } from "./raw-evidence-prepare-memo.js";
|
|
29
|
+
import { logVolatileReuse, trustVolatileEntry } from "./raw-evidence-volatile-reuse.js";
|
|
29
30
|
import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
|
|
30
31
|
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
31
32
|
/**
|
|
@@ -63,9 +64,9 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
63
64
|
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
64
65
|
return skip("file_too_large");
|
|
65
66
|
}
|
|
66
|
-
const { sanitized, contentHash, stubClaim, volatile,
|
|
67
|
+
const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
|
|
67
68
|
await rememberPreparedTranscript(collection, options, names, {
|
|
68
|
-
stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile,
|
|
69
|
+
stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
|
|
69
70
|
});
|
|
70
71
|
if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
|
|
71
72
|
collection.reused.push({
|
|
@@ -121,7 +122,7 @@ async function prepareTranscriptForUpload(collection, options, names, bytes) {
|
|
|
121
122
|
byteSize: sanitized.bytes.byteLength,
|
|
122
123
|
alreadyDurable: collection.skipContentHashes.has(contentHash),
|
|
123
124
|
});
|
|
124
|
-
return { sanitized, contentHash, stubClaim, volatile: images.volatile,
|
|
125
|
+
return { sanitized, contentHash, stubClaim, volatile: images.volatile, pendingImages: images.pendingImages, redactedEntry };
|
|
125
126
|
}
|
|
126
127
|
/**
|
|
127
128
|
* The transcript bytes after the image step. Unchanged (the very same buffer) unless this sync has
|
|
@@ -133,10 +134,10 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
133
134
|
const session = collection.budget.images;
|
|
134
135
|
const source = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
|
|
135
136
|
if (!session || !source)
|
|
136
|
-
return { bytes: file.bytes, volatile: false,
|
|
137
|
+
return { bytes: file.bytes, volatile: false, pendingImages: null };
|
|
137
138
|
// Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
|
|
138
139
|
if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.bytes))
|
|
139
|
-
return { bytes: file.bytes, volatile: false,
|
|
140
|
+
return { bytes: file.bytes, volatile: false, pendingImages: null };
|
|
140
141
|
try {
|
|
141
142
|
const prepared = await session.prepareTranscript({
|
|
142
143
|
text: file.bytes.toString("utf8"),
|
|
@@ -151,15 +152,16 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
151
152
|
: prepared.imageCount > 0;
|
|
152
153
|
const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
|
|
153
154
|
const volatile = imagesPending || slimFailed;
|
|
154
|
-
// BLI-
|
|
155
|
-
|
|
155
|
+
// BLI-22794: volatile ONLY because of images without a stored description (the slim step held
|
|
156
|
+
// up): the bytes change only when one of these gains one, so the memo may keep the entry.
|
|
157
|
+
const pendingImages = volatile && !slimFailed ? prepared.pendingImages : null;
|
|
156
158
|
if (!prepared.changed)
|
|
157
|
-
return { bytes: file.bytes, volatile,
|
|
158
|
-
return { bytes: Buffer.from(prepared.text, "utf8"), volatile,
|
|
159
|
+
return { bytes: file.bytes, volatile, pendingImages };
|
|
160
|
+
return { bytes: Buffer.from(prepared.text, "utf8"), volatile, pendingImages };
|
|
159
161
|
}
|
|
160
162
|
catch (error) {
|
|
161
163
|
console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
|
|
162
|
-
return { bytes: file.bytes, volatile: true,
|
|
164
|
+
return { bytes: file.bytes, volatile: true, pendingImages: null };
|
|
163
165
|
}
|
|
164
166
|
}
|
|
165
167
|
/**
|
|
@@ -182,6 +184,8 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
|
|
|
182
184
|
const known = memo.recall(names.sourceKey, stamp, recipe);
|
|
183
185
|
if (!known)
|
|
184
186
|
return false;
|
|
187
|
+
if (known.volatile && !trustVolatileEntry(collection, memo, known, options.kind, names.sourceKey, stamp.size))
|
|
188
|
+
return false;
|
|
185
189
|
if (!collection.skipContentHashes.has(known.content_hash)) {
|
|
186
190
|
if (known.volatile)
|
|
187
191
|
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "hash_not_durable_full_path");
|
|
@@ -202,24 +206,21 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
|
|
|
202
206
|
codex_session_id: options.sessionId,
|
|
203
207
|
});
|
|
204
208
|
if (known.volatile) {
|
|
205
|
-
// BLI-
|
|
209
|
+
// BLI-22794: only images without a description held it back, and none has one now.
|
|
206
210
|
memo.count("volatile_reused", stamp.size);
|
|
207
|
-
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "
|
|
211
|
+
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "pending_images_still_undescribed");
|
|
208
212
|
return true;
|
|
209
213
|
}
|
|
210
214
|
memo.count("skipped", stamp.size);
|
|
211
215
|
return true;
|
|
212
216
|
}
|
|
213
|
-
function logVolatileReuse(kind, sourceKey, contentHash, reason) {
|
|
214
|
-
console.error("[raw-evidence] volatile transcript " + (reason === "hash_not_durable_full_path" ? "rebuilt" : "reused this run"), JSON.stringify({ reason, kind, source_key: sourceKey, hash_prefix: contentHash.slice(0, 12) }));
|
|
215
|
-
}
|
|
216
217
|
/** Remember what this transcript became, but only if the file stayed put while it was built. */
|
|
217
218
|
async function rememberPreparedTranscript(collection, options, names, built) {
|
|
218
219
|
const memo = collection.budget.prepareMemo;
|
|
219
220
|
if (!memo)
|
|
220
221
|
return;
|
|
221
222
|
memo.count(built.volatile ? "volatile" : "rebuilt", built.readBytes);
|
|
222
|
-
if ((built.volatile && !built.
|
|
223
|
+
if ((built.volatile && !built.pendingImages) || !built.stamp)
|
|
223
224
|
return;
|
|
224
225
|
const after = await statSourceForMemo(options.filePath);
|
|
225
226
|
const stable = after !== null
|
|
@@ -237,15 +238,13 @@ async function rememberPreparedTranscript(collection, options, names, built) {
|
|
|
237
238
|
? { redaction: built.redactedEntry.redaction, completenessLabel: built.redactedEntry.completenessLabel }
|
|
238
239
|
: null,
|
|
239
240
|
};
|
|
240
|
-
if (built.volatile) {
|
|
241
|
-
// BLI-22785: in memory for this process only; the memo file keeps no volatile entry.
|
|
242
|
-
memo.rememberForThisRun(names.sourceKey, entry);
|
|
243
|
-
return;
|
|
244
|
-
}
|
|
245
241
|
const known = memo.recall(names.sourceKey, built.stamp, built.recipe);
|
|
246
|
-
|
|
242
|
+
const pending = built.volatile ? built.pendingImages : undefined;
|
|
243
|
+
if (known && known.volatile === built.volatile && known.content_hash === built.contentHash
|
|
244
|
+
&& (known.pending_images ?? []).join() === (pending ?? []).join())
|
|
247
245
|
return;
|
|
248
|
-
|
|
246
|
+
// BLI-22794: a volatile entry is kept across ticks with the images it is waiting on.
|
|
247
|
+
memo.remember(names.sourceKey, { ...entry, volatile: built.volatile, ...(pending ? { pending_images: pending } : {}) });
|
|
249
248
|
}
|
|
250
249
|
function nameEvidenceFile(options) {
|
|
251
250
|
const fileName = path.basename(options.filePath);
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLI-22794: may a remembered image-pending entry stand in for a rebuild? Not when one of its
|
|
3
|
+
* images has a stored description now (the bytes would differ, counted as
|
|
4
|
+
* `rebuilt_image_now_described_count`), and not while the describer could still take one of them
|
|
5
|
+
* this tick: a skipped transcript never reaches the describer, so reusing it would starve the
|
|
6
|
+
* images. Once the tick's budget is spent, the rest are reused until the next tick.
|
|
7
|
+
*/
|
|
8
|
+
export function trustVolatileEntry(collection, memo, known, kind, sourceKey, bytes) {
|
|
9
|
+
const session = collection.budget.images;
|
|
10
|
+
const pending = known.pending_images ?? [];
|
|
11
|
+
if (!session)
|
|
12
|
+
return false;
|
|
13
|
+
if (pending.some((sha256) => session.isDescribed(sha256))) {
|
|
14
|
+
memo.count("rebuilt_image_now_described", bytes);
|
|
15
|
+
logVolatileReuse(kind, sourceKey, known.content_hash, "rebuilt_image_now_described");
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
if (session.hasDescribeRoom(pending)) {
|
|
19
|
+
logVolatileReuse(kind, sourceKey, known.content_hash, "rebuilt_describer_has_room");
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
export function logVolatileReuse(kind, sourceKey, contentHash, reason) {
|
|
25
|
+
const reused = reason === "pending_images_still_undescribed";
|
|
26
|
+
console.error("[raw-evidence] volatile transcript " + (reused ? "reused across ticks" : "rebuilt"), JSON.stringify({ reason, kind, source_key: sourceKey, hash_prefix: contentHash.slice(0, 12) }));
|
|
27
|
+
}
|
|
@@ -106,6 +106,7 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
106
106
|
* made it possible. Counts and bytes only. Never throws: the memo is a cache.
|
|
107
107
|
*/
|
|
108
108
|
async function reportPrepareMemo(collection) {
|
|
109
|
+
await collection.budget.readMemo?.settle();
|
|
109
110
|
const counts = await collection.budget.prepareMemo?.settle();
|
|
110
111
|
if (!counts || counts.skipped_count + counts.rebuilt_count + counts.volatile_reused_count === 0)
|
|
111
112
|
return;
|
|
@@ -118,6 +119,7 @@ async function reportPrepareMemo(collection) {
|
|
|
118
119
|
rebuilt_volatile_count: counts.volatile_count,
|
|
119
120
|
rebuilt_volatile_reused_count: counts.volatile_reused_count,
|
|
120
121
|
rebuilt_volatile_reused_bytes: counts.volatile_reused_bytes,
|
|
122
|
+
rebuilt_image_now_described_count: counts.rebuilt_image_now_described_count,
|
|
121
123
|
}));
|
|
122
124
|
}
|
|
123
125
|
/**
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Remember what a transcript file said, so an unchanged file is not opened again (BLI-22797).
|
|
3
|
+
*
|
|
4
|
+
* A sync tick with nothing new still re-read every attributed session, every subagent sidecar and every
|
|
5
|
+
* transcript the image step scans: on the reference Mac 33 of the 37 seconds of main-thread JavaScript went to
|
|
6
|
+
* reading, parsing and hashing files whose size and mtime had not moved since the tick before. This is the
|
|
7
|
+
* sibling of the prepare memo (`raw-evidence-prepare-memo.ts`) and uses ITS stamp: size plus inode, mtime and
|
|
8
|
+
* ctime in nanoseconds, so an append, rewrite or replace is never mistaken for "unchanged". The entry cap, the
|
|
9
|
+
* `seen_ms` ageing and the atomic write are the prepare memo's own helpers.
|
|
10
|
+
*
|
|
11
|
+
* An entry is trusted only when the stamp AND the recipe match. The recipe names the rules that produced the
|
|
12
|
+
* remembered answer, so changing a rule rebuilds the memo once. What an entry holds is decided by its caller
|
|
13
|
+
* (`data`); this file never sees a byte of transcript content, and neither does the file on disk: signals,
|
|
14
|
+
* hashes, sizes and counts only. A memo that is lost, corrupt or stale in the safe direction costs one slow
|
|
15
|
+
* tick, never a wrong answer. The file path is hashed, so no path reaches the memo either.
|
|
16
|
+
*/
|
|
17
|
+
import { createHash } from "node:crypto";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { isMissingFileFailure } from "../health-detail.js";
|
|
20
|
+
import { readJsonFile, writeJsonFileAtomic } from "../local-state-files.js";
|
|
21
|
+
import { PREPARE_MEMO_MAX_ENTRIES, refreshSeen, trimEntries, } from "./raw-evidence-prepare-memo.js";
|
|
22
|
+
export { sourceStampOf, statSourceForMemo } from "./raw-evidence-prepare-memo.js";
|
|
23
|
+
const MEMO_FORMAT = "transcript-read-memo.v1";
|
|
24
|
+
const TAG = "[transcript-reads]";
|
|
25
|
+
export function transcriptReadMemoPath(stateDir, name) {
|
|
26
|
+
return path.join(stateDir, `transcript-read-memo-${name}.json`);
|
|
27
|
+
}
|
|
28
|
+
const keyOf = (filePath) => createHash("sha256").update(filePath).digest("hex").slice(0, 32);
|
|
29
|
+
export async function openTranscriptReadMemo(stateDir, name) {
|
|
30
|
+
const entries = await readEntries(stateDir, name);
|
|
31
|
+
let dirty = false;
|
|
32
|
+
let counts = { reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0 };
|
|
33
|
+
const recalled = new Set();
|
|
34
|
+
return {
|
|
35
|
+
recall(filePath, stamp, recipe, bytes, parse) {
|
|
36
|
+
const key = keyOf(filePath);
|
|
37
|
+
const entry = entries.get(key);
|
|
38
|
+
if (!entry || entry.recipe !== recipe || entry.size !== stamp.size || entry.identity !== stamp.identity)
|
|
39
|
+
return null;
|
|
40
|
+
const data = parse(entry.data);
|
|
41
|
+
if (data === null)
|
|
42
|
+
return null;
|
|
43
|
+
recalled.add(key);
|
|
44
|
+
counts.reused += 1;
|
|
45
|
+
counts.bytes_not_read += bytes;
|
|
46
|
+
return data;
|
|
47
|
+
},
|
|
48
|
+
remember(filePath, stamp, recipe, data, now = Date.now()) {
|
|
49
|
+
entries.set(keyOf(filePath), { ...stamp, recipe, data, seen_ms: now });
|
|
50
|
+
dirty = true;
|
|
51
|
+
},
|
|
52
|
+
note(outcome) {
|
|
53
|
+
if (outcome === "fresh")
|
|
54
|
+
counts.fresh += 1;
|
|
55
|
+
else
|
|
56
|
+
counts.stamp_missing += 1;
|
|
57
|
+
},
|
|
58
|
+
async settle(now = Date.now()) {
|
|
59
|
+
const done = counts;
|
|
60
|
+
counts = { reused: 0, fresh: 0, stamp_missing: 0, bytes_not_read: 0 };
|
|
61
|
+
dirty = refreshSeen(entries, recalled, now) || dirty;
|
|
62
|
+
recalled.clear();
|
|
63
|
+
let written = false;
|
|
64
|
+
let reason = done.reused > 0 ? "stamp_unchanged" : done.fresh > 0 ? "no_reusable_entry" : "stamp_missing";
|
|
65
|
+
if (dirty) {
|
|
66
|
+
const before = entries.size;
|
|
67
|
+
const kept = trimEntries(entries, now, PREPARE_MEMO_MAX_ENTRIES);
|
|
68
|
+
try {
|
|
69
|
+
await writeJsonFileAtomic(transcriptReadMemoPath(stateDir, name), { schema_version: MEMO_FORMAT, entries: Object.fromEntries(kept) });
|
|
70
|
+
dirty = false;
|
|
71
|
+
written = true;
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
reason = "memo_write_failed";
|
|
75
|
+
console.error(TAG + " memo not written; the next tick reads these files again", JSON.stringify({ memo: name, reason, entries: before, error_name: error instanceof Error ? error.name : typeof error }));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (done.reused + done.fresh + done.stamp_missing === 0)
|
|
79
|
+
return;
|
|
80
|
+
console.error(TAG + " tick", JSON.stringify({ memo: name, reason, reused: done.reused, read_fresh: done.fresh, stamp_missing: done.stamp_missing, bytes_not_read: done.bytes_not_read, entries: entries.size, memo_written: written }));
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
async function readEntries(stateDir, name) {
|
|
85
|
+
const entries = new Map();
|
|
86
|
+
let raw;
|
|
87
|
+
try {
|
|
88
|
+
raw = await readJsonFile(transcriptReadMemoPath(stateDir, name));
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
if (!isMissingFileFailure(error))
|
|
92
|
+
console.error(TAG + " memo unreadable; every file is read once", JSON.stringify({ memo: name, reason: "read_memo_unreadable" }));
|
|
93
|
+
return entries;
|
|
94
|
+
}
|
|
95
|
+
const stored = raw;
|
|
96
|
+
if (stored?.schema_version !== MEMO_FORMAT || !stored.entries || typeof stored.entries !== "object")
|
|
97
|
+
return entries;
|
|
98
|
+
let dropped = 0;
|
|
99
|
+
for (const [key, value] of Object.entries(stored.entries)) {
|
|
100
|
+
const e = value;
|
|
101
|
+
if (e && typeof e.size === "number" && typeof e.identity === "string" && typeof e.recipe === "string" && typeof e.seen_ms === "number" && e.data !== undefined)
|
|
102
|
+
entries.set(key, e);
|
|
103
|
+
else
|
|
104
|
+
dropped += 1;
|
|
105
|
+
}
|
|
106
|
+
if (dropped > 0)
|
|
107
|
+
console.error(TAG + " memo entries dropped as malformed", JSON.stringify({ memo: name, reason: "read_memo_entries_malformed", dropped, kept: entries.size }));
|
|
108
|
+
return entries;
|
|
109
|
+
}
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.253");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -15,6 +15,7 @@ import { CLAUDE_CURSOR_FILENAME } from "../cursors/raw-evidence-cursor.js";
|
|
|
15
15
|
import { clearSourceRetryFailure, recordSourceRetryFailure, } from "../spool/local-spool.js";
|
|
16
16
|
import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
|
|
17
17
|
import { widenForCatchUp } from "./session-sync-outside-roots.js";
|
|
18
|
+
import { openTranscriptReadMemo } from "../adapters/transcript-read-memo.js";
|
|
18
19
|
const CLAUDE_FIRST_RUN_BACKFILL_MINUTES = 14 * 24 * 60;
|
|
19
20
|
/** The Claude scan's own default window when nothing widens it: the last 24 hours. */
|
|
20
21
|
const CLAUDE_DEFAULT_WINDOW_MINUTES = 24 * 60;
|
|
@@ -51,6 +52,7 @@ export async function scanAndAttributeBothSources(options) {
|
|
|
51
52
|
retrySinceMinutes: plan.claudeRetrySinceMinutes,
|
|
52
53
|
firstRunBackfill,
|
|
53
54
|
outsideRootCatchUpPending: plan.outsideRootCatchUpPending,
|
|
55
|
+
stateDir: options.paths.state_dir,
|
|
54
56
|
});
|
|
55
57
|
if (plan.codexEnabled) {
|
|
56
58
|
await reconcileSourceScanRetry({
|
|
@@ -125,24 +127,33 @@ async function scanClaudeSessionsForSync(options) {
|
|
|
125
127
|
? CLAUDE_FIRST_RUN_BACKFILL_MINUTES
|
|
126
128
|
: undefined, options.outsideRootCatchUpPending, CLAUDE_DEFAULT_WINDOW_MINUTES);
|
|
127
129
|
const projectsDir = path.join(options.homeDir, ".claude", "projects");
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
collectionRoots: options.collectionRoots,
|
|
133
|
-
sinceMinutes,
|
|
134
|
-
});
|
|
135
|
-
if (scan.session_limit_applied) {
|
|
136
|
-
scan = await scanAndAttributeClaudeSessions({
|
|
130
|
+
// BLI-22797: one memo for both passes, settled once, so an unchanged file is never opened.
|
|
131
|
+
const readMemo = await openTranscriptReadMemo(options.stateDir, "attribution").catch(() => undefined);
|
|
132
|
+
try {
|
|
133
|
+
let scan = await scanAndAttributeClaudeSessions({
|
|
137
134
|
projectsDir,
|
|
138
135
|
worktrees: options.worktrees,
|
|
139
136
|
now: options.now,
|
|
140
137
|
collectionRoots: options.collectionRoots,
|
|
141
138
|
sinceMinutes,
|
|
142
|
-
|
|
139
|
+
readMemo,
|
|
143
140
|
});
|
|
141
|
+
if (scan.session_limit_applied) {
|
|
142
|
+
scan = await scanAndAttributeClaudeSessions({
|
|
143
|
+
projectsDir,
|
|
144
|
+
worktrees: options.worktrees,
|
|
145
|
+
now: options.now,
|
|
146
|
+
collectionRoots: options.collectionRoots,
|
|
147
|
+
sinceMinutes,
|
|
148
|
+
limit: scan.discovered_session_count,
|
|
149
|
+
readMemo,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
return scan;
|
|
153
|
+
}
|
|
154
|
+
finally {
|
|
155
|
+
await readMemo?.settle();
|
|
144
156
|
}
|
|
145
|
-
return scan;
|
|
146
157
|
}
|
|
147
158
|
export function sourceScanRetryReason(source, scan) {
|
|
148
159
|
const reasons = new Set();
|
package/dist/crash-guard.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { releaseHeldSyncLocksOnExit } from "./sync-lock.js";
|
|
1
2
|
/**
|
|
2
3
|
* A run that dies must not be able to say it worked (BLI-4110).
|
|
3
4
|
*
|
|
@@ -125,6 +126,14 @@ export function installCrashGuard(options = {}) {
|
|
|
125
126
|
}));
|
|
126
127
|
});
|
|
127
128
|
target.on("exit", () => {
|
|
129
|
+
// A lock still held at exit is an orphan, whatever the exit code: release
|
|
130
|
+
// it synchronously so the next tick is not blocked (BLI-22795).
|
|
131
|
+
try {
|
|
132
|
+
releaseHeldSyncLocksOnExit();
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
console.error("[sync-lock] exit release threw", JSON.stringify({ reason: "exit_release_threw", ...describeCrash(error) }));
|
|
136
|
+
}
|
|
128
137
|
if (commandReturned)
|
|
129
138
|
return;
|
|
130
139
|
// `EXIT_DIED_MID_RUN` has to be in this set, and leaving it out made this
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A deadline that keeps the event loop alive (BLI-22795).
|
|
3
|
+
*
|
|
4
|
+
* `AbortSignal.timeout()` is documented to use an unref'd timer. A fetch whose
|
|
5
|
+
* socket has gone (network dropped mid-upload) then has nothing ref'd behind
|
|
6
|
+
* it: the loop drains, Node exits with the await still pending, and the
|
|
7
|
+
* crash guard logs `died_mid_run` with the collection lock still held. Here
|
|
8
|
+
* the timer is ref'd, and the race rejects even when an injected or broken
|
|
9
|
+
* fetch ignores the abort, so the caller always gets a named failure.
|
|
10
|
+
*/
|
|
11
|
+
export const UPLOAD_DEADLINE_ERROR_NAME = "EvidenceUploadDeadlineError";
|
|
12
|
+
export function isUploadDeadlineError(error) {
|
|
13
|
+
return error instanceof Error && error.name === UPLOAD_DEADLINE_ERROR_NAME;
|
|
14
|
+
}
|
|
15
|
+
export async function fetchWithDeadline(fetchImpl, url, init, deadlineMs) {
|
|
16
|
+
const controller = new AbortController();
|
|
17
|
+
let timer;
|
|
18
|
+
const deadline = new Promise((_, reject) => {
|
|
19
|
+
timer = setTimeout(() => {
|
|
20
|
+
const error = new Error("upload request exceeded its deadline");
|
|
21
|
+
error.name = UPLOAD_DEADLINE_ERROR_NAME;
|
|
22
|
+
controller.abort();
|
|
23
|
+
reject(error);
|
|
24
|
+
}, deadlineMs);
|
|
25
|
+
});
|
|
26
|
+
try {
|
|
27
|
+
const pending = fetchImpl(url, { ...init, signal: controller.signal });
|
|
28
|
+
// If the deadline wins, the abandoned fetch may reject later; that must not
|
|
29
|
+
// become an unhandled rejection.
|
|
30
|
+
pending.catch(() => undefined);
|
|
31
|
+
return await Promise.race([pending, deadline]);
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
clearTimeout(timer);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { encodeEvidenceBytes } from "@bli-cockpit/telemetry-core/evidence-storage-encoding";
|
|
2
|
+
import { fetchWithDeadline, isUploadDeadlineError } from "./evidence-upload-deadline.js";
|
|
2
3
|
import { noteSyncProgress } from "./sync-lock.js";
|
|
3
4
|
/** The signed capability is the only credential sent to Storage. Never log it,
|
|
4
5
|
* its query string, provider bodies, or thrown errors that might contain it. */
|
|
@@ -13,10 +14,10 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index)
|
|
|
13
14
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
14
15
|
let status = 0;
|
|
15
16
|
try {
|
|
16
|
-
const response = await options.fetchImpl
|
|
17
|
+
const response = await fetchWithDeadline(options.fetchImpl, signedUrl, {
|
|
17
18
|
method: "PUT", headers: { "Content-Type": "application/octet-stream", "x-upsert": "false" },
|
|
18
|
-
body: new Uint8Array(encoded),
|
|
19
|
-
});
|
|
19
|
+
body: new Uint8Array(encoded), redirect: "error",
|
|
20
|
+
}, 60_000);
|
|
20
21
|
status = response.status;
|
|
21
22
|
await response.body?.cancel();
|
|
22
23
|
if (response.ok) {
|
|
@@ -28,8 +29,8 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index)
|
|
|
28
29
|
if (status < 500 && status !== 429)
|
|
29
30
|
return false;
|
|
30
31
|
}
|
|
31
|
-
catch {
|
|
32
|
-
log("signed_upload_network_failure", uploadId, index, attempt, status);
|
|
32
|
+
catch (error) {
|
|
33
|
+
log(isUploadDeadlineError(error) ? "signed_upload_deadline_exceeded" : "signed_upload_network_failure", uploadId, index, attempt, status);
|
|
33
34
|
}
|
|
34
35
|
if (attempt < attempts)
|
|
35
36
|
await (options.sleep ?? sleep)(250 * attempt);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import { describeError } from "./health-detail.js";
|
|
3
|
+
import { fetchWithDeadline, isUploadDeadlineError } from "./evidence-upload-deadline.js";
|
|
3
4
|
import { noteSyncProgress } from "./sync-lock.js";
|
|
4
5
|
/**
|
|
5
6
|
* The `send` stage: one retrying POST to an evidence-upload route, plus the
|
|
@@ -10,6 +11,7 @@ import { noteSyncProgress } from "./sync-lock.js";
|
|
|
10
11
|
*/
|
|
11
12
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
12
13
|
const RETRY_DELAY_MS = 250;
|
|
14
|
+
const REQUEST_DEADLINE_MS = 120_000;
|
|
13
15
|
export async function requestJson(options, routePath, body) {
|
|
14
16
|
const maxAttempts = options.maxAttemptsPerRequest ?? DEFAULT_MAX_ATTEMPTS;
|
|
15
17
|
const sleep = options.sleep ?? defaultSleep;
|
|
@@ -17,7 +19,7 @@ export async function requestJson(options, routePath, body) {
|
|
|
17
19
|
let lastBody = null;
|
|
18
20
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
19
21
|
try {
|
|
20
|
-
const response = await options.fetchImpl
|
|
22
|
+
const response = await fetchWithDeadline(options.fetchImpl, `${options.dashboardUrl}${routePath}`, {
|
|
21
23
|
method: "POST",
|
|
22
24
|
headers: {
|
|
23
25
|
"Authorization": `Bearer ${options.deviceToken}`,
|
|
@@ -26,7 +28,7 @@ export async function requestJson(options, routePath, body) {
|
|
|
26
28
|
? { "x-tower-evidence-direct": "staged-gzip-v1" } : {}),
|
|
27
29
|
},
|
|
28
30
|
body: JSON.stringify(body),
|
|
29
|
-
});
|
|
31
|
+
}, REQUEST_DEADLINE_MS);
|
|
30
32
|
lastStatus = response.status;
|
|
31
33
|
lastBody = await readResponseJson(response);
|
|
32
34
|
if (response.ok || (response.status < 500 && response.status !== 429)) {
|
|
@@ -42,7 +44,9 @@ export async function requestJson(options, routePath, body) {
|
|
|
42
44
|
// the request left the machine. This is the exact silence BLI-2528 sat
|
|
43
45
|
// behind for 57 days; the route is named, the body never is.
|
|
44
46
|
console.error("[evidence-upload] request failed before a status came back", JSON.stringify({
|
|
45
|
-
reason:
|
|
47
|
+
reason: isUploadDeadlineError(error)
|
|
48
|
+
? "upload_request_deadline_exceeded"
|
|
49
|
+
: "upload_request_transport_error",
|
|
46
50
|
route: routePath,
|
|
47
51
|
attempt,
|
|
48
52
|
max_attempts: maxAttempts,
|
|
@@ -13,9 +13,10 @@ import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedIma
|
|
|
13
13
|
/** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
|
|
14
14
|
export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
|
|
15
15
|
/**
|
|
16
|
-
* After this long spent DESCRIBING
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* After this long spent DESCRIBING, remaining images wait for the next tick. Only time inside the
|
|
17
|
+
* sends counts: collection reaches image-bearing transcripts minutes into a tick and keeps slimming
|
|
18
|
+
* other transcripts between sends, and a wall clock from the session open (or from the first send)
|
|
19
|
+
* spent the budget on that work, so almost nothing was described (BLI-22789: 1 of 1,874, then 10).
|
|
19
20
|
*/
|
|
20
21
|
export const IMAGE_DESCRIBE_MAX_RUN_MS = 120_000;
|
|
21
22
|
const CONCURRENCY = 3;
|
|
@@ -32,9 +33,11 @@ export async function openImageDescriptions(options) {
|
|
|
32
33
|
const maxPerRun = options.maxPerRun ?? IMAGE_DESCRIBE_MAX_PER_RUN;
|
|
33
34
|
const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
|
|
34
35
|
const sentThisRun = 0;
|
|
35
|
-
const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now,
|
|
36
|
+
const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun };
|
|
36
37
|
return {
|
|
37
38
|
mode,
|
|
39
|
+
isDescribed: (sha256) => described.has(imageStateKey(sha256)),
|
|
40
|
+
hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
|
|
38
41
|
async prepareTranscript(input) {
|
|
39
42
|
const afterImages = await describeAndReplace(run, input);
|
|
40
43
|
if (mode !== "describe_and_drop")
|
|
@@ -44,6 +47,19 @@ export async function openImageDescriptions(options) {
|
|
|
44
47
|
},
|
|
45
48
|
};
|
|
46
49
|
}
|
|
50
|
+
function describeRoomLeft(run, sha256s) {
|
|
51
|
+
if (run.sentThisRun >= run.maxPerRun)
|
|
52
|
+
return false;
|
|
53
|
+
if (run.describingMs > run.maxRunMs)
|
|
54
|
+
return false;
|
|
55
|
+
return sha256s.some((sha256) => {
|
|
56
|
+
// Already sent (or found too large) this run: it will not be sent again before the next tick.
|
|
57
|
+
if (run.settledThisRun.has(sha256))
|
|
58
|
+
return false;
|
|
59
|
+
const failure = run.failedImages[imageStateKey(sha256)];
|
|
60
|
+
return !(failure && failure.retry_after > run.now());
|
|
61
|
+
});
|
|
62
|
+
}
|
|
47
63
|
function markDescribed(run, sha256) {
|
|
48
64
|
const { described, orderedKeys } = run;
|
|
49
65
|
const key = imageStateKey(sha256);
|
|
@@ -55,7 +71,6 @@ function markDescribed(run, sha256) {
|
|
|
55
71
|
async function sendOne(run, image, text, input) {
|
|
56
72
|
const { options, fetchImpl, settledThisRun, failedImages } = run;
|
|
57
73
|
run.sentThisRun += 1;
|
|
58
|
-
run.firstSendAt ??= run.now();
|
|
59
74
|
const outcome = await postImageToDescribeDoor({ fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, sleep: options.sleep }, {
|
|
60
75
|
schema_version: "image-describe.v1",
|
|
61
76
|
sha256: image.sha256,
|
|
@@ -96,7 +111,7 @@ async function describeAndReplace(run, input) {
|
|
|
96
111
|
const { mode, described, options, orderedKeys, failedImages } = run;
|
|
97
112
|
const images = findEmbeddedImages(input.text);
|
|
98
113
|
if (images.length === 0)
|
|
99
|
-
return { text: input.text, imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
|
|
114
|
+
return { text: input.text, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
|
|
100
115
|
const unique = new Map();
|
|
101
116
|
for (const image of images)
|
|
102
117
|
if (!unique.has(image.sha256))
|
|
@@ -114,6 +129,7 @@ async function describeAndReplace(run, input) {
|
|
|
114
129
|
const replaced = mode === "describe_and_drop"
|
|
115
130
|
? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
|
|
116
131
|
: { text: input.text, replacedCount: 0, charsSaved: 0 };
|
|
132
|
+
const pendingImages = [...unique.keys()].filter((sha256) => !described.has(imageStateKey(sha256)));
|
|
117
133
|
console.error(TAG + " transcript prepared", JSON.stringify({
|
|
118
134
|
mode,
|
|
119
135
|
source: input.source,
|
|
@@ -128,6 +144,7 @@ async function describeAndReplace(run, input) {
|
|
|
128
144
|
}));
|
|
129
145
|
return {
|
|
130
146
|
text: replaced.text,
|
|
147
|
+
pendingImages,
|
|
131
148
|
imageCount: images.length,
|
|
132
149
|
uniqueCount: unique.size,
|
|
133
150
|
replacedCount: replaced.replacedCount,
|
|
@@ -164,11 +181,12 @@ function selectImagesToDescribe(run, unique, counts) {
|
|
|
164
181
|
}
|
|
165
182
|
async function sendPendingImages(run, toSend, counts, input) {
|
|
166
183
|
const { now, maxPerRun, maxRunMs } = run;
|
|
184
|
+
const callStartedAt = now();
|
|
167
185
|
let next = 0;
|
|
168
186
|
const worker = async () => {
|
|
169
187
|
while (next < toSend.length) {
|
|
170
188
|
const image = toSend[next++];
|
|
171
|
-
const timeUp = run.
|
|
189
|
+
const timeUp = run.describingMs + (now() - callStartedAt) > maxRunMs;
|
|
172
190
|
if (run.sentThisRun >= maxPerRun || timeUp) {
|
|
173
191
|
counts.over_budget += 1;
|
|
174
192
|
continue;
|
|
@@ -177,6 +195,7 @@ async function sendPendingImages(run, toSend, counts, input) {
|
|
|
177
195
|
}
|
|
178
196
|
};
|
|
179
197
|
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));
|
|
198
|
+
run.describingMs += now() - callStartedAt;
|
|
180
199
|
}
|
|
181
200
|
function countImageOutcomes(run, toSend, counts) {
|
|
182
201
|
const { settledThisRun } = run;
|
|
@@ -25,6 +25,17 @@
|
|
|
25
25
|
import { newClaudeSlimState, slimClaudeRecord } from "./claude-rules.js";
|
|
26
26
|
import { newCodexSlimState, slimCodexRecord } from "./codex-rules.js";
|
|
27
27
|
import { LineEdits, SeenBlocks, isJsonObject } from "./slim-support.js";
|
|
28
|
+
/**
|
|
29
|
+
* The version of the slim rule set (claude-rules.ts, codex-rules.ts, claude-tool-results.ts,
|
|
30
|
+
* claude-attachments.ts, slim-support.ts and this file), BLI-22794. The local prepare memo keys what a
|
|
31
|
+
* transcript became on this value (plus the secret and image rules, see telemetry-core's
|
|
32
|
+
* `evidenceRulesFingerprint`) instead of on the CLI version.
|
|
33
|
+
*
|
|
34
|
+
* CHANGING ANY SLIM RULE MEANS BUMPING THIS CONSTANT. That rebuilds every collector's prepare memo
|
|
35
|
+
* once, which is intended: the stored bytes are no longer what the new rules would produce.
|
|
36
|
+
* `slim-rules-version.test.ts` fails when a rule changes without this moving.
|
|
37
|
+
*/
|
|
38
|
+
export const SLIM_RULES_VERSION = "slim-rules.v1";
|
|
28
39
|
/** The text an image marker starts with; a line carrying one is a modified line (an image became a marker). */
|
|
29
40
|
const IMAGE_MARKER_START = "[image sha256:";
|
|
30
41
|
export function createSlimmer(kind, rulesForTests) {
|
package/dist/sync-lock.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import fsSync from "node:fs";
|
|
2
3
|
import fs from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { describeError } from "./health-detail.js";
|
|
@@ -44,23 +45,25 @@ export async function acquireSyncLock(paths, now = new Date(), options = {}) {
|
|
|
44
45
|
marks: 0,
|
|
45
46
|
lastMarkMs: now.getTime(),
|
|
46
47
|
longestGapMs: 0,
|
|
48
|
+
lockPath,
|
|
49
|
+
token,
|
|
47
50
|
};
|
|
48
51
|
const existing = await readLock(lockPath);
|
|
49
|
-
if (existing &&
|
|
52
|
+
if (existing && isFreshAndAlive(existing, now)) {
|
|
50
53
|
logHeldByLiveOwner(existing, now);
|
|
51
54
|
return { acquired: false, held_since: existing.heartbeat_at };
|
|
52
55
|
}
|
|
53
56
|
// Free or stale: try an exclusive create first; if another process created it
|
|
54
57
|
// in the gap, re-check staleness before taking over.
|
|
55
|
-
let takeover = existing ?
|
|
58
|
+
let takeover = existing ? staleReason(existing) : null;
|
|
56
59
|
let owned = await tryExclusiveCreate(lockPath, token, now, held);
|
|
57
60
|
if (!owned) {
|
|
58
61
|
const current = await readLock(lockPath);
|
|
59
|
-
if (current &&
|
|
62
|
+
if (current && isFreshAndAlive(current, now)) {
|
|
60
63
|
logHeldByLiveOwner(current, now);
|
|
61
64
|
return { acquired: false, held_since: current.heartbeat_at };
|
|
62
65
|
}
|
|
63
|
-
takeover = current ?
|
|
66
|
+
takeover = current ? staleReason(current) : "unreadable";
|
|
64
67
|
await writeLock(lockPath, token, now, held); // take over the stale lock
|
|
65
68
|
owned = true;
|
|
66
69
|
}
|
|
@@ -68,6 +71,63 @@ export async function acquireSyncLock(paths, now = new Date(), options = {}) {
|
|
|
68
71
|
heldStates.add(held);
|
|
69
72
|
return maintainOwnedLock(lockPath, token, held);
|
|
70
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Does a process with this pid exist? `kill(pid, 0)` sends nothing: ESRCH means
|
|
76
|
+
* no such process, EPERM means it exists but is not ours (alive). Same on
|
|
77
|
+
* Windows. An unknown pid (-1 from an older record) is reported alive, so only
|
|
78
|
+
* the heartbeat decides for it.
|
|
79
|
+
*/
|
|
80
|
+
export function isProcessAlive(pid) {
|
|
81
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
82
|
+
return true;
|
|
83
|
+
try {
|
|
84
|
+
process.kill(pid, 0);
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
return error.code !== "ESRCH";
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/** Blocks only when the heartbeat is fresh AND the owner process still exists. */
|
|
92
|
+
function isFreshAndAlive(record, now) {
|
|
93
|
+
return now.getTime() - record.heartbeat_ms <= STALE_TAKEOVER_MS && isProcessAlive(record.pid);
|
|
94
|
+
}
|
|
95
|
+
/** A fresh heartbeat reaching here means the owner pid is gone; otherwise it aged out. */
|
|
96
|
+
function staleReason(record) {
|
|
97
|
+
return isProcessAlive(record.pid) ? "stale" : "dead";
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Synchronous release of every lock this process still holds, for the `exit`
|
|
101
|
+
* handler (BLI-22795): a process that ends without `release()` must not keep
|
|
102
|
+
* the next tick out until the heartbeat ages. Ownership-safe by token.
|
|
103
|
+
*/
|
|
104
|
+
export function releaseHeldSyncLocksOnExit() {
|
|
105
|
+
let released = 0;
|
|
106
|
+
for (const held of [...heldStates]) {
|
|
107
|
+
heldStates.delete(held);
|
|
108
|
+
let outcome = "not_ours_or_gone";
|
|
109
|
+
try {
|
|
110
|
+
if (held.lockPath && held.token) {
|
|
111
|
+
const raw = JSON.parse(fsSync.readFileSync(held.lockPath, "utf8"));
|
|
112
|
+
if (raw.token === held.token) {
|
|
113
|
+
fsSync.rmSync(held.lockPath, { force: true });
|
|
114
|
+
outcome = "removed";
|
|
115
|
+
released += 1;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
outcome = `release_failed_${error.code ?? "unknown"}`;
|
|
121
|
+
}
|
|
122
|
+
console.error("[sync-lock] lock released on exit", JSON.stringify({
|
|
123
|
+
reason: "holder_exited_holding_lock",
|
|
124
|
+
outcome,
|
|
125
|
+
role: held.role,
|
|
126
|
+
held_ms: Math.max(0, Date.now() - held.startedMs),
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
return released;
|
|
130
|
+
}
|
|
71
131
|
function maintainOwnedLock(lockPath, token, held) {
|
|
72
132
|
const timer = setInterval(() => {
|
|
73
133
|
// Refresh only while the lock is still ours; if we were taken over, stop
|
|
@@ -132,7 +192,13 @@ function logAcquired(held, takeover, previous, now) {
|
|
|
132
192
|
? "[sync-lock] lock acquired"
|
|
133
193
|
: "[sync-lock] took over a lock whose owner stopped answering", JSON.stringify({
|
|
134
194
|
role: held.role,
|
|
135
|
-
reason: takeover === "stale"
|
|
195
|
+
reason: takeover === "stale"
|
|
196
|
+
? "heartbeat_stale"
|
|
197
|
+
: takeover === "dead"
|
|
198
|
+
? "previous_owner_dead"
|
|
199
|
+
: takeover === "unreadable"
|
|
200
|
+
? "lock_file_unreadable"
|
|
201
|
+
: "lock_free",
|
|
136
202
|
previous_pid: previous && previous.pid > 0 ? previous.pid : null,
|
|
137
203
|
previous_heartbeat_age_ms: previous ? Math.max(0, now.getTime() - previous.heartbeat_ms) : null,
|
|
138
204
|
}));
|
|
@@ -222,7 +288,10 @@ export async function inspectSyncLock(paths, now = Date.now()) {
|
|
|
222
288
|
const record = await readLock(path.join(paths.state_dir, LOCK_FILENAME));
|
|
223
289
|
if (!record)
|
|
224
290
|
return null;
|
|
225
|
-
return {
|
|
291
|
+
return {
|
|
292
|
+
...record,
|
|
293
|
+
held: now - record.heartbeat_ms <= STALE_TAKEOVER_MS && isProcessAlive(record.pid),
|
|
294
|
+
};
|
|
226
295
|
}
|
|
227
296
|
/**
|
|
228
297
|
* Remove the lock file only if it still carries `token`: the holder this caller
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.253",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.75",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.156",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.82"
|
|
33
33
|
},
|
|
34
34
|
"repository": {
|
|
35
35
|
"type": "git",
|