@bli-cockpit/cli 0.2.250 → 0.2.252
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-prepare-memo.js +39 -9
- package/dist/adapters/raw-evidence-transcript-file.js +36 -16
- package/dist/adapters/raw-evidence-volatile-reuse.js +27 -0
- package/dist/adapters/raw-evidence.js +5 -2
- package/dist/commands/backfill-live-window.js +41 -0
- package/dist/commands/backfill-report.js +11 -1
- package/dist/commands/backfill-scan.js +8 -4
- package/dist/commands/public-root.js +1 -1
- package/dist/crash-guard.js +9 -0
- package/dist/evidence-upload-deadline.js +36 -0
- package/dist/evidence-upload-direct.js +6 -5
- package/dist/evidence-upload-transport.js +7 -3
- package/dist/image-description-session.js +39 -9
- package/dist/slim/slim-transcript.js +11 -0
- package/dist/sync-lock.js +75 -6
- package/package.json +4 -4
|
@@ -13,11 +13,14 @@
|
|
|
13
13
|
*
|
|
14
14
|
* - the file's size, inode and mtime/ctime (nanoseconds) are exactly what they were when the
|
|
15
15
|
* entry was written, so the file has not been appended to, rewritten or replaced;
|
|
16
|
-
* - the recipe is the same:
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
16
|
+
* - the recipe is the same: a fingerprint of the rules that decide the bytes (the slim rules, the
|
|
17
|
+
* secret rules, the image marker format; NOT the CLI version, BLI-22794), the machine's
|
|
18
|
+
* image-handling mode, and whether an image step ran at all;
|
|
19
|
+
* - the entry is not `volatile`, or it is volatile ONLY because of images with no stored
|
|
20
|
+
* description (`pending_images`, BLI-22794). Such an entry is handed back to the caller, who
|
|
21
|
+
* trusts it while none of those images has been described since; a described one rebuilds. A
|
|
22
|
+
* volatile entry without `pending_images` (a failed slim step, or any older entry) is never
|
|
23
|
+
* recalled: it can change shape on the next tick without the file changing.
|
|
21
24
|
*
|
|
22
25
|
* The caller then reuses the entry ONLY when the remembered content hash is already durable
|
|
23
26
|
* remotely (`skipContentHashes`). Anything else, including every file that is not yet delivered,
|
|
@@ -25,11 +28,13 @@
|
|
|
25
28
|
* known. A memo that is lost, corrupt, or wrong in the safe direction costs one slow tick, never
|
|
26
29
|
* a missed upload. It holds ids, sizes, hashes and redaction metadata, never a byte of content.
|
|
27
30
|
*/
|
|
31
|
+
import { createHash } from "node:crypto";
|
|
28
32
|
import path from "node:path";
|
|
29
33
|
import fs from "node:fs/promises";
|
|
30
34
|
import { isMissingFileFailure } from "../health-detail.js";
|
|
31
35
|
import { readJsonFile, writeJsonFileAtomic } from "../local-state-files.js";
|
|
32
|
-
import {
|
|
36
|
+
import { evidenceRulesFingerprint } from "@bli-cockpit/telemetry-core";
|
|
37
|
+
import { SLIM_RULES_VERSION } from "../slim/slim-transcript.js";
|
|
33
38
|
export const PREPARE_MEMO_FILENAME = "transcript-prepare-memo.json";
|
|
34
39
|
const MEMO_FORMAT = "prepare-memo.v1";
|
|
35
40
|
/** About 700 bytes an entry, so this is at most a few MB; the oldest leave first. */
|
|
@@ -54,14 +59,25 @@ export async function statSourceForMemo(filePath) {
|
|
|
54
59
|
return null;
|
|
55
60
|
}
|
|
56
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* What the prepared bytes depend on besides the file: the slim rules and the secret and image rules.
|
|
64
|
+
* Computed once per process. NOT the CLI version (BLI-22794): an upgrade that changes none of these
|
|
65
|
+
* keeps the memo warm. Changing a rule changes this, which rebuilds the memo once; that is intended
|
|
66
|
+
* (see `SLIM_RULES_VERSION` and telemetry-core's `evidenceRulesFingerprint`).
|
|
67
|
+
*/
|
|
68
|
+
let rulesFingerprint;
|
|
69
|
+
export function rulesFingerprintOf(slimRulesVersion, evidenceRules) {
|
|
70
|
+
return createHash("sha256").update(`${slimRulesVersion}|${evidenceRules}`).digest("hex").slice(0, 32);
|
|
71
|
+
}
|
|
57
72
|
/** Everything besides the file that decides what its prepared bytes are. */
|
|
58
73
|
export function prepareRecipe(options) {
|
|
59
|
-
|
|
74
|
+
rulesFingerprint ??= rulesFingerprintOf(SLIM_RULES_VERSION, evidenceRulesFingerprint());
|
|
75
|
+
return [MEMO_FORMAT, rulesFingerprint, options.kind, options.imageMode ?? "no_image_step"].join("|");
|
|
60
76
|
}
|
|
61
77
|
export function prepareMemoPath(stateDir) {
|
|
62
78
|
return path.join(stateDir, PREPARE_MEMO_FILENAME);
|
|
63
79
|
}
|
|
64
|
-
const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0 };
|
|
80
|
+
const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0, volatile_reused_count: 0, volatile_reused_bytes: 0, rebuilt_image_now_described_count: 0 };
|
|
65
81
|
export async function openPrepareMemo(stateDir) {
|
|
66
82
|
const entries = await readMemoEntries(stateDir);
|
|
67
83
|
let dirty = false;
|
|
@@ -73,7 +89,9 @@ export async function openPrepareMemo(stateDir) {
|
|
|
73
89
|
const entry = entries.get(sourceKey);
|
|
74
90
|
if (!entry)
|
|
75
91
|
return null;
|
|
76
|
-
if (entry.volatile
|
|
92
|
+
if (entry.volatile && !entry.pending_images)
|
|
93
|
+
return null;
|
|
94
|
+
if (entry.recipe !== recipe)
|
|
77
95
|
return null;
|
|
78
96
|
if (entry.size !== stamp.size || entry.identity !== stamp.identity)
|
|
79
97
|
return null;
|
|
@@ -85,6 +103,15 @@ export async function openPrepareMemo(stateDir) {
|
|
|
85
103
|
dirty = true;
|
|
86
104
|
},
|
|
87
105
|
count(outcome, bytes) {
|
|
106
|
+
if (outcome === "rebuilt_image_now_described") {
|
|
107
|
+
counts.rebuilt_image_now_described_count += 1;
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
if (outcome === "volatile_reused") {
|
|
111
|
+
counts.volatile_reused_count += 1;
|
|
112
|
+
counts.volatile_reused_bytes += bytes;
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
88
115
|
if (outcome === "skipped") {
|
|
89
116
|
counts.skipped_count += 1;
|
|
90
117
|
counts.skipped_bytes += bytes;
|
|
@@ -184,6 +211,9 @@ function parseEntry(value) {
|
|
|
184
211
|
return null;
|
|
185
212
|
if (typeof entry["volatile"] !== "boolean" || typeof entry["seen_ms"] !== "number")
|
|
186
213
|
return null;
|
|
214
|
+
const pending = entry["pending_images"];
|
|
215
|
+
if (pending !== undefined && (!Array.isArray(pending) || !pending.every((item) => typeof item === "string")))
|
|
216
|
+
return null;
|
|
187
217
|
const redacted = entry["redacted"];
|
|
188
218
|
if (redacted !== null && (typeof redacted !== "object" || typeof redacted.completenessLabel !== "string"))
|
|
189
219
|
return null;
|
|
@@ -26,6 +26,7 @@ import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
|
|
|
26
26
|
import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
|
|
27
27
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
28
28
|
import { prepareRecipe, statSourceForMemo, } from "./raw-evidence-prepare-memo.js";
|
|
29
|
+
import { logVolatileReuse, trustVolatileEntry } from "./raw-evidence-volatile-reuse.js";
|
|
29
30
|
import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
|
|
30
31
|
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
31
32
|
/**
|
|
@@ -63,9 +64,9 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
63
64
|
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
64
65
|
return skip("file_too_large");
|
|
65
66
|
}
|
|
66
|
-
const { sanitized, contentHash, stubClaim, volatile, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
|
|
67
|
+
const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
|
|
67
68
|
await rememberPreparedTranscript(collection, options, names, {
|
|
68
|
-
stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, redactedEntry,
|
|
69
|
+
stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
|
|
69
70
|
});
|
|
70
71
|
if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
|
|
71
72
|
collection.reused.push({
|
|
@@ -121,7 +122,7 @@ async function prepareTranscriptForUpload(collection, options, names, bytes) {
|
|
|
121
122
|
byteSize: sanitized.bytes.byteLength,
|
|
122
123
|
alreadyDurable: collection.skipContentHashes.has(contentHash),
|
|
123
124
|
});
|
|
124
|
-
return { sanitized, contentHash, stubClaim, volatile: images.volatile, redactedEntry };
|
|
125
|
+
return { sanitized, contentHash, stubClaim, volatile: images.volatile, pendingImages: images.pendingImages, redactedEntry };
|
|
125
126
|
}
|
|
126
127
|
/**
|
|
127
128
|
* The transcript bytes after the image step. Unchanged (the very same buffer) unless this sync has
|
|
@@ -133,10 +134,10 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
133
134
|
const session = collection.budget.images;
|
|
134
135
|
const source = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
|
|
135
136
|
if (!session || !source)
|
|
136
|
-
return { bytes: file.bytes, volatile: false };
|
|
137
|
+
return { bytes: file.bytes, volatile: false, pendingImages: null };
|
|
137
138
|
// Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
|
|
138
139
|
if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.bytes))
|
|
139
|
-
return { bytes: file.bytes, volatile: false };
|
|
140
|
+
return { bytes: file.bytes, volatile: false, pendingImages: null };
|
|
140
141
|
try {
|
|
141
142
|
const prepared = await session.prepareTranscript({
|
|
142
143
|
text: file.bytes.toString("utf8"),
|
|
@@ -151,13 +152,16 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
151
152
|
: prepared.imageCount > 0;
|
|
152
153
|
const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
|
|
153
154
|
const volatile = imagesPending || slimFailed;
|
|
155
|
+
// BLI-22794: volatile ONLY because of images without a stored description (the slim step held
|
|
156
|
+
// up): the bytes change only when one of these gains one, so the memo may keep the entry.
|
|
157
|
+
const pendingImages = volatile && !slimFailed ? prepared.pendingImages : null;
|
|
154
158
|
if (!prepared.changed)
|
|
155
|
-
return { bytes: file.bytes, volatile };
|
|
156
|
-
return { bytes: Buffer.from(prepared.text, "utf8"), volatile };
|
|
159
|
+
return { bytes: file.bytes, volatile, pendingImages };
|
|
160
|
+
return { bytes: Buffer.from(prepared.text, "utf8"), volatile, pendingImages };
|
|
157
161
|
}
|
|
158
162
|
catch (error) {
|
|
159
163
|
console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
|
|
160
|
-
return { bytes: file.bytes, volatile: true };
|
|
164
|
+
return { bytes: file.bytes, volatile: true, pendingImages: null };
|
|
161
165
|
}
|
|
162
166
|
}
|
|
163
167
|
/**
|
|
@@ -178,8 +182,15 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
|
|
|
178
182
|
if (options.maxFileBytes && stamp.size > options.maxFileBytes)
|
|
179
183
|
return false;
|
|
180
184
|
const known = memo.recall(names.sourceKey, stamp, recipe);
|
|
181
|
-
if (!known
|
|
185
|
+
if (!known)
|
|
182
186
|
return false;
|
|
187
|
+
if (known.volatile && !trustVolatileEntry(collection, memo, known, options.kind, names.sourceKey, stamp.size))
|
|
188
|
+
return false;
|
|
189
|
+
if (!collection.skipContentHashes.has(known.content_hash)) {
|
|
190
|
+
if (known.volatile)
|
|
191
|
+
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "hash_not_durable_full_path");
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
183
194
|
if (known.redacted) {
|
|
184
195
|
collection.redacted.push({
|
|
185
196
|
kind: options.kind,
|
|
@@ -194,6 +205,12 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
|
|
|
194
205
|
content_hash_sha256: known.content_hash,
|
|
195
206
|
codex_session_id: options.sessionId,
|
|
196
207
|
});
|
|
208
|
+
if (known.volatile) {
|
|
209
|
+
// BLI-22794: only images without a description held it back, and none has one now.
|
|
210
|
+
memo.count("volatile_reused", stamp.size);
|
|
211
|
+
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "pending_images_still_undescribed");
|
|
212
|
+
return true;
|
|
213
|
+
}
|
|
197
214
|
memo.count("skipped", stamp.size);
|
|
198
215
|
return true;
|
|
199
216
|
}
|
|
@@ -203,7 +220,7 @@ async function rememberPreparedTranscript(collection, options, names, built) {
|
|
|
203
220
|
if (!memo)
|
|
204
221
|
return;
|
|
205
222
|
memo.count(built.volatile ? "volatile" : "rebuilt", built.readBytes);
|
|
206
|
-
if (built.volatile || !built.stamp)
|
|
223
|
+
if ((built.volatile && !built.pendingImages) || !built.stamp)
|
|
207
224
|
return;
|
|
208
225
|
const after = await statSourceForMemo(options.filePath);
|
|
209
226
|
const stable = after !== null
|
|
@@ -212,10 +229,7 @@ async function rememberPreparedTranscript(collection, options, names, built) {
|
|
|
212
229
|
&& built.readBytes === built.stamp.size;
|
|
213
230
|
if (!stable)
|
|
214
231
|
return;
|
|
215
|
-
const
|
|
216
|
-
if (known?.content_hash === built.contentHash)
|
|
217
|
-
return;
|
|
218
|
-
memo.remember(names.sourceKey, {
|
|
232
|
+
const entry = {
|
|
219
233
|
...built.stamp,
|
|
220
234
|
recipe: built.recipe,
|
|
221
235
|
content_hash: built.contentHash,
|
|
@@ -223,8 +237,14 @@ async function rememberPreparedTranscript(collection, options, names, built) {
|
|
|
223
237
|
redacted: built.redactedEntry
|
|
224
238
|
? { redaction: built.redactedEntry.redaction, completenessLabel: built.redactedEntry.completenessLabel }
|
|
225
239
|
: null,
|
|
226
|
-
|
|
227
|
-
|
|
240
|
+
};
|
|
241
|
+
const known = memo.recall(names.sourceKey, built.stamp, built.recipe);
|
|
242
|
+
const pending = built.volatile ? built.pendingImages : undefined;
|
|
243
|
+
if (known && known.volatile === built.volatile && known.content_hash === built.contentHash
|
|
244
|
+
&& (known.pending_images ?? []).join() === (pending ?? []).join())
|
|
245
|
+
return;
|
|
246
|
+
// BLI-22794: a volatile entry is kept across ticks with the images it is waiting on.
|
|
247
|
+
memo.remember(names.sourceKey, { ...entry, volatile: built.volatile, ...(pending ? { pending_images: pending } : {}) });
|
|
228
248
|
}
|
|
229
249
|
function nameEvidenceFile(options) {
|
|
230
250
|
const fileName = path.basename(options.filePath);
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLI-22794: may a remembered image-pending entry stand in for a rebuild? Not when one of its
|
|
3
|
+
* images has a stored description now (the bytes would differ, counted as
|
|
4
|
+
* `rebuilt_image_now_described_count`), and not while the describer could still take one of them
|
|
5
|
+
* this tick: a skipped transcript never reaches the describer, so reusing it would starve the
|
|
6
|
+
* images. Once the tick's budget is spent, the rest are reused until the next tick.
|
|
7
|
+
*/
|
|
8
|
+
export function trustVolatileEntry(collection, memo, known, kind, sourceKey, bytes) {
|
|
9
|
+
const session = collection.budget.images;
|
|
10
|
+
const pending = known.pending_images ?? [];
|
|
11
|
+
if (!session)
|
|
12
|
+
return false;
|
|
13
|
+
if (pending.some((sha256) => session.isDescribed(sha256))) {
|
|
14
|
+
memo.count("rebuilt_image_now_described", bytes);
|
|
15
|
+
logVolatileReuse(kind, sourceKey, known.content_hash, "rebuilt_image_now_described");
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
if (session.hasDescribeRoom(pending)) {
|
|
19
|
+
logVolatileReuse(kind, sourceKey, known.content_hash, "rebuilt_describer_has_room");
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
export function logVolatileReuse(kind, sourceKey, contentHash, reason) {
|
|
25
|
+
const reused = reason === "pending_images_still_undescribed";
|
|
26
|
+
console.error("[raw-evidence] volatile transcript " + (reused ? "reused across ticks" : "rebuilt"), JSON.stringify({ reason, kind, source_key: sourceKey, hash_prefix: contentHash.slice(0, 12) }));
|
|
27
|
+
}
|
|
@@ -107,15 +107,18 @@ export async function collectRawEvidencePack(context, options) {
|
|
|
107
107
|
*/
|
|
108
108
|
async function reportPrepareMemo(collection) {
|
|
109
109
|
const counts = await collection.budget.prepareMemo?.settle();
|
|
110
|
-
if (!counts || counts.skipped_count + counts.rebuilt_count === 0)
|
|
110
|
+
if (!counts || counts.skipped_count + counts.rebuilt_count + counts.volatile_reused_count === 0)
|
|
111
111
|
return;
|
|
112
|
-
console.error(counts.skipped_count > 0 ? "[raw-evidence] slim skipped unchanged" : "[raw-evidence] no unchanged transcripts to skip", JSON.stringify({
|
|
112
|
+
console.error(counts.skipped_count + counts.volatile_reused_count > 0 ? "[raw-evidence] slim skipped unchanged" : "[raw-evidence] no unchanged transcripts to skip", JSON.stringify({
|
|
113
113
|
reason: "source_unchanged_since_last_prepare",
|
|
114
114
|
count: counts.skipped_count,
|
|
115
115
|
bytes: counts.skipped_bytes,
|
|
116
116
|
rebuilt_count: counts.rebuilt_count,
|
|
117
117
|
rebuilt_bytes: counts.rebuilt_bytes,
|
|
118
118
|
rebuilt_volatile_count: counts.volatile_count,
|
|
119
|
+
rebuilt_volatile_reused_count: counts.volatile_reused_count,
|
|
120
|
+
rebuilt_volatile_reused_bytes: counts.volatile_reused_bytes,
|
|
121
|
+
rebuilt_image_now_described_count: counts.rebuilt_image_now_described_count,
|
|
119
122
|
}));
|
|
120
123
|
}
|
|
121
124
|
/**
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLI-22790: backfill is history. A session file modified inside the live window belongs to the
|
|
3
|
+
* normal sync tick, which collects the same `DEFAULT_SINCE_MINUTES` (24 h) every ~15 minutes and
|
|
4
|
+
* already uploads these sessions (`riding_already_durable`). Re-preparing a live, still-growing
|
|
5
|
+
* session in backfill held the collection lock for 232 s per tick and kept the run "partial"
|
|
6
|
+
* forever, so doctor re-ran the backfill on every tick.
|
|
7
|
+
*/
|
|
8
|
+
import { DEFAULT_SINCE_MINUTES } from "../adapters/raw-evidence-collection-state.js";
|
|
9
|
+
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
10
|
+
export const BACKFILL_LIVE_WINDOW_MS = DEFAULT_SINCE_MINUTES * 60 * 1000;
|
|
11
|
+
/** The window this run uses: none for a stub pass (it names its sessions), else the test seam or the default. */
|
|
12
|
+
export function liveWindowMsFor(command) {
|
|
13
|
+
return command.stubPass ? 0 : (command.liveWindowMs ?? BACKFILL_LIVE_WINDOW_MS);
|
|
14
|
+
}
|
|
15
|
+
/** `onlySessions` (`codex:<id>` / `claude_code:<id>`) when set, then the live window. */
|
|
16
|
+
export function scopeBackfillCandidates(candidates, options) {
|
|
17
|
+
const key = (c) => stubSessionKey(c.source, c.session_id);
|
|
18
|
+
const scoped = options.onlySessions ? candidates.filter((c) => options.onlySessions.has(key(c))) : candidates;
|
|
19
|
+
return leaveLiveSessionsToTheTick(scoped, options.live);
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Drop the candidates the tick owns: modified within `liveWindowMs` (0 keeps all), unless the
|
|
23
|
+
* server names the session a redaction stub (BLI-11102), which is re-offered even while live.
|
|
24
|
+
* Logs the count either way.
|
|
25
|
+
*/
|
|
26
|
+
export function leaveLiveSessionsToTheTick(candidates, options) {
|
|
27
|
+
let liveExcluded = 0;
|
|
28
|
+
const kept = options.liveWindowMs <= 0 ? candidates : candidates.filter((candidate) => {
|
|
29
|
+
const live = options.now.getTime() - candidate.session_file_mtime_ms < options.liveWindowMs
|
|
30
|
+
&& !options.stubbedSessions.has(stubSessionKey(candidate.source, candidate.session_id));
|
|
31
|
+
if (live)
|
|
32
|
+
liveExcluded += 1;
|
|
33
|
+
return !live;
|
|
34
|
+
});
|
|
35
|
+
console.error("[backfill] live sessions left to the tick", JSON.stringify({
|
|
36
|
+
reason: liveExcluded > 0 ? "live_owned_by_tick" : "no_live_sessions",
|
|
37
|
+
live_excluded: liveExcluded,
|
|
38
|
+
live_window_minutes: Math.round(options.liveWindowMs / 60_000),
|
|
39
|
+
}));
|
|
40
|
+
return { kept, liveExcluded };
|
|
41
|
+
}
|
|
@@ -228,7 +228,17 @@ function countRemainingHistory(options) {
|
|
|
228
228
|
async function writeAllHistoryCompletionMarker(ctx) {
|
|
229
229
|
const { paths, cursor, sources, scopedCursor, scan, oversizedCandidateKeys, now } = ctx;
|
|
230
230
|
const cappedSidecars = cappedSidecarCount(scan.candidates);
|
|
231
|
-
|
|
231
|
+
// BLI-22790: sessions left to the tick were NOT covered here. Mark coverage only up to the
|
|
232
|
+
// start of the live window, so once they age out a later backfill still sees them.
|
|
233
|
+
const coveredThrough = scan.live_excluded_count > 0
|
|
234
|
+
? new Date(now.getTime() - scan.live_window_ms)
|
|
235
|
+
: now;
|
|
236
|
+
console.error("[backfill] history coverage recorded", JSON.stringify({
|
|
237
|
+
reason: scan.live_excluded_count > 0 ? "covered_through_live_window_start" : "covered_through_now",
|
|
238
|
+
live_excluded: scan.live_excluded_count,
|
|
239
|
+
live_window_minutes: Math.round(scan.live_window_ms / 60_000),
|
|
240
|
+
}));
|
|
241
|
+
recordBackfillScanCoverage(cursor, sources, coveredThrough, now);
|
|
232
242
|
await writeBackfillCursor(paths, cursor);
|
|
233
243
|
const oversizedCandidates = scan.candidates.filter((candidate) => oversizedCandidateKeys.has(candidateCursorKey(candidate)));
|
|
234
244
|
await writeBackfillCompletionMarker(paths, {
|
|
@@ -14,6 +14,7 @@ import { scanAndAttributeClaudeSessions, } from "../adapters/claude-attribution.
|
|
|
14
14
|
import { defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
|
|
15
15
|
import { emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
16
16
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
17
|
+
import { liveWindowMsFor, scopeBackfillCandidates } from "./backfill-live-window.js";
|
|
17
18
|
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
18
19
|
import { candidateCursorKey, compareBackfillCandidates, isAfterCursor, } from "./backfill-candidates.js";
|
|
19
20
|
import { addReadOnlyGuardIssues, addRepoDiscoveryIssues, backfillScanIssues, countReadOnlyGuards, deterministicSkipCandidateKeys, oversizedBackfillCandidateKeys, retryableCandidateKeys, sortScanIssuesByPriority, } from "./backfill-issues.js";
|
|
@@ -23,7 +24,6 @@ import { blockedBackfillResult } from "./backfill-result.js";
|
|
|
23
24
|
import { emptyStubReofferPlan, loadStubReofferPlan, reopenStubbedSessions, } from "./backfill-stub-reoffer.js";
|
|
24
25
|
import { loadStubCursorHashes, locateStubbedSessions } from "./backfill-stub-locate.js";
|
|
25
26
|
import { resolveBackfillScope } from "./backfill-scope.js";
|
|
26
|
-
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
27
27
|
/**
|
|
28
28
|
* SCAN: is this machine paired, what roots/sources/window apply, and what did
|
|
29
29
|
* the archived Codex + Claude session stores actually contain. Returns either
|
|
@@ -149,11 +149,13 @@ async function scanBackfillSessions(options) {
|
|
|
149
149
|
cursorHashes: options.stubCursorHashes,
|
|
150
150
|
sinceMinutes: options.window.since_minutes,
|
|
151
151
|
});
|
|
152
|
+
const liveWindowMs = liveWindowMsFor(options.command);
|
|
152
153
|
const selection = selectBackfillCandidates({
|
|
153
154
|
scanned,
|
|
154
155
|
cursor: options.cursor,
|
|
155
156
|
pointerlessTerminalSessions: options.pointerlessTerminalSessions,
|
|
156
157
|
maxFiles: options.command.maxFiles,
|
|
158
|
+
live: { liveWindowMs, now: options.now, stubbedSessions: options.stubReoffer.sessions },
|
|
157
159
|
onlySessions: options.command.stubPass
|
|
158
160
|
? new Set(options.command.stubPass.plan.sessions.keys())
|
|
159
161
|
: undefined,
|
|
@@ -177,6 +179,8 @@ async function scanBackfillSessions(options) {
|
|
|
177
179
|
issues,
|
|
178
180
|
retryable_candidate_keys: retryableCandidateKeys(candidates),
|
|
179
181
|
omitted_candidate_count: selection.omittedCandidateCount,
|
|
182
|
+
live_excluded_count: selection.liveExcluded,
|
|
183
|
+
live_window_ms: liveWindowMs,
|
|
180
184
|
};
|
|
181
185
|
}
|
|
182
186
|
/**
|
|
@@ -220,9 +224,8 @@ async function scanClaudeHistory(request, projectsDir, scanLimit) {
|
|
|
220
224
|
* so a `--max-files` cap spends its budget on them before newer history.
|
|
221
225
|
*/
|
|
222
226
|
function selectBackfillCandidates(options) {
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
options.onlySessions.has(stubSessionKey(candidate.source, candidate.session_id)))
|
|
227
|
+
const { kept, liveExcluded } = scopeBackfillCandidates(options.scanned, options);
|
|
228
|
+
const allCandidates = kept
|
|
226
229
|
.filter((candidate) => isAfterCursor(candidate, options.cursor) ||
|
|
227
230
|
isPointerlessTerminalRetry(candidate, options.pointerlessTerminalSessions))
|
|
228
231
|
.sort((a, b) => compareBackfillCandidatesForRetry(a, b, options.pointerlessTerminalSessions));
|
|
@@ -230,6 +233,7 @@ function selectBackfillCandidates(options) {
|
|
|
230
233
|
return {
|
|
231
234
|
candidates,
|
|
232
235
|
omittedCandidateCount: allCandidates.length - candidates.length,
|
|
236
|
+
liveExcluded,
|
|
233
237
|
};
|
|
234
238
|
}
|
|
235
239
|
async function loadPointerlessTerminalSessions(paths, sources) {
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.252");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
package/dist/crash-guard.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { releaseHeldSyncLocksOnExit } from "./sync-lock.js";
|
|
1
2
|
/**
|
|
2
3
|
* A run that dies must not be able to say it worked (BLI-4110).
|
|
3
4
|
*
|
|
@@ -125,6 +126,14 @@ export function installCrashGuard(options = {}) {
|
|
|
125
126
|
}));
|
|
126
127
|
});
|
|
127
128
|
target.on("exit", () => {
|
|
129
|
+
// A lock still held at exit is an orphan, whatever the exit code: release
|
|
130
|
+
// it synchronously so the next tick is not blocked (BLI-22795).
|
|
131
|
+
try {
|
|
132
|
+
releaseHeldSyncLocksOnExit();
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
console.error("[sync-lock] exit release threw", JSON.stringify({ reason: "exit_release_threw", ...describeCrash(error) }));
|
|
136
|
+
}
|
|
128
137
|
if (commandReturned)
|
|
129
138
|
return;
|
|
130
139
|
// `EXIT_DIED_MID_RUN` has to be in this set, and leaving it out made this
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A deadline that keeps the event loop alive (BLI-22795).
|
|
3
|
+
*
|
|
4
|
+
* `AbortSignal.timeout()` is documented to use an unref'd timer. A fetch whose
|
|
5
|
+
* socket has gone (network dropped mid-upload) then has nothing ref'd behind
|
|
6
|
+
* it: the loop drains, Node exits with the await still pending, and the
|
|
7
|
+
* crash guard logs `died_mid_run` with the collection lock still held. Here
|
|
8
|
+
* the timer is ref'd, and the race rejects even when an injected or broken
|
|
9
|
+
* fetch ignores the abort, so the caller always gets a named failure.
|
|
10
|
+
*/
|
|
11
|
+
export const UPLOAD_DEADLINE_ERROR_NAME = "EvidenceUploadDeadlineError";
|
|
12
|
+
export function isUploadDeadlineError(error) {
|
|
13
|
+
return error instanceof Error && error.name === UPLOAD_DEADLINE_ERROR_NAME;
|
|
14
|
+
}
|
|
15
|
+
export async function fetchWithDeadline(fetchImpl, url, init, deadlineMs) {
|
|
16
|
+
const controller = new AbortController();
|
|
17
|
+
let timer;
|
|
18
|
+
const deadline = new Promise((_, reject) => {
|
|
19
|
+
timer = setTimeout(() => {
|
|
20
|
+
const error = new Error("upload request exceeded its deadline");
|
|
21
|
+
error.name = UPLOAD_DEADLINE_ERROR_NAME;
|
|
22
|
+
controller.abort();
|
|
23
|
+
reject(error);
|
|
24
|
+
}, deadlineMs);
|
|
25
|
+
});
|
|
26
|
+
try {
|
|
27
|
+
const pending = fetchImpl(url, { ...init, signal: controller.signal });
|
|
28
|
+
// If the deadline wins, the abandoned fetch may reject later; that must not
|
|
29
|
+
// become an unhandled rejection.
|
|
30
|
+
pending.catch(() => undefined);
|
|
31
|
+
return await Promise.race([pending, deadline]);
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
clearTimeout(timer);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { encodeEvidenceBytes } from "@bli-cockpit/telemetry-core/evidence-storage-encoding";
|
|
2
|
+
import { fetchWithDeadline, isUploadDeadlineError } from "./evidence-upload-deadline.js";
|
|
2
3
|
import { noteSyncProgress } from "./sync-lock.js";
|
|
3
4
|
/** The signed capability is the only credential sent to Storage. Never log it,
|
|
4
5
|
* its query string, provider bodies, or thrown errors that might contain it. */
|
|
@@ -13,10 +14,10 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index)
|
|
|
13
14
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
14
15
|
let status = 0;
|
|
15
16
|
try {
|
|
16
|
-
const response = await options.fetchImpl
|
|
17
|
+
const response = await fetchWithDeadline(options.fetchImpl, signedUrl, {
|
|
17
18
|
method: "PUT", headers: { "Content-Type": "application/octet-stream", "x-upsert": "false" },
|
|
18
|
-
body: new Uint8Array(encoded),
|
|
19
|
-
});
|
|
19
|
+
body: new Uint8Array(encoded), redirect: "error",
|
|
20
|
+
}, 60_000);
|
|
20
21
|
status = response.status;
|
|
21
22
|
await response.body?.cancel();
|
|
22
23
|
if (response.ok) {
|
|
@@ -28,8 +29,8 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index)
|
|
|
28
29
|
if (status < 500 && status !== 429)
|
|
29
30
|
return false;
|
|
30
31
|
}
|
|
31
|
-
catch {
|
|
32
|
-
log("signed_upload_network_failure", uploadId, index, attempt, status);
|
|
32
|
+
catch (error) {
|
|
33
|
+
log(isUploadDeadlineError(error) ? "signed_upload_deadline_exceeded" : "signed_upload_network_failure", uploadId, index, attempt, status);
|
|
33
34
|
}
|
|
34
35
|
if (attempt < attempts)
|
|
35
36
|
await (options.sleep ?? sleep)(250 * attempt);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import { describeError } from "./health-detail.js";
|
|
3
|
+
import { fetchWithDeadline, isUploadDeadlineError } from "./evidence-upload-deadline.js";
|
|
3
4
|
import { noteSyncProgress } from "./sync-lock.js";
|
|
4
5
|
/**
|
|
5
6
|
* The `send` stage: one retrying POST to an evidence-upload route, plus the
|
|
@@ -10,6 +11,7 @@ import { noteSyncProgress } from "./sync-lock.js";
|
|
|
10
11
|
*/
|
|
11
12
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
12
13
|
const RETRY_DELAY_MS = 250;
|
|
14
|
+
const REQUEST_DEADLINE_MS = 120_000;
|
|
13
15
|
export async function requestJson(options, routePath, body) {
|
|
14
16
|
const maxAttempts = options.maxAttemptsPerRequest ?? DEFAULT_MAX_ATTEMPTS;
|
|
15
17
|
const sleep = options.sleep ?? defaultSleep;
|
|
@@ -17,7 +19,7 @@ export async function requestJson(options, routePath, body) {
|
|
|
17
19
|
let lastBody = null;
|
|
18
20
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
19
21
|
try {
|
|
20
|
-
const response = await options.fetchImpl
|
|
22
|
+
const response = await fetchWithDeadline(options.fetchImpl, `${options.dashboardUrl}${routePath}`, {
|
|
21
23
|
method: "POST",
|
|
22
24
|
headers: {
|
|
23
25
|
"Authorization": `Bearer ${options.deviceToken}`,
|
|
@@ -26,7 +28,7 @@ export async function requestJson(options, routePath, body) {
|
|
|
26
28
|
? { "x-tower-evidence-direct": "staged-gzip-v1" } : {}),
|
|
27
29
|
},
|
|
28
30
|
body: JSON.stringify(body),
|
|
29
|
-
});
|
|
31
|
+
}, REQUEST_DEADLINE_MS);
|
|
30
32
|
lastStatus = response.status;
|
|
31
33
|
lastBody = await readResponseJson(response);
|
|
32
34
|
if (response.ok || (response.status < 500 && response.status !== 429)) {
|
|
@@ -42,7 +44,9 @@ export async function requestJson(options, routePath, body) {
|
|
|
42
44
|
// the request left the machine. This is the exact silence BLI-2528 sat
|
|
43
45
|
// behind for 57 days; the route is named, the body never is.
|
|
44
46
|
console.error("[evidence-upload] request failed before a status came back", JSON.stringify({
|
|
45
|
-
reason:
|
|
47
|
+
reason: isUploadDeadlineError(error)
|
|
48
|
+
? "upload_request_deadline_exceeded"
|
|
49
|
+
: "upload_request_transport_error",
|
|
46
50
|
route: routePath,
|
|
47
51
|
attempt,
|
|
48
52
|
max_attempts: maxAttempts,
|
|
@@ -10,9 +10,14 @@ import { postImageToDescribeDoor } from "./image-describe-client.js";
|
|
|
10
10
|
import { resolveImageHandlingMode } from "./image-mode-state.js";
|
|
11
11
|
import { slimOrKeep, slimTranscript } from "./slim/slim-transcript.js";
|
|
12
12
|
import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
|
|
13
|
-
/** New images sent to the door in one sync (all worktrees together). */
|
|
13
|
+
/** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
|
|
14
14
|
export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* After this long spent DESCRIBING, remaining images wait for the next tick. Only time inside the
|
|
17
|
+
* sends counts: collection reaches image-bearing transcripts minutes into a tick and keeps slimming
|
|
18
|
+
* other transcripts between sends, and a wall clock from the session open (or from the first send)
|
|
19
|
+
* spent the budget on that work, so almost nothing was described (BLI-22789: 1 of 1,874, then 10).
|
|
20
|
+
*/
|
|
16
21
|
export const IMAGE_DESCRIBE_MAX_RUN_MS = 120_000;
|
|
17
22
|
const CONCURRENCY = 3;
|
|
18
23
|
const TAG = "[image-descriptions]";
|
|
@@ -25,13 +30,14 @@ export async function openImageDescriptions(options) {
|
|
|
25
30
|
const stateDirty = false;
|
|
26
31
|
const settledThisRun = new Map();
|
|
27
32
|
const now = options.now ?? Date.now;
|
|
28
|
-
const startedAt = now();
|
|
29
33
|
const maxPerRun = options.maxPerRun ?? IMAGE_DESCRIBE_MAX_PER_RUN;
|
|
30
34
|
const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
|
|
31
35
|
const sentThisRun = 0;
|
|
32
|
-
const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now,
|
|
36
|
+
const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun };
|
|
33
37
|
return {
|
|
34
38
|
mode,
|
|
39
|
+
isDescribed: (sha256) => described.has(imageStateKey(sha256)),
|
|
40
|
+
hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
|
|
35
41
|
async prepareTranscript(input) {
|
|
36
42
|
const afterImages = await describeAndReplace(run, input);
|
|
37
43
|
if (mode !== "describe_and_drop")
|
|
@@ -41,6 +47,19 @@ export async function openImageDescriptions(options) {
|
|
|
41
47
|
},
|
|
42
48
|
};
|
|
43
49
|
}
|
|
50
|
+
function describeRoomLeft(run, sha256s) {
|
|
51
|
+
if (run.sentThisRun >= run.maxPerRun)
|
|
52
|
+
return false;
|
|
53
|
+
if (run.describingMs > run.maxRunMs)
|
|
54
|
+
return false;
|
|
55
|
+
return sha256s.some((sha256) => {
|
|
56
|
+
// Already sent (or found too large) this run: it will not be sent again before the next tick.
|
|
57
|
+
if (run.settledThisRun.has(sha256))
|
|
58
|
+
return false;
|
|
59
|
+
const failure = run.failedImages[imageStateKey(sha256)];
|
|
60
|
+
return !(failure && failure.retry_after > run.now());
|
|
61
|
+
});
|
|
62
|
+
}
|
|
44
63
|
function markDescribed(run, sha256) {
|
|
45
64
|
const { described, orderedKeys } = run;
|
|
46
65
|
const key = imageStateKey(sha256);
|
|
@@ -92,12 +111,12 @@ async function describeAndReplace(run, input) {
|
|
|
92
111
|
const { mode, described, options, orderedKeys, failedImages } = run;
|
|
93
112
|
const images = findEmbeddedImages(input.text);
|
|
94
113
|
if (images.length === 0)
|
|
95
|
-
return { text: input.text, imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0 };
|
|
114
|
+
return { text: input.text, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
|
|
96
115
|
const unique = new Map();
|
|
97
116
|
for (const image of images)
|
|
98
117
|
if (!unique.has(image.sha256))
|
|
99
118
|
unique.set(image.sha256, image);
|
|
100
|
-
const counts = { already: 0, described_now: 0, known: 0, too_large: 0, over_budget: 0, failed: 0, backing_off: 0 };
|
|
119
|
+
const counts = { settled_earlier: 0, already: 0, described_now: 0, known: 0, too_large: 0, over_budget: 0, failed: 0, backing_off: 0 };
|
|
101
120
|
const toSend = selectImagesToDescribe(run, unique, counts);
|
|
102
121
|
await sendPendingImages(run, toSend, counts, input);
|
|
103
122
|
const failedReasons = countImageOutcomes(run, toSend, counts);
|
|
@@ -105,15 +124,19 @@ async function describeAndReplace(run, input) {
|
|
|
105
124
|
await writeImageDescriptionState(options.stateDir, { described: orderedKeys, failed: failedImages });
|
|
106
125
|
run.stateDirty = false;
|
|
107
126
|
}
|
|
127
|
+
const deferredOnly = counts.over_budget + counts.backing_off > 0
|
|
128
|
+
&& counts.described_now + counts.known + counts.failed + counts.too_large + counts.settled_earlier === 0;
|
|
108
129
|
const replaced = mode === "describe_and_drop"
|
|
109
130
|
? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
|
|
110
131
|
: { text: input.text, replacedCount: 0, charsSaved: 0 };
|
|
132
|
+
const pendingImages = [...unique.keys()].filter((sha256) => !described.has(imageStateKey(sha256)));
|
|
111
133
|
console.error(TAG + " transcript prepared", JSON.stringify({
|
|
112
134
|
mode,
|
|
113
135
|
source: input.source,
|
|
114
136
|
images: images.length,
|
|
115
137
|
unique: unique.size,
|
|
116
138
|
...counts,
|
|
139
|
+
deferred_only: deferredOnly,
|
|
117
140
|
failed_reasons: failedReasons,
|
|
118
141
|
replaced: replaced.replacedCount,
|
|
119
142
|
chars_saved: replaced.charsSaved,
|
|
@@ -121,10 +144,12 @@ async function describeAndReplace(run, input) {
|
|
|
121
144
|
}));
|
|
122
145
|
return {
|
|
123
146
|
text: replaced.text,
|
|
147
|
+
pendingImages,
|
|
124
148
|
imageCount: images.length,
|
|
125
149
|
uniqueCount: unique.size,
|
|
126
150
|
replacedCount: replaced.replacedCount,
|
|
127
151
|
charsSaved: replaced.charsSaved,
|
|
152
|
+
deferredOnly,
|
|
128
153
|
};
|
|
129
154
|
}
|
|
130
155
|
function selectImagesToDescribe(run, unique, counts) {
|
|
@@ -136,8 +161,10 @@ function selectImagesToDescribe(run, unique, counts) {
|
|
|
136
161
|
continue;
|
|
137
162
|
}
|
|
138
163
|
const earlier = settledThisRun.get(image.sha256);
|
|
139
|
-
if (earlier)
|
|
164
|
+
if (earlier) {
|
|
165
|
+
counts.settled_earlier += 1;
|
|
140
166
|
continue;
|
|
167
|
+
}
|
|
141
168
|
if (image.byteSize > IMAGE_DESCRIBE_MAX_BYTES) {
|
|
142
169
|
settledThisRun.set(image.sha256, "too_large");
|
|
143
170
|
counts.too_large += 1;
|
|
@@ -153,12 +180,14 @@ function selectImagesToDescribe(run, unique, counts) {
|
|
|
153
180
|
return toSend;
|
|
154
181
|
}
|
|
155
182
|
async function sendPendingImages(run, toSend, counts, input) {
|
|
156
|
-
const { now, maxPerRun,
|
|
183
|
+
const { now, maxPerRun, maxRunMs } = run;
|
|
184
|
+
const callStartedAt = now();
|
|
157
185
|
let next = 0;
|
|
158
186
|
const worker = async () => {
|
|
159
187
|
while (next < toSend.length) {
|
|
160
188
|
const image = toSend[next++];
|
|
161
|
-
|
|
189
|
+
const timeUp = run.describingMs + (now() - callStartedAt) > maxRunMs;
|
|
190
|
+
if (run.sentThisRun >= maxPerRun || timeUp) {
|
|
162
191
|
counts.over_budget += 1;
|
|
163
192
|
continue;
|
|
164
193
|
}
|
|
@@ -166,6 +195,7 @@ async function sendPendingImages(run, toSend, counts, input) {
|
|
|
166
195
|
}
|
|
167
196
|
};
|
|
168
197
|
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));
|
|
198
|
+
run.describingMs += now() - callStartedAt;
|
|
169
199
|
}
|
|
170
200
|
function countImageOutcomes(run, toSend, counts) {
|
|
171
201
|
const { settledThisRun } = run;
|
|
@@ -25,6 +25,17 @@
|
|
|
25
25
|
import { newClaudeSlimState, slimClaudeRecord } from "./claude-rules.js";
|
|
26
26
|
import { newCodexSlimState, slimCodexRecord } from "./codex-rules.js";
|
|
27
27
|
import { LineEdits, SeenBlocks, isJsonObject } from "./slim-support.js";
|
|
28
|
+
/**
|
|
29
|
+
* The version of the slim rule set (claude-rules.ts, codex-rules.ts, claude-tool-results.ts,
|
|
30
|
+
* claude-attachments.ts, slim-support.ts and this file), BLI-22794. The local prepare memo keys what a
|
|
31
|
+
* transcript became on this value (plus the secret and image rules, see telemetry-core's
|
|
32
|
+
* `evidenceRulesFingerprint`) instead of on the CLI version.
|
|
33
|
+
*
|
|
34
|
+
* CHANGING ANY SLIM RULE MEANS BUMPING THIS CONSTANT. That rebuilds every collector's prepare memo
|
|
35
|
+
* once, which is intended: the stored bytes are no longer what the new rules would produce.
|
|
36
|
+
* `slim-rules-version.test.ts` fails when a rule changes without this moving.
|
|
37
|
+
*/
|
|
38
|
+
export const SLIM_RULES_VERSION = "slim-rules.v1";
|
|
28
39
|
/** The text an image marker starts with; a line carrying one is a modified line (an image became a marker). */
|
|
29
40
|
const IMAGE_MARKER_START = "[image sha256:";
|
|
30
41
|
export function createSlimmer(kind, rulesForTests) {
|
package/dist/sync-lock.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import fsSync from "node:fs";
|
|
2
3
|
import fs from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { describeError } from "./health-detail.js";
|
|
@@ -44,23 +45,25 @@ export async function acquireSyncLock(paths, now = new Date(), options = {}) {
|
|
|
44
45
|
marks: 0,
|
|
45
46
|
lastMarkMs: now.getTime(),
|
|
46
47
|
longestGapMs: 0,
|
|
48
|
+
lockPath,
|
|
49
|
+
token,
|
|
47
50
|
};
|
|
48
51
|
const existing = await readLock(lockPath);
|
|
49
|
-
if (existing &&
|
|
52
|
+
if (existing && isFreshAndAlive(existing, now)) {
|
|
50
53
|
logHeldByLiveOwner(existing, now);
|
|
51
54
|
return { acquired: false, held_since: existing.heartbeat_at };
|
|
52
55
|
}
|
|
53
56
|
// Free or stale: try an exclusive create first; if another process created it
|
|
54
57
|
// in the gap, re-check staleness before taking over.
|
|
55
|
-
let takeover = existing ?
|
|
58
|
+
let takeover = existing ? staleReason(existing) : null;
|
|
56
59
|
let owned = await tryExclusiveCreate(lockPath, token, now, held);
|
|
57
60
|
if (!owned) {
|
|
58
61
|
const current = await readLock(lockPath);
|
|
59
|
-
if (current &&
|
|
62
|
+
if (current && isFreshAndAlive(current, now)) {
|
|
60
63
|
logHeldByLiveOwner(current, now);
|
|
61
64
|
return { acquired: false, held_since: current.heartbeat_at };
|
|
62
65
|
}
|
|
63
|
-
takeover = current ?
|
|
66
|
+
takeover = current ? staleReason(current) : "unreadable";
|
|
64
67
|
await writeLock(lockPath, token, now, held); // take over the stale lock
|
|
65
68
|
owned = true;
|
|
66
69
|
}
|
|
@@ -68,6 +71,63 @@ export async function acquireSyncLock(paths, now = new Date(), options = {}) {
|
|
|
68
71
|
heldStates.add(held);
|
|
69
72
|
return maintainOwnedLock(lockPath, token, held);
|
|
70
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Does a process with this pid exist? `kill(pid, 0)` sends nothing: ESRCH means
|
|
76
|
+
* no such process, EPERM means it exists but is not ours (alive). Same on
|
|
77
|
+
* Windows. An unknown pid (-1 from an older record) is reported alive, so only
|
|
78
|
+
* the heartbeat decides for it.
|
|
79
|
+
*/
|
|
80
|
+
export function isProcessAlive(pid) {
|
|
81
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
82
|
+
return true;
|
|
83
|
+
try {
|
|
84
|
+
process.kill(pid, 0);
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
return error.code !== "ESRCH";
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/** Blocks only when the heartbeat is fresh AND the owner process still exists. */
|
|
92
|
+
function isFreshAndAlive(record, now) {
|
|
93
|
+
return now.getTime() - record.heartbeat_ms <= STALE_TAKEOVER_MS && isProcessAlive(record.pid);
|
|
94
|
+
}
|
|
95
|
+
/** A fresh heartbeat reaching here means the owner pid is gone; otherwise it aged out. */
|
|
96
|
+
function staleReason(record) {
|
|
97
|
+
return isProcessAlive(record.pid) ? "stale" : "dead";
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Synchronous release of every lock this process still holds, for the `exit`
|
|
101
|
+
* handler (BLI-22795): a process that ends without `release()` must not keep
|
|
102
|
+
* the next tick out until the heartbeat ages. Ownership-safe by token.
|
|
103
|
+
*/
|
|
104
|
+
export function releaseHeldSyncLocksOnExit() {
|
|
105
|
+
let released = 0;
|
|
106
|
+
for (const held of [...heldStates]) {
|
|
107
|
+
heldStates.delete(held);
|
|
108
|
+
let outcome = "not_ours_or_gone";
|
|
109
|
+
try {
|
|
110
|
+
if (held.lockPath && held.token) {
|
|
111
|
+
const raw = JSON.parse(fsSync.readFileSync(held.lockPath, "utf8"));
|
|
112
|
+
if (raw.token === held.token) {
|
|
113
|
+
fsSync.rmSync(held.lockPath, { force: true });
|
|
114
|
+
outcome = "removed";
|
|
115
|
+
released += 1;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
outcome = `release_failed_${error.code ?? "unknown"}`;
|
|
121
|
+
}
|
|
122
|
+
console.error("[sync-lock] lock released on exit", JSON.stringify({
|
|
123
|
+
reason: "holder_exited_holding_lock",
|
|
124
|
+
outcome,
|
|
125
|
+
role: held.role,
|
|
126
|
+
held_ms: Math.max(0, Date.now() - held.startedMs),
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
return released;
|
|
130
|
+
}
|
|
71
131
|
function maintainOwnedLock(lockPath, token, held) {
|
|
72
132
|
const timer = setInterval(() => {
|
|
73
133
|
// Refresh only while the lock is still ours; if we were taken over, stop
|
|
@@ -132,7 +192,13 @@ function logAcquired(held, takeover, previous, now) {
|
|
|
132
192
|
? "[sync-lock] lock acquired"
|
|
133
193
|
: "[sync-lock] took over a lock whose owner stopped answering", JSON.stringify({
|
|
134
194
|
role: held.role,
|
|
135
|
-
reason: takeover === "stale"
|
|
195
|
+
reason: takeover === "stale"
|
|
196
|
+
? "heartbeat_stale"
|
|
197
|
+
: takeover === "dead"
|
|
198
|
+
? "previous_owner_dead"
|
|
199
|
+
: takeover === "unreadable"
|
|
200
|
+
? "lock_file_unreadable"
|
|
201
|
+
: "lock_free",
|
|
136
202
|
previous_pid: previous && previous.pid > 0 ? previous.pid : null,
|
|
137
203
|
previous_heartbeat_age_ms: previous ? Math.max(0, now.getTime() - previous.heartbeat_ms) : null,
|
|
138
204
|
}));
|
|
@@ -222,7 +288,10 @@ export async function inspectSyncLock(paths, now = Date.now()) {
|
|
|
222
288
|
const record = await readLock(path.join(paths.state_dir, LOCK_FILENAME));
|
|
223
289
|
if (!record)
|
|
224
290
|
return null;
|
|
225
|
-
return {
|
|
291
|
+
return {
|
|
292
|
+
...record,
|
|
293
|
+
held: now - record.heartbeat_ms <= STALE_TAKEOVER_MS && isProcessAlive(record.pid),
|
|
294
|
+
};
|
|
226
295
|
}
|
|
227
296
|
/**
|
|
228
297
|
* Remove the lock file only if it still carries `token`: the holder this caller
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.252",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.74",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.155",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.81"
|
|
33
33
|
},
|
|
34
34
|
"repository": {
|
|
35
35
|
"type": "git",
|