@bli-cockpit/cli 0.2.251 → 0.2.252
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-prepare-memo.js +34 -25
- package/dist/adapters/raw-evidence-transcript-file.js +22 -23
- package/dist/adapters/raw-evidence-volatile-reuse.js +27 -0
- package/dist/adapters/raw-evidence.js +1 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/crash-guard.js +9 -0
- package/dist/evidence-upload-deadline.js +36 -0
- package/dist/evidence-upload-direct.js +6 -5
- package/dist/evidence-upload-transport.js +7 -3
- package/dist/image-description-session.js +26 -7
- package/dist/slim/slim-transcript.js +11 -0
- package/dist/sync-lock.js +75 -6
- package/package.json +4 -4
|
@@ -13,11 +13,14 @@
|
|
|
13
13
|
*
|
|
14
14
|
* - the file's size, inode and mtime/ctime (nanoseconds) are exactly what they were when the
|
|
15
15
|
* entry was written, so the file has not been appended to, rewritten or replaced;
|
|
16
|
-
* - the recipe is the same:
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
16
|
+
* - the recipe is the same: a fingerprint of the rules that decide the bytes (the slim rules, the
|
|
17
|
+
* secret rules, the image marker format; NOT the CLI version, BLI-22794), the machine's
|
|
18
|
+
* image-handling mode, and whether an image step ran at all;
|
|
19
|
+
* - the entry is not `volatile`, or it is volatile ONLY because of images with no stored
|
|
20
|
+
* description (`pending_images`, BLI-22794). Such an entry is handed back to the caller, who
|
|
21
|
+
* trusts it while none of those images has been described since; a described one rebuilds. A
|
|
22
|
+
* volatile entry without `pending_images` (a failed slim step, or any older entry) is never
|
|
23
|
+
* recalled: it can change shape on the next tick without the file changing.
|
|
21
24
|
*
|
|
22
25
|
* The caller then reuses the entry ONLY when the remembered content hash is already durable
|
|
23
26
|
* remotely (`skipContentHashes`). Anything else, including every file that is not yet delivered,
|
|
@@ -25,11 +28,13 @@
|
|
|
25
28
|
* known. A memo that is lost, corrupt, or wrong in the safe direction costs one slow tick, never
|
|
26
29
|
* a missed upload. It holds ids, sizes, hashes and redaction metadata, never a byte of content.
|
|
27
30
|
*/
|
|
31
|
+
import { createHash } from "node:crypto";
|
|
28
32
|
import path from "node:path";
|
|
29
33
|
import fs from "node:fs/promises";
|
|
30
34
|
import { isMissingFileFailure } from "../health-detail.js";
|
|
31
35
|
import { readJsonFile, writeJsonFileAtomic } from "../local-state-files.js";
|
|
32
|
-
import {
|
|
36
|
+
import { evidenceRulesFingerprint } from "@bli-cockpit/telemetry-core";
|
|
37
|
+
import { SLIM_RULES_VERSION } from "../slim/slim-transcript.js";
|
|
33
38
|
export const PREPARE_MEMO_FILENAME = "transcript-prepare-memo.json";
|
|
34
39
|
const MEMO_FORMAT = "prepare-memo.v1";
|
|
35
40
|
/** About 700 bytes an entry, so this is at most a few MB; the oldest leave first. */
|
|
@@ -54,29 +59,38 @@ export async function statSourceForMemo(filePath) {
|
|
|
54
59
|
return null;
|
|
55
60
|
}
|
|
56
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* What the prepared bytes depend on besides the file: the slim rules and the secret and image rules.
|
|
64
|
+
* Computed once per process. NOT the CLI version (BLI-22794): an upgrade that changes none of these
|
|
65
|
+
* keeps the memo warm. Changing a rule changes this, which rebuilds the memo once; that is intended
|
|
66
|
+
* (see `SLIM_RULES_VERSION` and telemetry-core's `evidenceRulesFingerprint`).
|
|
67
|
+
*/
|
|
68
|
+
let rulesFingerprint;
|
|
69
|
+
export function rulesFingerprintOf(slimRulesVersion, evidenceRules) {
|
|
70
|
+
return createHash("sha256").update(`${slimRulesVersion}|${evidenceRules}`).digest("hex").slice(0, 32);
|
|
71
|
+
}
|
|
57
72
|
/** Everything besides the file that decides what its prepared bytes are. */
|
|
58
73
|
export function prepareRecipe(options) {
|
|
59
|
-
|
|
74
|
+
rulesFingerprint ??= rulesFingerprintOf(SLIM_RULES_VERSION, evidenceRulesFingerprint());
|
|
75
|
+
return [MEMO_FORMAT, rulesFingerprint, options.kind, options.imageMode ?? "no_image_step"].join("|");
|
|
60
76
|
}
|
|
61
77
|
export function prepareMemoPath(stateDir) {
|
|
62
78
|
return path.join(stateDir, PREPARE_MEMO_FILENAME);
|
|
63
79
|
}
|
|
64
|
-
const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0, volatile_reused_count: 0, volatile_reused_bytes: 0 };
|
|
80
|
+
const EMPTY_COUNTS = { skipped_count: 0, skipped_bytes: 0, rebuilt_count: 0, rebuilt_bytes: 0, volatile_count: 0, volatile_reused_count: 0, volatile_reused_bytes: 0, rebuilt_image_now_described_count: 0 };
|
|
65
81
|
export async function openPrepareMemo(stateDir) {
|
|
66
82
|
const entries = await readMemoEntries(stateDir);
|
|
67
83
|
let dirty = false;
|
|
68
84
|
let counts = { ...EMPTY_COUNTS };
|
|
69
85
|
/** Sources recalled since the last `settle`: only these have their `seen_ms` refreshed. */
|
|
70
86
|
const recalled = new Set();
|
|
71
|
-
/** In-memory only, never persisted: see `rememberForThisRun`. */
|
|
72
|
-
const thisRun = new Map();
|
|
73
87
|
return {
|
|
74
88
|
recall(sourceKey, stamp, recipe) {
|
|
75
89
|
const entry = entries.get(sourceKey);
|
|
76
90
|
if (!entry)
|
|
77
|
-
return
|
|
78
|
-
if (entry.volatile)
|
|
79
|
-
return
|
|
91
|
+
return null;
|
|
92
|
+
if (entry.volatile && !entry.pending_images)
|
|
93
|
+
return null;
|
|
80
94
|
if (entry.recipe !== recipe)
|
|
81
95
|
return null;
|
|
82
96
|
if (entry.size !== stamp.size || entry.identity !== stamp.identity)
|
|
@@ -86,13 +100,13 @@ export async function openPrepareMemo(stateDir) {
|
|
|
86
100
|
},
|
|
87
101
|
remember(sourceKey, entry, now = Date.now()) {
|
|
88
102
|
entries.set(sourceKey, { ...entry, seen_ms: now });
|
|
89
|
-
thisRun.delete(sourceKey);
|
|
90
103
|
dirty = true;
|
|
91
104
|
},
|
|
92
|
-
rememberForThisRun(sourceKey, entry) {
|
|
93
|
-
thisRun.set(sourceKey, { ...entry, volatile: true, seen_ms: Date.now() });
|
|
94
|
-
},
|
|
95
105
|
count(outcome, bytes) {
|
|
106
|
+
if (outcome === "rebuilt_image_now_described") {
|
|
107
|
+
counts.rebuilt_image_now_described_count += 1;
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
96
110
|
if (outcome === "volatile_reused") {
|
|
97
111
|
counts.volatile_reused_count += 1;
|
|
98
112
|
counts.volatile_reused_bytes += bytes;
|
|
@@ -127,14 +141,6 @@ export async function openPrepareMemo(stateDir) {
|
|
|
127
141
|
},
|
|
128
142
|
};
|
|
129
143
|
}
|
|
130
|
-
function recallThisRun(thisRun, sourceKey, stamp, recipe) {
|
|
131
|
-
const entry = thisRun.get(sourceKey);
|
|
132
|
-
if (!entry || entry.recipe !== recipe)
|
|
133
|
-
return null;
|
|
134
|
-
if (entry.size !== stamp.size || entry.identity !== stamp.identity)
|
|
135
|
-
return null;
|
|
136
|
-
return entry;
|
|
137
|
-
}
|
|
138
144
|
/**
|
|
139
145
|
* Entries recalled this pass keep their `seen_ms` fresh, lazily, so they are not forgotten while in
|
|
140
146
|
* use. Only those: an entry nobody recalls (a deleted or rotated session) ages out after
|
|
@@ -205,6 +211,9 @@ function parseEntry(value) {
|
|
|
205
211
|
return null;
|
|
206
212
|
if (typeof entry["volatile"] !== "boolean" || typeof entry["seen_ms"] !== "number")
|
|
207
213
|
return null;
|
|
214
|
+
const pending = entry["pending_images"];
|
|
215
|
+
if (pending !== undefined && (!Array.isArray(pending) || !pending.every((item) => typeof item === "string")))
|
|
216
|
+
return null;
|
|
208
217
|
const redacted = entry["redacted"];
|
|
209
218
|
if (redacted !== null && (typeof redacted !== "object" || typeof redacted.completenessLabel !== "string"))
|
|
210
219
|
return null;
|
|
@@ -26,6 +26,7 @@ import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
|
|
|
26
26
|
import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
|
|
27
27
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
28
28
|
import { prepareRecipe, statSourceForMemo, } from "./raw-evidence-prepare-memo.js";
|
|
29
|
+
import { logVolatileReuse, trustVolatileEntry } from "./raw-evidence-volatile-reuse.js";
|
|
29
30
|
import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
|
|
30
31
|
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
31
32
|
/**
|
|
@@ -63,9 +64,9 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
63
64
|
markCapApplied(collection, options.kind, "max_file_bytes");
|
|
64
65
|
return skip("file_too_large");
|
|
65
66
|
}
|
|
66
|
-
const { sanitized, contentHash, stubClaim, volatile,
|
|
67
|
+
const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
|
|
67
68
|
await rememberPreparedTranscript(collection, options, names, {
|
|
68
|
-
stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile,
|
|
69
|
+
stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
|
|
69
70
|
});
|
|
70
71
|
if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
|
|
71
72
|
collection.reused.push({
|
|
@@ -121,7 +122,7 @@ async function prepareTranscriptForUpload(collection, options, names, bytes) {
|
|
|
121
122
|
byteSize: sanitized.bytes.byteLength,
|
|
122
123
|
alreadyDurable: collection.skipContentHashes.has(contentHash),
|
|
123
124
|
});
|
|
124
|
-
return { sanitized, contentHash, stubClaim, volatile: images.volatile,
|
|
125
|
+
return { sanitized, contentHash, stubClaim, volatile: images.volatile, pendingImages: images.pendingImages, redactedEntry };
|
|
125
126
|
}
|
|
126
127
|
/**
|
|
127
128
|
* The transcript bytes after the image step. Unchanged (the very same buffer) unless this sync has
|
|
@@ -133,10 +134,10 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
133
134
|
const session = collection.budget.images;
|
|
134
135
|
const source = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
|
|
135
136
|
if (!session || !source)
|
|
136
|
-
return { bytes: file.bytes, volatile: false,
|
|
137
|
+
return { bytes: file.bytes, volatile: false, pendingImages: null };
|
|
137
138
|
// Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
|
|
138
139
|
if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.bytes))
|
|
139
|
-
return { bytes: file.bytes, volatile: false,
|
|
140
|
+
return { bytes: file.bytes, volatile: false, pendingImages: null };
|
|
140
141
|
try {
|
|
141
142
|
const prepared = await session.prepareTranscript({
|
|
142
143
|
text: file.bytes.toString("utf8"),
|
|
@@ -151,15 +152,16 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
151
152
|
: prepared.imageCount > 0;
|
|
152
153
|
const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
|
|
153
154
|
const volatile = imagesPending || slimFailed;
|
|
154
|
-
// BLI-
|
|
155
|
-
|
|
155
|
+
// BLI-22794: volatile ONLY because of images without a stored description (the slim step held
|
|
156
|
+
// up): the bytes change only when one of these gains one, so the memo may keep the entry.
|
|
157
|
+
const pendingImages = volatile && !slimFailed ? prepared.pendingImages : null;
|
|
156
158
|
if (!prepared.changed)
|
|
157
|
-
return { bytes: file.bytes, volatile,
|
|
158
|
-
return { bytes: Buffer.from(prepared.text, "utf8"), volatile,
|
|
159
|
+
return { bytes: file.bytes, volatile, pendingImages };
|
|
160
|
+
return { bytes: Buffer.from(prepared.text, "utf8"), volatile, pendingImages };
|
|
159
161
|
}
|
|
160
162
|
catch (error) {
|
|
161
163
|
console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
|
|
162
|
-
return { bytes: file.bytes, volatile: true,
|
|
164
|
+
return { bytes: file.bytes, volatile: true, pendingImages: null };
|
|
163
165
|
}
|
|
164
166
|
}
|
|
165
167
|
/**
|
|
@@ -182,6 +184,8 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
|
|
|
182
184
|
const known = memo.recall(names.sourceKey, stamp, recipe);
|
|
183
185
|
if (!known)
|
|
184
186
|
return false;
|
|
187
|
+
if (known.volatile && !trustVolatileEntry(collection, memo, known, options.kind, names.sourceKey, stamp.size))
|
|
188
|
+
return false;
|
|
185
189
|
if (!collection.skipContentHashes.has(known.content_hash)) {
|
|
186
190
|
if (known.volatile)
|
|
187
191
|
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "hash_not_durable_full_path");
|
|
@@ -202,24 +206,21 @@ async function reuseUnchangedTranscript(collection, options, names, stamp, recip
|
|
|
202
206
|
codex_session_id: options.sessionId,
|
|
203
207
|
});
|
|
204
208
|
if (known.volatile) {
|
|
205
|
-
// BLI-
|
|
209
|
+
// BLI-22794: only images without a description held it back, and none has one now.
|
|
206
210
|
memo.count("volatile_reused", stamp.size);
|
|
207
|
-
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "
|
|
211
|
+
logVolatileReuse(options.kind, names.sourceKey, known.content_hash, "pending_images_still_undescribed");
|
|
208
212
|
return true;
|
|
209
213
|
}
|
|
210
214
|
memo.count("skipped", stamp.size);
|
|
211
215
|
return true;
|
|
212
216
|
}
|
|
213
|
-
function logVolatileReuse(kind, sourceKey, contentHash, reason) {
|
|
214
|
-
console.error("[raw-evidence] volatile transcript " + (reason === "hash_not_durable_full_path" ? "rebuilt" : "reused this run"), JSON.stringify({ reason, kind, source_key: sourceKey, hash_prefix: contentHash.slice(0, 12) }));
|
|
215
|
-
}
|
|
216
217
|
/** Remember what this transcript became, but only if the file stayed put while it was built. */
|
|
217
218
|
async function rememberPreparedTranscript(collection, options, names, built) {
|
|
218
219
|
const memo = collection.budget.prepareMemo;
|
|
219
220
|
if (!memo)
|
|
220
221
|
return;
|
|
221
222
|
memo.count(built.volatile ? "volatile" : "rebuilt", built.readBytes);
|
|
222
|
-
if ((built.volatile && !built.
|
|
223
|
+
if ((built.volatile && !built.pendingImages) || !built.stamp)
|
|
223
224
|
return;
|
|
224
225
|
const after = await statSourceForMemo(options.filePath);
|
|
225
226
|
const stable = after !== null
|
|
@@ -237,15 +238,13 @@ async function rememberPreparedTranscript(collection, options, names, built) {
|
|
|
237
238
|
? { redaction: built.redactedEntry.redaction, completenessLabel: built.redactedEntry.completenessLabel }
|
|
238
239
|
: null,
|
|
239
240
|
};
|
|
240
|
-
if (built.volatile) {
|
|
241
|
-
// BLI-22785: in memory for this process only; the memo file keeps no volatile entry.
|
|
242
|
-
memo.rememberForThisRun(names.sourceKey, entry);
|
|
243
|
-
return;
|
|
244
|
-
}
|
|
245
241
|
const known = memo.recall(names.sourceKey, built.stamp, built.recipe);
|
|
246
|
-
|
|
242
|
+
const pending = built.volatile ? built.pendingImages : undefined;
|
|
243
|
+
if (known && known.volatile === built.volatile && known.content_hash === built.contentHash
|
|
244
|
+
&& (known.pending_images ?? []).join() === (pending ?? []).join())
|
|
247
245
|
return;
|
|
248
|
-
|
|
246
|
+
// BLI-22794: a volatile entry is kept across ticks with the images it is waiting on.
|
|
247
|
+
memo.remember(names.sourceKey, { ...entry, volatile: built.volatile, ...(pending ? { pending_images: pending } : {}) });
|
|
249
248
|
}
|
|
250
249
|
function nameEvidenceFile(options) {
|
|
251
250
|
const fileName = path.basename(options.filePath);
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLI-22794: may a remembered image-pending entry stand in for a rebuild? Not when one of its
|
|
3
|
+
* images has a stored description now (the bytes would differ, counted as
|
|
4
|
+
* `rebuilt_image_now_described_count`), and not while the describer could still take one of them
|
|
5
|
+
* this tick: a skipped transcript never reaches the describer, so reusing it would starve the
|
|
6
|
+
* images. Once the tick's budget is spent, the rest are reused until the next tick.
|
|
7
|
+
*/
|
|
8
|
+
export function trustVolatileEntry(collection, memo, known, kind, sourceKey, bytes) {
|
|
9
|
+
const session = collection.budget.images;
|
|
10
|
+
const pending = known.pending_images ?? [];
|
|
11
|
+
if (!session)
|
|
12
|
+
return false;
|
|
13
|
+
if (pending.some((sha256) => session.isDescribed(sha256))) {
|
|
14
|
+
memo.count("rebuilt_image_now_described", bytes);
|
|
15
|
+
logVolatileReuse(kind, sourceKey, known.content_hash, "rebuilt_image_now_described");
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
if (session.hasDescribeRoom(pending)) {
|
|
19
|
+
logVolatileReuse(kind, sourceKey, known.content_hash, "rebuilt_describer_has_room");
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
export function logVolatileReuse(kind, sourceKey, contentHash, reason) {
|
|
25
|
+
const reused = reason === "pending_images_still_undescribed";
|
|
26
|
+
console.error("[raw-evidence] volatile transcript " + (reused ? "reused across ticks" : "rebuilt"), JSON.stringify({ reason, kind, source_key: sourceKey, hash_prefix: contentHash.slice(0, 12) }));
|
|
27
|
+
}
|
|
@@ -118,6 +118,7 @@ async function reportPrepareMemo(collection) {
|
|
|
118
118
|
rebuilt_volatile_count: counts.volatile_count,
|
|
119
119
|
rebuilt_volatile_reused_count: counts.volatile_reused_count,
|
|
120
120
|
rebuilt_volatile_reused_bytes: counts.volatile_reused_bytes,
|
|
121
|
+
rebuilt_image_now_described_count: counts.rebuilt_image_now_described_count,
|
|
121
122
|
}));
|
|
122
123
|
}
|
|
123
124
|
/**
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.252");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
package/dist/crash-guard.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { releaseHeldSyncLocksOnExit } from "./sync-lock.js";
|
|
1
2
|
/**
|
|
2
3
|
* A run that dies must not be able to say it worked (BLI-4110).
|
|
3
4
|
*
|
|
@@ -125,6 +126,14 @@ export function installCrashGuard(options = {}) {
|
|
|
125
126
|
}));
|
|
126
127
|
});
|
|
127
128
|
target.on("exit", () => {
|
|
129
|
+
// A lock still held at exit is an orphan, whatever the exit code: release
|
|
130
|
+
// it synchronously so the next tick is not blocked (BLI-22795).
|
|
131
|
+
try {
|
|
132
|
+
releaseHeldSyncLocksOnExit();
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
console.error("[sync-lock] exit release threw", JSON.stringify({ reason: "exit_release_threw", ...describeCrash(error) }));
|
|
136
|
+
}
|
|
128
137
|
if (commandReturned)
|
|
129
138
|
return;
|
|
130
139
|
// `EXIT_DIED_MID_RUN` has to be in this set, and leaving it out made this
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A deadline that keeps the event loop alive (BLI-22795).
|
|
3
|
+
*
|
|
4
|
+
* `AbortSignal.timeout()` is documented to use an unref'd timer. A fetch whose
|
|
5
|
+
* socket has gone (network dropped mid-upload) then has nothing ref'd behind
|
|
6
|
+
* it: the loop drains, Node exits with the await still pending, and the
|
|
7
|
+
* crash guard logs `died_mid_run` with the collection lock still held. Here
|
|
8
|
+
* the timer is ref'd, and the race rejects even when an injected or broken
|
|
9
|
+
* fetch ignores the abort, so the caller always gets a named failure.
|
|
10
|
+
*/
|
|
11
|
+
export const UPLOAD_DEADLINE_ERROR_NAME = "EvidenceUploadDeadlineError";
|
|
12
|
+
export function isUploadDeadlineError(error) {
|
|
13
|
+
return error instanceof Error && error.name === UPLOAD_DEADLINE_ERROR_NAME;
|
|
14
|
+
}
|
|
15
|
+
export async function fetchWithDeadline(fetchImpl, url, init, deadlineMs) {
|
|
16
|
+
const controller = new AbortController();
|
|
17
|
+
let timer;
|
|
18
|
+
const deadline = new Promise((_, reject) => {
|
|
19
|
+
timer = setTimeout(() => {
|
|
20
|
+
const error = new Error("upload request exceeded its deadline");
|
|
21
|
+
error.name = UPLOAD_DEADLINE_ERROR_NAME;
|
|
22
|
+
controller.abort();
|
|
23
|
+
reject(error);
|
|
24
|
+
}, deadlineMs);
|
|
25
|
+
});
|
|
26
|
+
try {
|
|
27
|
+
const pending = fetchImpl(url, { ...init, signal: controller.signal });
|
|
28
|
+
// If the deadline wins, the abandoned fetch may reject later; that must not
|
|
29
|
+
// become an unhandled rejection.
|
|
30
|
+
pending.catch(() => undefined);
|
|
31
|
+
return await Promise.race([pending, deadline]);
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
clearTimeout(timer);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { encodeEvidenceBytes } from "@bli-cockpit/telemetry-core/evidence-storage-encoding";
|
|
2
|
+
import { fetchWithDeadline, isUploadDeadlineError } from "./evidence-upload-deadline.js";
|
|
2
3
|
import { noteSyncProgress } from "./sync-lock.js";
|
|
3
4
|
/** The signed capability is the only credential sent to Storage. Never log it,
|
|
4
5
|
* its query string, provider bodies, or thrown errors that might contain it. */
|
|
@@ -13,10 +14,10 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index)
|
|
|
13
14
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
14
15
|
let status = 0;
|
|
15
16
|
try {
|
|
16
|
-
const response = await options.fetchImpl
|
|
17
|
+
const response = await fetchWithDeadline(options.fetchImpl, signedUrl, {
|
|
17
18
|
method: "PUT", headers: { "Content-Type": "application/octet-stream", "x-upsert": "false" },
|
|
18
|
-
body: new Uint8Array(encoded),
|
|
19
|
-
});
|
|
19
|
+
body: new Uint8Array(encoded), redirect: "error",
|
|
20
|
+
}, 60_000);
|
|
20
21
|
status = response.status;
|
|
21
22
|
await response.body?.cancel();
|
|
22
23
|
if (response.ok) {
|
|
@@ -28,8 +29,8 @@ export async function putSignedChunk(options, signedUrl, bytes, uploadId, index)
|
|
|
28
29
|
if (status < 500 && status !== 429)
|
|
29
30
|
return false;
|
|
30
31
|
}
|
|
31
|
-
catch {
|
|
32
|
-
log("signed_upload_network_failure", uploadId, index, attempt, status);
|
|
32
|
+
catch (error) {
|
|
33
|
+
log(isUploadDeadlineError(error) ? "signed_upload_deadline_exceeded" : "signed_upload_network_failure", uploadId, index, attempt, status);
|
|
33
34
|
}
|
|
34
35
|
if (attempt < attempts)
|
|
35
36
|
await (options.sleep ?? sleep)(250 * attempt);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import { describeError } from "./health-detail.js";
|
|
3
|
+
import { fetchWithDeadline, isUploadDeadlineError } from "./evidence-upload-deadline.js";
|
|
3
4
|
import { noteSyncProgress } from "./sync-lock.js";
|
|
4
5
|
/**
|
|
5
6
|
* The `send` stage: one retrying POST to an evidence-upload route, plus the
|
|
@@ -10,6 +11,7 @@ import { noteSyncProgress } from "./sync-lock.js";
|
|
|
10
11
|
*/
|
|
11
12
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
12
13
|
const RETRY_DELAY_MS = 250;
|
|
14
|
+
const REQUEST_DEADLINE_MS = 120_000;
|
|
13
15
|
export async function requestJson(options, routePath, body) {
|
|
14
16
|
const maxAttempts = options.maxAttemptsPerRequest ?? DEFAULT_MAX_ATTEMPTS;
|
|
15
17
|
const sleep = options.sleep ?? defaultSleep;
|
|
@@ -17,7 +19,7 @@ export async function requestJson(options, routePath, body) {
|
|
|
17
19
|
let lastBody = null;
|
|
18
20
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
19
21
|
try {
|
|
20
|
-
const response = await options.fetchImpl
|
|
22
|
+
const response = await fetchWithDeadline(options.fetchImpl, `${options.dashboardUrl}${routePath}`, {
|
|
21
23
|
method: "POST",
|
|
22
24
|
headers: {
|
|
23
25
|
"Authorization": `Bearer ${options.deviceToken}`,
|
|
@@ -26,7 +28,7 @@ export async function requestJson(options, routePath, body) {
|
|
|
26
28
|
? { "x-tower-evidence-direct": "staged-gzip-v1" } : {}),
|
|
27
29
|
},
|
|
28
30
|
body: JSON.stringify(body),
|
|
29
|
-
});
|
|
31
|
+
}, REQUEST_DEADLINE_MS);
|
|
30
32
|
lastStatus = response.status;
|
|
31
33
|
lastBody = await readResponseJson(response);
|
|
32
34
|
if (response.ok || (response.status < 500 && response.status !== 429)) {
|
|
@@ -42,7 +44,9 @@ export async function requestJson(options, routePath, body) {
|
|
|
42
44
|
// the request left the machine. This is the exact silence BLI-2528 sat
|
|
43
45
|
// behind for 57 days; the route is named, the body never is.
|
|
44
46
|
console.error("[evidence-upload] request failed before a status came back", JSON.stringify({
|
|
45
|
-
reason:
|
|
47
|
+
reason: isUploadDeadlineError(error)
|
|
48
|
+
? "upload_request_deadline_exceeded"
|
|
49
|
+
: "upload_request_transport_error",
|
|
46
50
|
route: routePath,
|
|
47
51
|
attempt,
|
|
48
52
|
max_attempts: maxAttempts,
|
|
@@ -13,9 +13,10 @@ import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedIma
|
|
|
13
13
|
/** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
|
|
14
14
|
export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
|
|
15
15
|
/**
|
|
16
|
-
* After this long spent DESCRIBING
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* After this long spent DESCRIBING, remaining images wait for the next tick. Only time inside the
|
|
17
|
+
* sends counts: collection reaches image-bearing transcripts minutes into a tick and keeps slimming
|
|
18
|
+
* other transcripts between sends, and a wall clock from the session open (or from the first send)
|
|
19
|
+
* spent the budget on that work, so almost nothing was described (BLI-22789: 1 of 1,874, then 10).
|
|
19
20
|
*/
|
|
20
21
|
export const IMAGE_DESCRIBE_MAX_RUN_MS = 120_000;
|
|
21
22
|
const CONCURRENCY = 3;
|
|
@@ -32,9 +33,11 @@ export async function openImageDescriptions(options) {
|
|
|
32
33
|
const maxPerRun = options.maxPerRun ?? IMAGE_DESCRIBE_MAX_PER_RUN;
|
|
33
34
|
const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
|
|
34
35
|
const sentThisRun = 0;
|
|
35
|
-
const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now,
|
|
36
|
+
const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun };
|
|
36
37
|
return {
|
|
37
38
|
mode,
|
|
39
|
+
isDescribed: (sha256) => described.has(imageStateKey(sha256)),
|
|
40
|
+
hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
|
|
38
41
|
async prepareTranscript(input) {
|
|
39
42
|
const afterImages = await describeAndReplace(run, input);
|
|
40
43
|
if (mode !== "describe_and_drop")
|
|
@@ -44,6 +47,19 @@ export async function openImageDescriptions(options) {
|
|
|
44
47
|
},
|
|
45
48
|
};
|
|
46
49
|
}
|
|
50
|
+
function describeRoomLeft(run, sha256s) {
|
|
51
|
+
if (run.sentThisRun >= run.maxPerRun)
|
|
52
|
+
return false;
|
|
53
|
+
if (run.describingMs > run.maxRunMs)
|
|
54
|
+
return false;
|
|
55
|
+
return sha256s.some((sha256) => {
|
|
56
|
+
// Already sent (or found too large) this run: it will not be sent again before the next tick.
|
|
57
|
+
if (run.settledThisRun.has(sha256))
|
|
58
|
+
return false;
|
|
59
|
+
const failure = run.failedImages[imageStateKey(sha256)];
|
|
60
|
+
return !(failure && failure.retry_after > run.now());
|
|
61
|
+
});
|
|
62
|
+
}
|
|
47
63
|
function markDescribed(run, sha256) {
|
|
48
64
|
const { described, orderedKeys } = run;
|
|
49
65
|
const key = imageStateKey(sha256);
|
|
@@ -55,7 +71,6 @@ function markDescribed(run, sha256) {
|
|
|
55
71
|
async function sendOne(run, image, text, input) {
|
|
56
72
|
const { options, fetchImpl, settledThisRun, failedImages } = run;
|
|
57
73
|
run.sentThisRun += 1;
|
|
58
|
-
run.firstSendAt ??= run.now();
|
|
59
74
|
const outcome = await postImageToDescribeDoor({ fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, sleep: options.sleep }, {
|
|
60
75
|
schema_version: "image-describe.v1",
|
|
61
76
|
sha256: image.sha256,
|
|
@@ -96,7 +111,7 @@ async function describeAndReplace(run, input) {
|
|
|
96
111
|
const { mode, described, options, orderedKeys, failedImages } = run;
|
|
97
112
|
const images = findEmbeddedImages(input.text);
|
|
98
113
|
if (images.length === 0)
|
|
99
|
-
return { text: input.text, imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
|
|
114
|
+
return { text: input.text, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
|
|
100
115
|
const unique = new Map();
|
|
101
116
|
for (const image of images)
|
|
102
117
|
if (!unique.has(image.sha256))
|
|
@@ -114,6 +129,7 @@ async function describeAndReplace(run, input) {
|
|
|
114
129
|
const replaced = mode === "describe_and_drop"
|
|
115
130
|
? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
|
|
116
131
|
: { text: input.text, replacedCount: 0, charsSaved: 0 };
|
|
132
|
+
const pendingImages = [...unique.keys()].filter((sha256) => !described.has(imageStateKey(sha256)));
|
|
117
133
|
console.error(TAG + " transcript prepared", JSON.stringify({
|
|
118
134
|
mode,
|
|
119
135
|
source: input.source,
|
|
@@ -128,6 +144,7 @@ async function describeAndReplace(run, input) {
|
|
|
128
144
|
}));
|
|
129
145
|
return {
|
|
130
146
|
text: replaced.text,
|
|
147
|
+
pendingImages,
|
|
131
148
|
imageCount: images.length,
|
|
132
149
|
uniqueCount: unique.size,
|
|
133
150
|
replacedCount: replaced.replacedCount,
|
|
@@ -164,11 +181,12 @@ function selectImagesToDescribe(run, unique, counts) {
|
|
|
164
181
|
}
|
|
165
182
|
async function sendPendingImages(run, toSend, counts, input) {
|
|
166
183
|
const { now, maxPerRun, maxRunMs } = run;
|
|
184
|
+
const callStartedAt = now();
|
|
167
185
|
let next = 0;
|
|
168
186
|
const worker = async () => {
|
|
169
187
|
while (next < toSend.length) {
|
|
170
188
|
const image = toSend[next++];
|
|
171
|
-
const timeUp = run.
|
|
189
|
+
const timeUp = run.describingMs + (now() - callStartedAt) > maxRunMs;
|
|
172
190
|
if (run.sentThisRun >= maxPerRun || timeUp) {
|
|
173
191
|
counts.over_budget += 1;
|
|
174
192
|
continue;
|
|
@@ -177,6 +195,7 @@ async function sendPendingImages(run, toSend, counts, input) {
|
|
|
177
195
|
}
|
|
178
196
|
};
|
|
179
197
|
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));
|
|
198
|
+
run.describingMs += now() - callStartedAt;
|
|
180
199
|
}
|
|
181
200
|
function countImageOutcomes(run, toSend, counts) {
|
|
182
201
|
const { settledThisRun } = run;
|
|
@@ -25,6 +25,17 @@
|
|
|
25
25
|
import { newClaudeSlimState, slimClaudeRecord } from "./claude-rules.js";
|
|
26
26
|
import { newCodexSlimState, slimCodexRecord } from "./codex-rules.js";
|
|
27
27
|
import { LineEdits, SeenBlocks, isJsonObject } from "./slim-support.js";
|
|
28
|
+
/**
|
|
29
|
+
* The version of the slim rule set (claude-rules.ts, codex-rules.ts, claude-tool-results.ts,
|
|
30
|
+
* claude-attachments.ts, slim-support.ts and this file), BLI-22794. The local prepare memo keys what a
|
|
31
|
+
* transcript became on this value (plus the secret and image rules, see telemetry-core's
|
|
32
|
+
* `evidenceRulesFingerprint`) instead of on the CLI version.
|
|
33
|
+
*
|
|
34
|
+
* CHANGING ANY SLIM RULE MEANS BUMPING THIS CONSTANT. That rebuilds every collector's prepare memo
|
|
35
|
+
* once, which is intended: the stored bytes are no longer what the new rules would produce.
|
|
36
|
+
* `slim-rules-version.test.ts` fails when a rule changes without this moving.
|
|
37
|
+
*/
|
|
38
|
+
export const SLIM_RULES_VERSION = "slim-rules.v1";
|
|
28
39
|
/** The text an image marker starts with; a line carrying one is a modified line (an image became a marker). */
|
|
29
40
|
const IMAGE_MARKER_START = "[image sha256:";
|
|
30
41
|
export function createSlimmer(kind, rulesForTests) {
|
package/dist/sync-lock.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import fsSync from "node:fs";
|
|
2
3
|
import fs from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { describeError } from "./health-detail.js";
|
|
@@ -44,23 +45,25 @@ export async function acquireSyncLock(paths, now = new Date(), options = {}) {
|
|
|
44
45
|
marks: 0,
|
|
45
46
|
lastMarkMs: now.getTime(),
|
|
46
47
|
longestGapMs: 0,
|
|
48
|
+
lockPath,
|
|
49
|
+
token,
|
|
47
50
|
};
|
|
48
51
|
const existing = await readLock(lockPath);
|
|
49
|
-
if (existing &&
|
|
52
|
+
if (existing && isFreshAndAlive(existing, now)) {
|
|
50
53
|
logHeldByLiveOwner(existing, now);
|
|
51
54
|
return { acquired: false, held_since: existing.heartbeat_at };
|
|
52
55
|
}
|
|
53
56
|
// Free or stale: try an exclusive create first; if another process created it
|
|
54
57
|
// in the gap, re-check staleness before taking over.
|
|
55
|
-
let takeover = existing ?
|
|
58
|
+
let takeover = existing ? staleReason(existing) : null;
|
|
56
59
|
let owned = await tryExclusiveCreate(lockPath, token, now, held);
|
|
57
60
|
if (!owned) {
|
|
58
61
|
const current = await readLock(lockPath);
|
|
59
|
-
if (current &&
|
|
62
|
+
if (current && isFreshAndAlive(current, now)) {
|
|
60
63
|
logHeldByLiveOwner(current, now);
|
|
61
64
|
return { acquired: false, held_since: current.heartbeat_at };
|
|
62
65
|
}
|
|
63
|
-
takeover = current ?
|
|
66
|
+
takeover = current ? staleReason(current) : "unreadable";
|
|
64
67
|
await writeLock(lockPath, token, now, held); // take over the stale lock
|
|
65
68
|
owned = true;
|
|
66
69
|
}
|
|
@@ -68,6 +71,63 @@ export async function acquireSyncLock(paths, now = new Date(), options = {}) {
|
|
|
68
71
|
heldStates.add(held);
|
|
69
72
|
return maintainOwnedLock(lockPath, token, held);
|
|
70
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Does a process with this pid exist? `kill(pid, 0)` sends nothing: ESRCH means
|
|
76
|
+
* no such process, EPERM means it exists but is not ours (alive). Same on
|
|
77
|
+
* Windows. An unknown pid (-1 from an older record) is reported alive, so only
|
|
78
|
+
* the heartbeat decides for it.
|
|
79
|
+
*/
|
|
80
|
+
export function isProcessAlive(pid) {
|
|
81
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
82
|
+
return true;
|
|
83
|
+
try {
|
|
84
|
+
process.kill(pid, 0);
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
return error.code !== "ESRCH";
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/** Blocks only when the heartbeat is fresh AND the owner process still exists. */
|
|
92
|
+
function isFreshAndAlive(record, now) {
|
|
93
|
+
return now.getTime() - record.heartbeat_ms <= STALE_TAKEOVER_MS && isProcessAlive(record.pid);
|
|
94
|
+
}
|
|
95
|
+
/** A fresh heartbeat reaching here means the owner pid is gone; otherwise it aged out. */
|
|
96
|
+
function staleReason(record) {
|
|
97
|
+
return isProcessAlive(record.pid) ? "stale" : "dead";
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Synchronous release of every lock this process still holds, for the `exit`
|
|
101
|
+
* handler (BLI-22795): a process that ends without `release()` must not keep
|
|
102
|
+
* the next tick out until the heartbeat ages. Ownership-safe by token.
|
|
103
|
+
*/
|
|
104
|
+
export function releaseHeldSyncLocksOnExit() {
|
|
105
|
+
let released = 0;
|
|
106
|
+
for (const held of [...heldStates]) {
|
|
107
|
+
heldStates.delete(held);
|
|
108
|
+
let outcome = "not_ours_or_gone";
|
|
109
|
+
try {
|
|
110
|
+
if (held.lockPath && held.token) {
|
|
111
|
+
const raw = JSON.parse(fsSync.readFileSync(held.lockPath, "utf8"));
|
|
112
|
+
if (raw.token === held.token) {
|
|
113
|
+
fsSync.rmSync(held.lockPath, { force: true });
|
|
114
|
+
outcome = "removed";
|
|
115
|
+
released += 1;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
outcome = `release_failed_${error.code ?? "unknown"}`;
|
|
121
|
+
}
|
|
122
|
+
console.error("[sync-lock] lock released on exit", JSON.stringify({
|
|
123
|
+
reason: "holder_exited_holding_lock",
|
|
124
|
+
outcome,
|
|
125
|
+
role: held.role,
|
|
126
|
+
held_ms: Math.max(0, Date.now() - held.startedMs),
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
return released;
|
|
130
|
+
}
|
|
71
131
|
function maintainOwnedLock(lockPath, token, held) {
|
|
72
132
|
const timer = setInterval(() => {
|
|
73
133
|
// Refresh only while the lock is still ours; if we were taken over, stop
|
|
@@ -132,7 +192,13 @@ function logAcquired(held, takeover, previous, now) {
|
|
|
132
192
|
? "[sync-lock] lock acquired"
|
|
133
193
|
: "[sync-lock] took over a lock whose owner stopped answering", JSON.stringify({
|
|
134
194
|
role: held.role,
|
|
135
|
-
reason: takeover === "stale"
|
|
195
|
+
reason: takeover === "stale"
|
|
196
|
+
? "heartbeat_stale"
|
|
197
|
+
: takeover === "dead"
|
|
198
|
+
? "previous_owner_dead"
|
|
199
|
+
: takeover === "unreadable"
|
|
200
|
+
? "lock_file_unreadable"
|
|
201
|
+
: "lock_free",
|
|
136
202
|
previous_pid: previous && previous.pid > 0 ? previous.pid : null,
|
|
137
203
|
previous_heartbeat_age_ms: previous ? Math.max(0, now.getTime() - previous.heartbeat_ms) : null,
|
|
138
204
|
}));
|
|
@@ -222,7 +288,10 @@ export async function inspectSyncLock(paths, now = Date.now()) {
|
|
|
222
288
|
const record = await readLock(path.join(paths.state_dir, LOCK_FILENAME));
|
|
223
289
|
if (!record)
|
|
224
290
|
return null;
|
|
225
|
-
return {
|
|
291
|
+
return {
|
|
292
|
+
...record,
|
|
293
|
+
held: now - record.heartbeat_ms <= STALE_TAKEOVER_MS && isProcessAlive(record.pid),
|
|
294
|
+
};
|
|
226
295
|
}
|
|
227
296
|
/**
|
|
228
297
|
* Remove the lock file only if it still carries `token`: the holder this caller
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.252",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.74",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.155",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.81"
|
|
33
33
|
},
|
|
34
34
|
"repository": {
|
|
35
35
|
"type": "git",
|