@bli-cockpit/cli 0.2.262 → 0.2.264
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/agent-image-evidence.js +4 -4
- package/dist/adapters/raw-evidence-collection-state.js +14 -2
- package/dist/adapters/raw-evidence-sanitize-held.js +191 -0
- package/dist/adapters/raw-evidence-sanitize.js +10 -10
- package/dist/adapters/raw-evidence-transcript-body.js +69 -0
- package/dist/adapters/raw-evidence-transcript-file.js +11 -54
- package/dist/adapters/transcript-line-stream.js +47 -0
- package/dist/adapters/transcript-read-memo.js +1 -1
- package/dist/command-guard.js +202 -0
- package/dist/commands/backfill.js +6 -1
- package/dist/commands/cli-io.js +3 -1
- package/dist/commands/doctor-disk-words.js +5 -1
- package/dist/commands/doctor.js +9 -2
- package/dist/commands/heartbeat.js +40 -0
- package/dist/commands/local-args-collector-status.js +4 -0
- package/dist/commands/local-help-commands.js +8 -0
- package/dist/commands/local.js +2 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sessions-forget.js +62 -0
- package/dist/commands/sync-heartbeat.js +5 -1
- package/dist/commands/sync.js +9 -0
- package/dist/cursors/raw-evidence-cursor.js +1 -1
- package/dist/disk-usage-files.js +5 -5
- package/dist/evidence-upload-client.js +2 -1
- package/dist/evidence-upload-legacy.js +160 -0
- package/dist/evidence-upload-object.js +19 -10
- package/dist/evidence-upload-plan.js +20 -16
- package/dist/evidence-upload-staged.js +115 -0
- package/dist/evidence-upload-terminal.js +14 -152
- package/dist/http1-fetch.js +243 -0
- package/dist/image-description-session.js +27 -9
- package/dist/local-state-files.js +4 -2
- package/dist/raw-evidence-gc.js +5 -5
- package/dist/slim/slim-transcript.js +99 -22
- package/dist/spool/install-event-outbox.js +30 -0
- package/dist/staging-gate.js +6 -3
- package/dist/sync-tick-timing.js +4 -0
- package/package.json +4 -4
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import http from "node:http";
|
|
2
|
+
import https from "node:https";
|
|
3
|
+
import { Readable } from "node:stream";
|
|
4
|
+
import zlib from "node:zlib";
|
|
5
|
+
/**
|
|
6
|
+
* A fetch that can only speak HTTP/1.1 (BLI-23313).
|
|
7
|
+
*
|
|
8
|
+
* Node 26's built-in fetch (undici 8) negotiates HTTP/2 by default. One sync
|
|
9
|
+
* tick was caught at 55-69 GB (166 GB on a 16 GB Mac) of NATIVE memory, 100% of
|
|
10
|
+
* its main-thread sample inside `Http2Session::SendPendingData` growing a
|
|
11
|
+
* `std::vector<NgHttp2StreamWrite>`, while stuck in repeated evidence-upload
|
|
12
|
+
* deadline failures. That memory is off the JS heap, so no JS limit sees it.
|
|
13
|
+
* The memory daemon's external undici 6 defaults to HTTP/1.1 and stayed flat.
|
|
14
|
+
*
|
|
15
|
+
* `node:https` never offers `h2` in ALPN, so this transport removes the trigger
|
|
16
|
+
* instead of capping its symptom. It is fetch-shaped (`Request`/`URL`/string
|
|
17
|
+
* input, `Headers`, string/bytes/Blob/stream bodies, `AbortSignal`, redirects)
|
|
18
|
+
* and returns a standard `Response` whose body streams. It adds no dependency.
|
|
19
|
+
*
|
|
20
|
+
* Differences from the built-in, on purpose: no env-proxy support (the built-in
|
|
21
|
+
* had none either unless NODE_USE_ENV_PROXY was set), no request cache, and a
|
|
22
|
+
* `Accept-Encoding` header is only sent when the caller sets one.
|
|
23
|
+
*
|
|
24
|
+
* `COCKPIT_FETCH_TRANSPORT=native` leaves `globalThis.fetch` alone, an escape
|
|
25
|
+
* hatch if this transport ever misbehaves on a machine.
|
|
26
|
+
*/
|
|
27
|
+
const MAX_REDIRECTS = 20;
|
|
28
|
+
const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]);
|
|
29
|
+
const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 16 });
|
|
30
|
+
const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 16 });
|
|
31
|
+
function abortError(signal) {
|
|
32
|
+
const reason = signal?.reason;
|
|
33
|
+
if (reason instanceof Error)
|
|
34
|
+
return reason;
|
|
35
|
+
const error = new Error("This operation was aborted");
|
|
36
|
+
error.name = "AbortError";
|
|
37
|
+
return error;
|
|
38
|
+
}
|
|
39
|
+
function toHeaderRecord(headers) {
|
|
40
|
+
const record = {};
|
|
41
|
+
headers.forEach((value, key) => {
|
|
42
|
+
record[key] = value;
|
|
43
|
+
});
|
|
44
|
+
return record;
|
|
45
|
+
}
|
|
46
|
+
async function resolveBody(body, headers) {
|
|
47
|
+
if (body === null || body === undefined)
|
|
48
|
+
return { payload: null, length: null };
|
|
49
|
+
if (typeof body === "string") {
|
|
50
|
+
const bytes = Buffer.from(body, "utf8");
|
|
51
|
+
return { payload: bytes, length: bytes.byteLength };
|
|
52
|
+
}
|
|
53
|
+
if (body instanceof URLSearchParams) {
|
|
54
|
+
if (!headers.has("content-type")) {
|
|
55
|
+
headers.set("content-type", "application/x-www-form-urlencoded;charset=UTF-8");
|
|
56
|
+
}
|
|
57
|
+
const bytes = Buffer.from(body.toString(), "utf8");
|
|
58
|
+
return { payload: bytes, length: bytes.byteLength };
|
|
59
|
+
}
|
|
60
|
+
if (body instanceof ArrayBuffer)
|
|
61
|
+
return { payload: new Uint8Array(body), length: body.byteLength };
|
|
62
|
+
if (ArrayBuffer.isView(body)) {
|
|
63
|
+
const view = new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
|
|
64
|
+
return { payload: view, length: view.byteLength };
|
|
65
|
+
}
|
|
66
|
+
if (typeof Blob !== "undefined" && body instanceof Blob) {
|
|
67
|
+
if (body.type && !headers.has("content-type"))
|
|
68
|
+
headers.set("content-type", body.type);
|
|
69
|
+
return { payload: Readable.fromWeb(body.stream()), length: body.size };
|
|
70
|
+
}
|
|
71
|
+
if (typeof FormData !== "undefined" && body instanceof FormData) {
|
|
72
|
+
// Let the platform serialize the multipart boundary exactly as fetch would.
|
|
73
|
+
const wrapped = new Response(body);
|
|
74
|
+
const type = wrapped.headers.get("content-type");
|
|
75
|
+
if (type)
|
|
76
|
+
headers.set("content-type", type);
|
|
77
|
+
const bytes = Buffer.from(await wrapped.arrayBuffer());
|
|
78
|
+
return { payload: bytes, length: bytes.byteLength };
|
|
79
|
+
}
|
|
80
|
+
if (typeof body.getReader === "function") {
|
|
81
|
+
return { payload: Readable.fromWeb(body), length: null };
|
|
82
|
+
}
|
|
83
|
+
throw new TypeError("http1-fetch: unsupported request body type");
|
|
84
|
+
}
|
|
85
|
+
function decodeBody(res, headers) {
|
|
86
|
+
const encoding = (res.headers["content-encoding"] ?? "").toString().toLowerCase().trim();
|
|
87
|
+
let stream = res;
|
|
88
|
+
if (encoding === "gzip" || encoding === "x-gzip")
|
|
89
|
+
stream = res.pipe(zlib.createGunzip());
|
|
90
|
+
else if (encoding === "deflate")
|
|
91
|
+
stream = res.pipe(zlib.createInflate());
|
|
92
|
+
else if (encoding === "br")
|
|
93
|
+
stream = res.pipe(zlib.createBrotliDecompress());
|
|
94
|
+
else
|
|
95
|
+
return stream;
|
|
96
|
+
headers.delete("content-encoding");
|
|
97
|
+
headers.delete("content-length");
|
|
98
|
+
res.on("error", (error) => stream.destroy(error));
|
|
99
|
+
return stream;
|
|
100
|
+
}
|
|
101
|
+
function sendOnce(hop) {
|
|
102
|
+
return new Promise((resolve, reject) => {
|
|
103
|
+
const { url, signal } = hop;
|
|
104
|
+
if (signal?.aborted) {
|
|
105
|
+
reject(abortError(signal));
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const secure = url.protocol === "https:";
|
|
109
|
+
if (!secure && url.protocol !== "http:") {
|
|
110
|
+
reject(new TypeError(`http1-fetch: unsupported protocol ${url.protocol}`));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
const headers = toHeaderRecord(hop.headers);
|
|
114
|
+
if (hop.length !== null && hop.payload !== null && !("content-length" in headers)) {
|
|
115
|
+
headers["content-length"] = String(hop.length);
|
|
116
|
+
}
|
|
117
|
+
const request = (secure ? https : http).request({
|
|
118
|
+
protocol: url.protocol,
|
|
119
|
+
hostname: url.hostname.replace(/^\[|\]$/g, ""),
|
|
120
|
+
port: url.port || undefined,
|
|
121
|
+
path: `${url.pathname}${url.search}`,
|
|
122
|
+
method: hop.method,
|
|
123
|
+
headers,
|
|
124
|
+
agent: secure ? httpsAgent : httpAgent,
|
|
125
|
+
// Never negotiate h2: node:https only offers http/1.1 in ALPN.
|
|
126
|
+
...(secure ? { ALPNProtocols: ["http/1.1"] } : {}),
|
|
127
|
+
}, (response) => {
|
|
128
|
+
signal?.removeEventListener("abort", onAbort);
|
|
129
|
+
resolve(response);
|
|
130
|
+
});
|
|
131
|
+
const onAbort = () => {
|
|
132
|
+
const error = abortError(signal);
|
|
133
|
+
request.destroy(error);
|
|
134
|
+
reject(error);
|
|
135
|
+
};
|
|
136
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
137
|
+
request.on("error", (error) => {
|
|
138
|
+
signal?.removeEventListener("abort", onAbort);
|
|
139
|
+
reject(error);
|
|
140
|
+
});
|
|
141
|
+
const { payload } = hop;
|
|
142
|
+
if (payload === null)
|
|
143
|
+
request.end();
|
|
144
|
+
else if (payload instanceof Readable) {
|
|
145
|
+
payload.on("error", (error) => request.destroy(error));
|
|
146
|
+
payload.pipe(request);
|
|
147
|
+
}
|
|
148
|
+
else
|
|
149
|
+
request.end(payload);
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
function buildResponse(res, url, signal) {
|
|
153
|
+
const headers = new Headers();
|
|
154
|
+
for (const [name, value] of Object.entries(res.headers)) {
|
|
155
|
+
if (Array.isArray(value))
|
|
156
|
+
for (const item of value)
|
|
157
|
+
headers.append(name, item);
|
|
158
|
+
else if (value !== undefined)
|
|
159
|
+
headers.set(name, value);
|
|
160
|
+
}
|
|
161
|
+
const status = res.statusCode ?? 0;
|
|
162
|
+
const body = decodeBody(res, headers);
|
|
163
|
+
const onAbort = () => {
|
|
164
|
+
body.destroy(abortError(signal));
|
|
165
|
+
};
|
|
166
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
167
|
+
body.on("close", () => signal?.removeEventListener("abort", onAbort));
|
|
168
|
+
const response = new Response(NULL_BODY_STATUSES.has(status) ? null : Readable.toWeb(body), { status, statusText: res.statusMessage ?? "", headers });
|
|
169
|
+
Object.defineProperty(response, "url", { value: url });
|
|
170
|
+
// Test and diagnostic seam: which protocol actually carried this response.
|
|
171
|
+
Object.defineProperty(response, "httpVersion", { value: res.httpVersion });
|
|
172
|
+
if (NULL_BODY_STATUSES.has(status))
|
|
173
|
+
res.resume();
|
|
174
|
+
return response;
|
|
175
|
+
}
|
|
176
|
+
export async function http1Fetch(input, init = {}) {
|
|
177
|
+
const source = input instanceof Request ? input : null;
|
|
178
|
+
let url = new URL(source ? source.url : String(input));
|
|
179
|
+
const headers = new Headers(source?.headers);
|
|
180
|
+
new Headers(init.headers).forEach((value, key) => headers.set(key, value));
|
|
181
|
+
let method = (init.method ?? source?.method ?? "GET").toUpperCase();
|
|
182
|
+
const signal = init.signal ?? source?.signal ?? null;
|
|
183
|
+
const redirect = init.redirect ?? source?.redirect ?? "follow";
|
|
184
|
+
const rawBody = init.body ?? (source && source.body ? Buffer.from(await source.arrayBuffer()) : null);
|
|
185
|
+
let resolved = await resolveBody(rawBody, headers);
|
|
186
|
+
if (!headers.has("user-agent"))
|
|
187
|
+
headers.set("user-agent", "cockpit-collector");
|
|
188
|
+
for (let hops = 0;; hops += 1) {
|
|
189
|
+
const response = await sendOnce({ url, method, headers, ...resolved, signal });
|
|
190
|
+
const status = response.statusCode ?? 0;
|
|
191
|
+
const location = response.headers.location;
|
|
192
|
+
const redirects = [301, 302, 303, 307, 308].includes(status) && location;
|
|
193
|
+
if (!redirects || redirect === "manual")
|
|
194
|
+
return buildResponse(response, url.href, signal);
|
|
195
|
+
response.resume();
|
|
196
|
+
if (redirect === "error")
|
|
197
|
+
throw new TypeError("http1-fetch: redirect mode is error");
|
|
198
|
+
if (hops >= MAX_REDIRECTS)
|
|
199
|
+
throw new TypeError("http1-fetch: too many redirects");
|
|
200
|
+
const next = new URL(location, url);
|
|
201
|
+
if (next.origin !== url.origin) {
|
|
202
|
+
headers.delete("authorization");
|
|
203
|
+
headers.delete("cookie");
|
|
204
|
+
}
|
|
205
|
+
const dropBody = status === 303 || ((status === 301 || status === 302) && method === "POST");
|
|
206
|
+
if (dropBody) {
|
|
207
|
+
method = method === "HEAD" ? "HEAD" : "GET";
|
|
208
|
+
resolved = { payload: null, length: null };
|
|
209
|
+
headers.delete("content-type");
|
|
210
|
+
headers.delete("content-length");
|
|
211
|
+
}
|
|
212
|
+
else if (resolved.payload instanceof Readable) {
|
|
213
|
+
throw new TypeError("http1-fetch: cannot follow a redirect with a streamed body");
|
|
214
|
+
}
|
|
215
|
+
url = next;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
let installed = false;
|
|
219
|
+
/**
|
|
220
|
+
* Replace `globalThis.fetch` with the HTTP/1.1 transport. Idempotent; a no-op
|
|
221
|
+
* under `COCKPIT_FETCH_TRANSPORT=native`. Logs the decision either way.
|
|
222
|
+
*/
|
|
223
|
+
export function installHttp1GlobalFetch(env = process.env) {
|
|
224
|
+
if (installed)
|
|
225
|
+
return true;
|
|
226
|
+
if (env["COCKPIT_FETCH_TRANSPORT"] === "native") {
|
|
227
|
+
console.error("[http1-fetch] native fetch kept", JSON.stringify({ reason: "COCKPIT_FETCH_TRANSPORT=native" }));
|
|
228
|
+
return false;
|
|
229
|
+
}
|
|
230
|
+
if (globalThis.fetch.cockpitFetchGuard) {
|
|
231
|
+
// scripts/public-cli-fetch-guard.mjs: a smoke run that must fail on any network call.
|
|
232
|
+
console.error("[http1-fetch] fetch guard present; left in place", JSON.stringify({ reason: "cockpitFetchGuard" }));
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
globalThis.fetch = http1Fetch;
|
|
236
|
+
installed = true;
|
|
237
|
+
console.error("[http1-fetch] http/1.1 transport installed", JSON.stringify({ reason: "BLI-23313_h2_native_memory" }));
|
|
238
|
+
return true;
|
|
239
|
+
}
|
|
240
|
+
/** Test seam: forget the install (the caller restores `globalThis.fetch`). */
|
|
241
|
+
export function resetHttp1InstallForTest() {
|
|
242
|
+
installed = false;
|
|
243
|
+
}
|
|
@@ -5,10 +5,10 @@
|
|
|
5
5
|
* Failures never fail the transcript. Logs contain metadata, never image bytes
|
|
6
6
|
* or descriptions.
|
|
7
7
|
*/
|
|
8
|
-
import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
|
|
8
|
+
import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, formatImageMarker, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
|
|
9
9
|
import { postImageToDescribeDoor } from "./image-describe-client.js";
|
|
10
10
|
import { resolveImageHandlingMode } from "./image-mode-state.js";
|
|
11
|
-
import { slimOrKeep, slimTranscript } from "./slim/slim-transcript.js";
|
|
11
|
+
import { slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
|
|
12
12
|
import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
|
|
13
13
|
/** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
|
|
14
14
|
export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
|
|
@@ -39,10 +39,16 @@ export async function openImageDescriptions(options) {
|
|
|
39
39
|
isDescribed: (sha256) => described.has(imageStateKey(sha256)),
|
|
40
40
|
hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
|
|
41
41
|
async prepareTranscript(input) {
|
|
42
|
-
const afterImages = await describeAndReplace(run, input);
|
|
42
|
+
const { swap, ...afterImages } = await describeAndReplace(run, input);
|
|
43
43
|
if (mode !== "describe_and_drop")
|
|
44
44
|
return { ...afterImages, text: input.text, changed: false, slim: null };
|
|
45
|
-
|
|
45
|
+
// BLI-23317: the image swap and the slim rules are ONE pass over the lines (one joined string, no
|
|
46
|
+
// intermediate copy). A pass that cannot run falls back to the two-step order, which is the same bytes.
|
|
47
|
+
const pass = slimPassOrNull(input.text, input.source, { sessionId: input.sessionId }, swap);
|
|
48
|
+
if (pass)
|
|
49
|
+
return { ...afterImages, text: pass.text, changed: pass.changed, slim: pass.stats };
|
|
50
|
+
const replaced = replaceImagesWithMarkers(input.text, swap.images, swap.replace);
|
|
51
|
+
const slim = slimOrKeep(replaced.text, input.source, { sessionId: input.sessionId });
|
|
46
52
|
return { ...afterImages, text: slim.text, changed: slim.text !== input.text, slim: slim.stats };
|
|
47
53
|
},
|
|
48
54
|
};
|
|
@@ -110,8 +116,9 @@ function rememberFailure(run, key, outcome) {
|
|
|
110
116
|
async function describeAndReplace(run, input) {
|
|
111
117
|
const { mode, described, options, orderedKeys, failedImages } = run;
|
|
112
118
|
const images = findEmbeddedImages(input.text);
|
|
119
|
+
const swap = { images, replace: (image) => described.has(imageStateKey(image.sha256)) };
|
|
113
120
|
if (images.length === 0)
|
|
114
|
-
return {
|
|
121
|
+
return { swap, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
|
|
115
122
|
const unique = new Map();
|
|
116
123
|
for (const image of images)
|
|
117
124
|
if (!unique.has(image.sha256))
|
|
@@ -126,9 +133,7 @@ async function describeAndReplace(run, input) {
|
|
|
126
133
|
}
|
|
127
134
|
const deferredOnly = counts.over_budget + counts.backing_off > 0
|
|
128
135
|
&& counts.described_now + counts.known + counts.failed + counts.too_large + counts.settled_earlier === 0;
|
|
129
|
-
const replaced = mode === "describe_and_drop"
|
|
130
|
-
? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
|
|
131
|
-
: { text: input.text, replacedCount: 0, charsSaved: 0 };
|
|
136
|
+
const replaced = mode === "describe_and_drop" ? countImageSwaps(swap) : { replacedCount: 0, charsSaved: 0 };
|
|
132
137
|
const pendingImages = [...unique.keys()].filter((sha256) => !described.has(imageStateKey(sha256)));
|
|
133
138
|
console.error(TAG + " transcript prepared", JSON.stringify({
|
|
134
139
|
mode,
|
|
@@ -143,7 +148,7 @@ async function describeAndReplace(run, input) {
|
|
|
143
148
|
kept_without_description: mode === "describe_and_drop" ? images.length - replaced.replacedCount : 0,
|
|
144
149
|
}));
|
|
145
150
|
return {
|
|
146
|
-
|
|
151
|
+
swap,
|
|
147
152
|
pendingImages,
|
|
148
153
|
imageCount: images.length,
|
|
149
154
|
uniqueCount: unique.size,
|
|
@@ -152,6 +157,19 @@ async function describeAndReplace(run, input) {
|
|
|
152
157
|
deferredOnly,
|
|
153
158
|
};
|
|
154
159
|
}
|
|
160
|
+
/** What `replaceImagesWithMarkers` would report for this swap, without building the text it would make. */
|
|
161
|
+
function countImageSwaps(swap) {
|
|
162
|
+
let replacedCount = 0;
|
|
163
|
+
let charsSaved = 0;
|
|
164
|
+
for (const image of swap.images) {
|
|
165
|
+
if (!swap.replace(image))
|
|
166
|
+
continue;
|
|
167
|
+
const marker = formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
|
|
168
|
+
replacedCount += 1;
|
|
169
|
+
charsSaved += image.replaceEnd - image.replaceStart - marker.length;
|
|
170
|
+
}
|
|
171
|
+
return { replacedCount, charsSaved };
|
|
172
|
+
}
|
|
155
173
|
function selectImagesToDescribe(run, unique, counts) {
|
|
156
174
|
const { described, settledThisRun, failedImages, now } = run;
|
|
157
175
|
const toSend = [];
|
|
@@ -52,11 +52,13 @@ export async function writeJsonFile(filePath, value) {
|
|
|
52
52
|
* a reader (or a crash) never sees a half-written file (BLI-17273: a torn image-mode file made a slim
|
|
53
53
|
* machine forget it had ever run slim). The temp file is removed when the write or the rename fails.
|
|
54
54
|
*/
|
|
55
|
-
export async function writeJsonFileAtomic(filePath, value) {
|
|
55
|
+
export async function writeJsonFileAtomic(filePath, value, options = {}) {
|
|
56
56
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
57
57
|
const temp = `${filePath}.${process.pid}.${Math.random().toString(36).slice(2, 10)}.tmp`;
|
|
58
|
+
// Compact for the big memo files (BLI-23317): indentation was pure transient heap; readers take either.
|
|
59
|
+
const body = options.compact ? JSON.stringify(value) : JSON.stringify(value, null, 2);
|
|
58
60
|
try {
|
|
59
|
-
await fs.writeFile(temp, `${
|
|
61
|
+
await fs.writeFile(temp, `${body}\n`, { mode: 0o600 });
|
|
60
62
|
if (process.platform !== "win32") {
|
|
61
63
|
await fs.chmod(temp, 0o600).catch(() => undefined);
|
|
62
64
|
}
|
package/dist/raw-evidence-gc.js
CHANGED
|
@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { readRawEvidenceCursor } from "./cursors/raw-evidence-cursor.js";
|
|
5
5
|
import { readRawEvidenceStagingState } from "./raw-evidence-staging.js";
|
|
6
|
+
import { hashFileStreaming } from "./evidence-upload-staged.js";
|
|
6
7
|
const RAW_EVIDENCE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
7
8
|
/**
|
|
8
9
|
* A staging directory belongs to one in-flight collection pass. Anything this
|
|
@@ -284,14 +285,13 @@ async function inspectRawEvidenceDirForGc(dir, uploadedHashes) {
|
|
|
284
285
|
return { deletable: true, byteSize: 0 };
|
|
285
286
|
let byteSize = 0;
|
|
286
287
|
for (const file of files) {
|
|
287
|
-
const
|
|
288
|
-
if (!
|
|
288
|
+
const hashed = await hashFileStreaming(file);
|
|
289
|
+
if (!hashed)
|
|
289
290
|
return { deletable: false, byteSize: 0 };
|
|
290
|
-
|
|
291
|
-
if (!uploadedHashes.has(digest)) {
|
|
291
|
+
if (!uploadedHashes.has(hashed.digest)) {
|
|
292
292
|
return { deletable: false, byteSize: 0 };
|
|
293
293
|
}
|
|
294
|
-
byteSize +=
|
|
294
|
+
byteSize += hashed.byteSize;
|
|
295
295
|
}
|
|
296
296
|
return { deletable: true, byteSize };
|
|
297
297
|
}
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
* What each rule keeps and drops: claude-rules.ts and codex-rules.ts. The table for people:
|
|
23
23
|
* docs/architecture/image-descriptions.md.
|
|
24
24
|
*/
|
|
25
|
+
import { replaceImagesWithMarkers } from "@bli-cockpit/telemetry-core";
|
|
25
26
|
import { newClaudeSlimState, slimClaudeRecord } from "./claude-rules.js";
|
|
26
27
|
import { newCodexSlimState, slimCodexRecord } from "./codex-rules.js";
|
|
27
28
|
import { LineEdits, SeenBlocks, isJsonObject } from "./slim-support.js";
|
|
@@ -103,17 +104,89 @@ function slimOneLine(line, applyRules, stats) {
|
|
|
103
104
|
stats.linesModified += 1;
|
|
104
105
|
return JSON.stringify(record) + (line.endsWith("\r") ? "\r" : "");
|
|
105
106
|
}
|
|
106
|
-
/**
|
|
107
|
-
|
|
107
|
+
/**
|
|
108
|
+
* The slim form of a whole transcript's text, line for line, with the image swap folded into the
|
|
109
|
+
* same pass (BLI-23317). Output is byte-identical to `replaceImagesWithMarkers` followed by the old
|
|
110
|
+
* split/map/join: lines are found by `indexOf`, never `split`; a line no rule touched stays part of
|
|
111
|
+
* a slice of the input and is not copied on its own, so the only new allocation is the one joined
|
|
112
|
+
* result (and only when some line changed). An image sits inside one line (its characters exclude
|
|
113
|
+
* "\n"), so it is swapped inside that line before the rules read it, as the two-step order did.
|
|
114
|
+
*/
|
|
115
|
+
export function slimPass(text, kind, swap) {
|
|
108
116
|
const slimmer = createSlimmer(kind);
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
117
|
+
const pieces = [];
|
|
118
|
+
let emitted = 0;
|
|
119
|
+
let nextImage = 0;
|
|
120
|
+
let replacedCount = 0;
|
|
121
|
+
let charsSaved = 0;
|
|
122
|
+
for (let position = 0;;) {
|
|
123
|
+
const newline = text.indexOf("\n", position);
|
|
124
|
+
const end = newline === -1 ? text.length : newline;
|
|
125
|
+
let line = text.slice(position, end);
|
|
126
|
+
let swapped = false;
|
|
127
|
+
if (swap) {
|
|
128
|
+
const local = [];
|
|
129
|
+
while (nextImage < swap.images.length && swap.images[nextImage].replaceStart < end) {
|
|
130
|
+
const image = swap.images[nextImage];
|
|
131
|
+
nextImage += 1;
|
|
132
|
+
if (image.replaceStart < position || image.replaceEnd > end)
|
|
133
|
+
throw new Error("image_spans_lines");
|
|
134
|
+
local.push({
|
|
135
|
+
...image,
|
|
136
|
+
replaceStart: image.replaceStart - position,
|
|
137
|
+
replaceEnd: image.replaceEnd - position,
|
|
138
|
+
base64Start: image.base64Start - position,
|
|
139
|
+
base64End: image.base64End - position,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
if (local.length > 0) {
|
|
143
|
+
const replaced = replaceImagesWithMarkers(line, local, swap.replace);
|
|
144
|
+
if (replaced.replacedCount > 0) {
|
|
145
|
+
line = replaced.text;
|
|
146
|
+
swapped = true;
|
|
147
|
+
replacedCount += replaced.replacedCount;
|
|
148
|
+
charsSaved += replaced.charsSaved;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
const out = slimmer.slimLine(line);
|
|
153
|
+
if (swapped || out !== line) {
|
|
154
|
+
pieces.push(text.slice(emitted, position), out);
|
|
155
|
+
emitted = end;
|
|
156
|
+
}
|
|
157
|
+
if (newline === -1)
|
|
158
|
+
break;
|
|
159
|
+
position = newline + 1;
|
|
160
|
+
}
|
|
113
161
|
const stats = slimmer.stats();
|
|
114
|
-
|
|
162
|
+
if (pieces.length === 0)
|
|
163
|
+
return { text, stats, replacedCount, charsSaved, changed: false };
|
|
164
|
+
pieces.push(text.slice(emitted));
|
|
165
|
+
return { text: pieces.join(""), stats, replacedCount, charsSaved, changed: true };
|
|
166
|
+
}
|
|
167
|
+
/** The slim form of a whole transcript's text, line for line. */
|
|
168
|
+
export function slimTranscript(text, kind) {
|
|
169
|
+
const slim = slimPass(text, kind);
|
|
170
|
+
return { text: slim.text, stats: slim.stats };
|
|
115
171
|
}
|
|
116
172
|
const TAG = "[slim-transcript]";
|
|
173
|
+
function logSlimmed(kind, sessionId, stats, startedAt) {
|
|
174
|
+
console.error(TAG + (stats.linesModified === 0 ? " nothing to strip" : " transcript slimmed"), JSON.stringify({
|
|
175
|
+
source: kind,
|
|
176
|
+
session_id: sessionId.slice(0, 80),
|
|
177
|
+
lines: stats.lines,
|
|
178
|
+
lines_modified: stats.linesModified,
|
|
179
|
+
lines_failed: stats.linesFailed,
|
|
180
|
+
bytes_before: stats.bytesBefore,
|
|
181
|
+
bytes_after: stats.bytesAfter,
|
|
182
|
+
chars_by_rule: stats.charsByRule,
|
|
183
|
+
lines_by_rule: stats.linesByRule,
|
|
184
|
+
elapsed_ms: Date.now() - startedAt,
|
|
185
|
+
}));
|
|
186
|
+
}
|
|
187
|
+
function logSlimFailed(kind, error, bytes) {
|
|
188
|
+
console.error(TAG + " failed, uploading the transcript as it was", JSON.stringify({ reason: "slim_failed", source: kind, error_name: error instanceof Error ? error.name : typeof error, bytes }));
|
|
189
|
+
}
|
|
117
190
|
/**
|
|
118
191
|
* `slimTranscript` that can never fail a transcript: any exception returns the text it was given and
|
|
119
192
|
* logs the reason. Metadata only: counts, sizes, rule names. Never a line, never a field value.
|
|
@@ -122,23 +195,27 @@ export function slimOrKeep(text, kind, label) {
|
|
|
122
195
|
const startedAt = Date.now();
|
|
123
196
|
try {
|
|
124
197
|
const slim = slimTranscript(text, kind);
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
source: kind,
|
|
128
|
-
session_id: label.sessionId.slice(0, 80),
|
|
129
|
-
lines: stats.lines,
|
|
130
|
-
lines_modified: stats.linesModified,
|
|
131
|
-
lines_failed: stats.linesFailed,
|
|
132
|
-
bytes_before: stats.bytesBefore,
|
|
133
|
-
bytes_after: stats.bytesAfter,
|
|
134
|
-
chars_by_rule: stats.charsByRule,
|
|
135
|
-
lines_by_rule: stats.linesByRule,
|
|
136
|
-
elapsed_ms: Date.now() - startedAt,
|
|
137
|
-
}));
|
|
138
|
-
return { text: slim.text, stats };
|
|
198
|
+
logSlimmed(kind, label.sessionId, slim.stats, startedAt);
|
|
199
|
+
return { text: slim.text, stats: slim.stats };
|
|
139
200
|
}
|
|
140
201
|
catch (error) {
|
|
141
|
-
|
|
202
|
+
logSlimFailed(kind, error, typeof text === "string" ? text.length : 0);
|
|
142
203
|
return { text, stats: null };
|
|
143
204
|
}
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* `slimPass` that can never fail a transcript: a pass that throws (an image that would span lines,
|
|
208
|
+
* an exhausted string) logs the reason and returns null, and the caller takes the two-step path.
|
|
209
|
+
*/
|
|
210
|
+
export function slimPassOrNull(text, kind, label, swap) {
|
|
211
|
+
const startedAt = Date.now();
|
|
212
|
+
try {
|
|
213
|
+
const slim = slimPass(text, kind, swap);
|
|
214
|
+
logSlimmed(kind, label.sessionId, slim.stats, startedAt);
|
|
215
|
+
return slim;
|
|
216
|
+
}
|
|
217
|
+
catch (error) {
|
|
218
|
+
logSlimFailed(kind, error, text.length);
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
144
221
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import fsSync from "node:fs";
|
|
2
3
|
import fs from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { describeError } from "../health-detail.js";
|
|
@@ -36,6 +37,35 @@ export async function enqueueInstallEventEntry(paths, options) {
|
|
|
36
37
|
await pruneInstallEventOutbox(paths);
|
|
37
38
|
return entry;
|
|
38
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* Synchronous enqueue for a process that is about to exit (BLI-23313): the
|
|
42
|
+
* memory and wall-clock ceilings cannot await a write. Same entry shape the
|
|
43
|
+
* async path writes, so the next tick replays it like any other receipt.
|
|
44
|
+
*/
|
|
45
|
+
export function enqueueInstallEventEntrySyncForExit(paths, options) {
|
|
46
|
+
const createdAt = new Date().toISOString();
|
|
47
|
+
const outboxId = `install-event-${crypto.randomUUID()}`;
|
|
48
|
+
const entry = {
|
|
49
|
+
schema_version: "cockpit-install-event-outbox.v1",
|
|
50
|
+
outbox_id: outboxId,
|
|
51
|
+
created_at: createdAt,
|
|
52
|
+
last_attempt_at: null,
|
|
53
|
+
retry_count: 0,
|
|
54
|
+
last_failure_reason: null,
|
|
55
|
+
dashboard_url: options.dashboardUrl,
|
|
56
|
+
cli_version: options.cliVersion,
|
|
57
|
+
command: options.command,
|
|
58
|
+
os_platform: options.osPlatform,
|
|
59
|
+
events: options.events.map((event) => ({ ...event, at: event.at ?? createdAt })),
|
|
60
|
+
};
|
|
61
|
+
const directory = installEventOutboxDirectory(paths);
|
|
62
|
+
fsSync.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
63
|
+
const filePath = entryPath(paths, outboxId);
|
|
64
|
+
const tempPath = `${filePath}.tmp-${process.pid}`;
|
|
65
|
+
fsSync.writeFileSync(tempPath, `${JSON.stringify(entry, null, 2)}\n`, { mode: 0o600 });
|
|
66
|
+
fsSync.renameSync(tempPath, filePath);
|
|
67
|
+
return outboxId;
|
|
68
|
+
}
|
|
39
69
|
export async function readPendingInstallEventEntries(paths) {
|
|
40
70
|
const directory = installEventOutboxDirectory(paths);
|
|
41
71
|
const names = await fs.readdir(directory).catch(() => []);
|
package/dist/staging-gate.js
CHANGED
|
@@ -73,7 +73,10 @@ export function stagingGateLimitsFromEnv(env) {
|
|
|
73
73
|
* parent: a fresh machine has no state dir yet). `null` when the platform
|
|
74
74
|
* cannot say, which is logged and never closes the gate on its own.
|
|
75
75
|
*/
|
|
76
|
-
export async function readVolumeSpace(dir, statfs = fs.statfs
|
|
76
|
+
export async function readVolumeSpace(dir, statfs = fs.statfs,
|
|
77
|
+
// The host's own path rules; injectable so a Linux test can walk a
|
|
78
|
+
// `C:\Users\...` state dir up to its drive root exactly as Windows does.
|
|
79
|
+
pathApi = path) {
|
|
77
80
|
let target = dir;
|
|
78
81
|
for (let hops = 0; hops < 64; hops += 1) {
|
|
79
82
|
try {
|
|
@@ -85,7 +88,7 @@ export async function readVolumeSpace(dir, statfs = fs.statfs) {
|
|
|
85
88
|
};
|
|
86
89
|
}
|
|
87
90
|
catch (error) {
|
|
88
|
-
const parent =
|
|
91
|
+
const parent = pathApi.dirname(target);
|
|
89
92
|
if (parent === target) {
|
|
90
93
|
console.error("[staging-gate] free space unreadable, low-disk check skipped", JSON.stringify({ reason: "statfs_unavailable", ...describeError(error) }));
|
|
91
94
|
return null;
|
|
@@ -136,7 +139,7 @@ export async function readStagingGate(stateDir, env, deps = {}) {
|
|
|
136
139
|
const [stagingBytes, collectorBytes, volume] = await Promise.all([
|
|
137
140
|
directoryBytes(path.join(stateDir, "raw-evidence")).catch(() => 0),
|
|
138
141
|
measureCollectorBytes(stateDir).catch(() => 0),
|
|
139
|
-
readVolumeSpace(stateDir, deps.statfs),
|
|
142
|
+
readVolumeSpace(stateDir, deps.statfs, deps.pathApi),
|
|
140
143
|
]);
|
|
141
144
|
const facts = {
|
|
142
145
|
staging_bytes: stagingBytes,
|
package/dist/sync-tick-timing.js
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
export function startTickTimer(now = () => Date.now()) {
|
|
2
2
|
const started = now();
|
|
3
3
|
const stages = {};
|
|
4
|
+
let running = null;
|
|
4
5
|
const wrap = (name) => async (run) => {
|
|
5
6
|
const stageStarted = now();
|
|
7
|
+
running = name;
|
|
6
8
|
try {
|
|
7
9
|
return await run();
|
|
8
10
|
}
|
|
9
11
|
finally {
|
|
12
|
+
running = null;
|
|
10
13
|
stages[name] = (stages[name] ?? 0) + (now() - stageStarted);
|
|
11
14
|
}
|
|
12
15
|
};
|
|
@@ -28,6 +31,7 @@ export function startTickTimer(now = () => Date.now()) {
|
|
|
28
31
|
outcomeReport: wrap("outcome_report"),
|
|
29
32
|
converge: wrap("converge"),
|
|
30
33
|
prune: wrap("prune"),
|
|
34
|
+
current: () => running,
|
|
31
35
|
finish: (exitCode) => report(exitCode === 0 ? "ok" : "failed", exitCode, null),
|
|
32
36
|
threw: (error) => report("threw", null, error instanceof Error ? error.name : "unknown"),
|
|
33
37
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.264",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.78",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.162",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.85"
|
|
33
33
|
},
|
|
34
34
|
"repository": {
|
|
35
35
|
"type": "git",
|