@bli-cockpit/cli 0.2.50 → 0.2.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/clean.js +191 -0
- package/dist/commands/doctor.js +68 -0
- package/dist/commands/jarvis.js +101 -9
- package/dist/commands/local-args-collector.js +21 -0
- package/dist/commands/local-args.js +3 -1
- package/dist/commands/local-help.js +18 -0
- package/dist/commands/local.js +3 -0
- package/dist/commands/memory-install-contract.js +55 -0
- package/dist/commands/memory-install.js +113 -26
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-followups.js +95 -2
- package/dist/commands/sync.js +4 -1
- package/dist/disk-prune.js +246 -0
- package/dist/disk-retention.js +157 -0
- package/dist/disk-usage.js +337 -0
- package/dist/log-rotation.js +106 -2
- package/dist/tower-stream.js +5 -1
- package/package.json +2 -2
|
@@ -17,8 +17,10 @@
|
|
|
17
17
|
*
|
|
18
18
|
* - **Where the server is.** `bli-memory-mcp` ships as a DEPENDENCY of
|
|
19
19
|
* `@bli-cockpit/cli`, so the canonical lookup is the `node_modules/.bin` on
|
|
20
|
-
* the way up from
|
|
21
|
-
*
|
|
20
|
+
* the way up from THIS MODULE's own file — realpath-ed first, because a
|
|
21
|
+
* global `cockpit` is a symlink and walking up from the symlink finds
|
|
22
|
+
* nothing; PATH is the fallback. Nobody ever installs a second global
|
|
23
|
+
* package. See `resolveMemoryMcpBin`, which carries the receipt.
|
|
22
24
|
* - **No server, no write.** If the bin does not resolve, not one file is
|
|
23
25
|
* opened and the outcome is `skipped bin_missing` — see `no_bin_no_write`
|
|
24
26
|
* below for why a written-but-inert registration is the worse option.
|
|
@@ -32,10 +34,12 @@
|
|
|
32
34
|
* - **Never claim an install it did not read back** (BLI-2541). Both halves
|
|
33
35
|
* re-read and re-parse; this module only aggregates what they proved.
|
|
34
36
|
*/
|
|
37
|
+
import fs from "node:fs";
|
|
35
38
|
import os from "node:os";
|
|
36
39
|
import path from "node:path";
|
|
40
|
+
import { fileURLToPath } from "node:url";
|
|
37
41
|
import { writeLine } from "./cli-io.js";
|
|
38
|
-
import { builtinMemoryInstallConfig, isUnsafeBinPath, MEMORY_MCP_BIN, parsePrintedMemoryInstallConfig, } from "./memory-install-contract.js";
|
|
42
|
+
import { builtinMemoryInstallConfig, isUnsafeBinPath, MEMORY_MCP_BIN, parsePrintedMemoryInstallConfig, withResolvedBinPath, } from "./memory-install-contract.js";
|
|
39
43
|
import { defaultMemoryFileIo, } from "./memory-install-files.js";
|
|
40
44
|
import { installClaudeMemoryIntegration, inspectClaudeMemoryIntegration, } from "./memory-install-claude.js";
|
|
41
45
|
import { installCodexMemoryIntegration, inspectCodexMemoryIntegration, } from "./memory-install-codex.js";
|
|
@@ -140,6 +144,7 @@ async function resolveMemoryConfig(command, io, platform, deps) {
|
|
|
140
144
|
platform,
|
|
141
145
|
fileExists: deps.fileExists,
|
|
142
146
|
cliEntryPoint: deps.cliEntryPoint,
|
|
147
|
+
realpath: deps.realpath,
|
|
143
148
|
});
|
|
144
149
|
if (!found) {
|
|
145
150
|
return {
|
|
@@ -172,10 +177,35 @@ async function resolveMemoryConfig(command, io, platform, deps) {
|
|
|
172
177
|
}
|
|
173
178
|
const printed = await printedMemoryConfig(io, found.path);
|
|
174
179
|
if (printed) {
|
|
180
|
+
// The bin prints a BARE command name — it cannot know where it was
|
|
181
|
+
// installed, and it is nested inside the CLI's node_modules rather than on
|
|
182
|
+
// PATH. Path-qualifying it here is what makes the registration runnable at
|
|
183
|
+
// all; see `withResolvedBinPath`.
|
|
184
|
+
const qualified = withResolvedBinPath(printed, {
|
|
185
|
+
binPath: found.path,
|
|
186
|
+
platform,
|
|
187
|
+
dashboardUrl,
|
|
188
|
+
});
|
|
189
|
+
if (qualified) {
|
|
190
|
+
return {
|
|
191
|
+
config: qualified,
|
|
192
|
+
source: "bin",
|
|
193
|
+
binTarget: { target: "bin", status: "already", reason: "bin_printed_config" },
|
|
194
|
+
bin_source: found.source,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
// A printed hook command this installer cannot re-point. The template is
|
|
198
|
+
// always path-qualified, so it is the safe answer — and the reason says
|
|
199
|
+
// which of the two fallbacks happened.
|
|
175
200
|
return {
|
|
176
|
-
config:
|
|
177
|
-
source: "
|
|
178
|
-
binTarget: {
|
|
201
|
+
config: builtinMemoryInstallConfig({ binPath: found.path, platform, dashboardUrl }),
|
|
202
|
+
source: "template",
|
|
203
|
+
binTarget: {
|
|
204
|
+
target: "bin",
|
|
205
|
+
status: "already",
|
|
206
|
+
reason: "bin_printed_config_unqualifiable",
|
|
207
|
+
detail: "the bin printed a hook command this installer could not re-point at the resolved path; the built-in shape was used",
|
|
208
|
+
},
|
|
179
209
|
bin_source: found.source,
|
|
180
210
|
};
|
|
181
211
|
}
|
|
@@ -222,12 +252,11 @@ async function printedMemoryConfig(io, binPath) {
|
|
|
222
252
|
*
|
|
223
253
|
* **1. Beside the CLI that is running.** `bli-memory-mcp` ships as a DEPENDENCY
|
|
224
254
|
* of `@bli-cockpit/cli`, so installing the CLI installs the server, and npm
|
|
225
|
-
* links its bin into a `node_modules/.bin` on the path from this
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
* second package.
|
|
255
|
+
* links its bin into a `node_modules/.bin` on the path from this package up to
|
|
256
|
+
* the install root — `…/@bli-cockpit/cli/node_modules/.bin` when it is nested,
|
|
257
|
+
* `…/lib/node_modules/.bin` when npm hoists it. It is the ONLY lookup that
|
|
258
|
+
* cannot find somebody else's `bli-memory-mcp`. Nobody ever runs `npm i -g` for
|
|
259
|
+
* a second package.
|
|
231
260
|
*
|
|
232
261
|
* **2. PATH, as a fallback**, for a linked checkout or a hand-installed server.
|
|
233
262
|
* Done in this process rather than through `which`/`where`: it spawns nothing,
|
|
@@ -236,6 +265,26 @@ async function printedMemoryConfig(io, binPath) {
|
|
|
236
265
|
* standard layouts, which is why the caller passes a PATH that already includes
|
|
237
266
|
* it (`envWithNodeRuntimeOnPath`) — the launchd tick's PATH is otherwise
|
|
238
267
|
* `/usr/bin:/bin:/usr/sbin:/sbin` and would find nothing.
|
|
268
|
+
*
|
|
269
|
+
* **What step 1 got wrong on the first real machine** (BLI-3580, CLI 0.2.50):
|
|
270
|
+
* it walked up from `process.argv[1]`, resolved with `path.resolve` and no
|
|
271
|
+
* symlink following. For a global install `argv[1]` is the SHIM — on Edward's
|
|
272
|
+
* Mac `/opt/homebrew/bin/cockpit`, a symlink into
|
|
273
|
+
* `/opt/homebrew/lib/node_modules/@bli-cockpit/cli/dist/cli.js`. The walk
|
|
274
|
+
* therefore started in `/opt/homebrew/bin`, went up through `/opt` to `/`, and
|
|
275
|
+
* never came within reach of the nested `.bin` that was sitting right there.
|
|
276
|
+
* Every machine reported `skipped bin_missing` with the server installed.
|
|
277
|
+
*
|
|
278
|
+
* Two changes, both in `besideAnchors` / `resolveBesideCli`:
|
|
279
|
+
*
|
|
280
|
+
* - The anchor is **this module's own file** first (`import.meta.url`), which
|
|
281
|
+
* is inside the installed package by construction and is never a shim. The
|
|
282
|
+
* entry point (injected, or `argv[1]`) stays as a second anchor for a build
|
|
283
|
+
* layout where this module has been bundled somewhere else.
|
|
284
|
+
* - Every anchor is **realpath-ed** before the walk, so a symlinked entry
|
|
285
|
+
* lands in the real tree. A path that will not resolve is used as given —
|
|
286
|
+
* the same bounded silence `container-tag.ts` uses, because a path that
|
|
287
|
+
* cannot be realpath-ed still identifies a directory well enough to look in.
|
|
239
288
|
*/
|
|
240
289
|
export async function resolveMemoryMcpBin(options) {
|
|
241
290
|
const exists = options.fileExists ?? defaultFileExists;
|
|
@@ -245,28 +294,66 @@ export async function resolveMemoryMcpBin(options) {
|
|
|
245
294
|
const onPath = await resolveOnPath(options, exists);
|
|
246
295
|
return onPath ? { path: onPath, source: "path" } : null;
|
|
247
296
|
}
|
|
248
|
-
|
|
297
|
+
/**
|
|
298
|
+
* Where to start walking, in order of trustworthiness. This module's own
|
|
299
|
+
* location first: it is inside the installed package and cannot be a shim.
|
|
300
|
+
*/
|
|
301
|
+
function besideAnchors(options) {
|
|
302
|
+
const anchors = [];
|
|
303
|
+
const own = currentModulePath();
|
|
304
|
+
if (own)
|
|
305
|
+
anchors.push(own);
|
|
249
306
|
const entry = options.cliEntryPoint ?? process.argv[1];
|
|
250
|
-
if (
|
|
307
|
+
if (entry)
|
|
308
|
+
anchors.push(entry);
|
|
309
|
+
return anchors;
|
|
310
|
+
}
|
|
311
|
+
function currentModulePath() {
|
|
312
|
+
try {
|
|
313
|
+
return fileURLToPath(import.meta.url);
|
|
314
|
+
}
|
|
315
|
+
catch {
|
|
316
|
+
// A bundler that dropped `import.meta` support. The entry-point anchor
|
|
317
|
+
// still covers it, so this is a narrowing rather than a failure.
|
|
251
318
|
return null;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
async function resolveBesideCli(options, exists) {
|
|
252
322
|
const platformPath = options.platform === "win32" ? path.win32 : path.posix;
|
|
253
323
|
const extensions = binExtensions(options.platform);
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
324
|
+
const realpath = options.realpath ?? defaultRealpath;
|
|
325
|
+
for (const anchor of besideAnchors(options)) {
|
|
326
|
+
let directory = platformPath.dirname(realpath(platformPath.resolve(anchor)));
|
|
327
|
+
// Bounded walk: deep enough for `…/node_modules/@scope/pkg/dist/cli.js`
|
|
328
|
+
// plus a hoisted root above it, and it stops at the filesystem root anyway.
|
|
329
|
+
for (let depth = 0; depth < 12; depth += 1) {
|
|
330
|
+
for (const extension of extensions) {
|
|
331
|
+
const candidate = platformPath.join(directory, "node_modules", ".bin", `${MEMORY_MCP_BIN}${extension}`);
|
|
332
|
+
if (await exists(candidate))
|
|
333
|
+
return candidate;
|
|
334
|
+
}
|
|
335
|
+
const parent = platformPath.dirname(directory);
|
|
336
|
+
if (parent === directory)
|
|
337
|
+
break;
|
|
338
|
+
directory = parent;
|
|
262
339
|
}
|
|
263
|
-
const parent = platformPath.dirname(directory);
|
|
264
|
-
if (parent === directory)
|
|
265
|
-
break;
|
|
266
|
-
directory = parent;
|
|
267
340
|
}
|
|
268
341
|
return null;
|
|
269
342
|
}
|
|
343
|
+
/**
|
|
344
|
+
* `realpathSync.native` follows the symlink npm writes for a global bin. A
|
|
345
|
+
* path that does not resolve — a Windows path being reasoned about from a Mac
|
|
346
|
+
* in a test, a directory that has since moved — comes back unchanged rather
|
|
347
|
+
* than throwing, because the walk above can still look inside it.
|
|
348
|
+
*/
|
|
349
|
+
function defaultRealpath(value) {
|
|
350
|
+
try {
|
|
351
|
+
return fs.realpathSync.native(value);
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
return value;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
270
357
|
async function resolveOnPath(options, exists) {
|
|
271
358
|
// The TARGET platform's path rules, not the running one's. On a real machine
|
|
272
359
|
// they are the same; asking for them explicitly is what lets the Windows
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.52");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* What the sync tick does AFTER collection's own outcome is decided and
|
|
3
3
|
* reported: keep this machine's CLI current on npm `latest` (BLI-2601), put a
|
|
4
|
-
* broken scheduler registration back (BLI-2721),
|
|
5
|
-
*
|
|
4
|
+
* broken scheduler registration back (BLI-2721), keep BLI Memory registered
|
|
5
|
+
* with both agent hosts (BLI-3580), and stop the laptop filling up (BLI-3619).
|
|
6
6
|
*
|
|
7
7
|
* Split out of commands/sync.ts (BLI-3578), moved verbatim. They belong
|
|
8
8
|
* together because they share one rule, and it is the reason both are called
|
|
@@ -18,6 +18,7 @@ import { redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./insta
|
|
|
18
18
|
import { runSelfUpdate, SelfUpdateError } from "./install-update.js";
|
|
19
19
|
import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
|
|
20
20
|
import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
|
|
21
|
+
import { runStagingPrune } from "../disk-prune.js";
|
|
21
22
|
import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
|
|
22
23
|
/**
|
|
23
24
|
* BLI-2721: after the tick's collection and self-update are done and
|
|
@@ -183,6 +184,98 @@ function memoryInstallEvent(outcome) {
|
|
|
183
184
|
}
|
|
184
185
|
return { step: "memory_install", status: "ok", error_detail: detail };
|
|
185
186
|
}
|
|
187
|
+
/**
|
|
188
|
+
* BLI-3619: staged raw evidence Tower has already accepted stops living on the
|
|
189
|
+
* laptop forever.
|
|
190
|
+
*
|
|
191
|
+
* Same shape as the memory install above and for the same reasons: it runs only
|
|
192
|
+
* once collection's own outcome is decided and reported, at most once a day, it
|
|
193
|
+
* cannot throw, and its outcome is its own named receipt rather than a sync
|
|
194
|
+
* failure. A machine that cannot prune is a machine short of disk, and that
|
|
195
|
+
* must never also be a machine that stops collecting.
|
|
196
|
+
*
|
|
197
|
+
* The rule itself is `disk-retention.ts`: only objects the upload ledger
|
|
198
|
+
* vouches for are ever deleted, and anything undelivered is counted, aged and
|
|
199
|
+
* named instead.
|
|
200
|
+
*/
|
|
201
|
+
export async function runStagingPruneAfterSync(command, io, dashboardUrl, options = {}) {
|
|
202
|
+
let result;
|
|
203
|
+
try {
|
|
204
|
+
result = await runStagingPrune(getCollectorRuntimePaths(command.homeDir), {
|
|
205
|
+
env: io.env,
|
|
206
|
+
...(options.now ? { now: options.now } : {}),
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
catch (error) {
|
|
210
|
+
// runStagingPrune already catches everything it can reach; this is the
|
|
211
|
+
// last-resort net so a prune crash truly cannot touch the sync result.
|
|
212
|
+
result = { ...prunedNothing(), reason: "prune_threw", status: "fail" };
|
|
213
|
+
console.error("[collector prune] the prune follow-up threw", JSON.stringify({
|
|
214
|
+
reason: "prune_followup_threw",
|
|
215
|
+
detail: redactedSyncErrorDetail(error),
|
|
216
|
+
}));
|
|
217
|
+
}
|
|
218
|
+
// The daily throttle is the steady state — reporting it would post a receipt
|
|
219
|
+
// on 95 of every 96 ticks for no new information.
|
|
220
|
+
if (result.reason === "throttled_recent_run")
|
|
221
|
+
return;
|
|
222
|
+
await reportInstallEventsBestEffort({
|
|
223
|
+
homeDir: command.homeDir,
|
|
224
|
+
dashboardUrl,
|
|
225
|
+
command: "sync",
|
|
226
|
+
events: [stagingPruneEvent(result)],
|
|
227
|
+
json: command.json,
|
|
228
|
+
io,
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
function prunedNothing() {
|
|
232
|
+
return {
|
|
233
|
+
status: "skipped",
|
|
234
|
+
reason: "prune_threw",
|
|
235
|
+
deleted_files: 0,
|
|
236
|
+
deleted_bytes: 0,
|
|
237
|
+
removed_packs: 0,
|
|
238
|
+
kept_uncommitted: 0,
|
|
239
|
+
kept_uncommitted_bytes: 0,
|
|
240
|
+
oldest_uncommitted_age_ms: 0,
|
|
241
|
+
kept_unknown: 0,
|
|
242
|
+
kept_unknown_bytes: 0,
|
|
243
|
+
kept_in_window: 0,
|
|
244
|
+
cap_bytes: 0,
|
|
245
|
+
bytes_before: 0,
|
|
246
|
+
bytes_after: 0,
|
|
247
|
+
cap_blocked_by_uncommitted: false,
|
|
248
|
+
cap_blocked_count: 0,
|
|
249
|
+
failed_deletions: 0,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
/** Counts and byte totals only; no pack id and no path travels in a receipt. */
|
|
253
|
+
function stagingPruneEvent(result) {
|
|
254
|
+
const detail = [
|
|
255
|
+
`freed ${result.deleted_bytes}B in ${result.deleted_files} file(s)`,
|
|
256
|
+
`held ${result.kept_uncommitted_bytes}B uncommitted`,
|
|
257
|
+
result.cap_blocked_by_uncommitted
|
|
258
|
+
? `staging_cap_blocked_by_uncommitted ${result.cap_blocked_count}`
|
|
259
|
+
: null,
|
|
260
|
+
result.failed_deletions > 0
|
|
261
|
+
? `failed_deletions ${result.failed_deletions}`
|
|
262
|
+
: null,
|
|
263
|
+
]
|
|
264
|
+
.filter((part) => Boolean(part))
|
|
265
|
+
.join("; ");
|
|
266
|
+
if (result.status === "fail") {
|
|
267
|
+
return {
|
|
268
|
+
step: "staging_prune",
|
|
269
|
+
status: "fail",
|
|
270
|
+
error_code: result.reason,
|
|
271
|
+
error_detail: detail,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
if (result.status === "skipped") {
|
|
275
|
+
return { step: "staging_prune", status: "skipped", error_code: result.reason };
|
|
276
|
+
}
|
|
277
|
+
return { step: "staging_prune", status: "ok", error_detail: detail };
|
|
278
|
+
}
|
|
186
279
|
/**
|
|
187
280
|
* BLI-2601: the fleet keeps itself current on npm `latest` without anyone
|
|
188
281
|
* re-running `npm i -g @bli-cockpit/cli` by hand after day 0. This always
|
package/dist/commands/sync.js
CHANGED
|
@@ -5,7 +5,7 @@ import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
|
5
5
|
import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
|
|
6
6
|
import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
7
7
|
import { runAttributedWorktreeSync, } from "./session-sync.js";
|
|
8
|
-
import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, } from "./sync-followups.js";
|
|
8
|
+
import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
|
|
9
9
|
import { describeError } from "../health-detail.js";
|
|
10
10
|
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
11
11
|
import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
|
|
@@ -52,6 +52,9 @@ export async function runSync(command, io) {
|
|
|
52
52
|
// BLI-3580: BLI Memory's registration converges the same way — after
|
|
53
53
|
// collection, at most once a day, its own receipt either way.
|
|
54
54
|
await runMemoryInstallAfterSync(command, io, dashboardUrl);
|
|
55
|
+
// BLI-3619: and the disk stops growing without bound — same daily cadence,
|
|
56
|
+
// same rule that a follow-up never blocks or fails collection.
|
|
57
|
+
await runStagingPruneAfterSync(command, io, dashboardUrl);
|
|
55
58
|
return result.exitCode;
|
|
56
59
|
}
|
|
57
60
|
catch (error) {
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Carrying out the retention plan, and saying what it did.
|
|
3
|
+
*
|
|
4
|
+
* BLI-3619. The decision is `disk-retention.ts` and the reading is
|
|
5
|
+
* `disk-usage.ts`; this file is the only one that removes a staged byte. Three
|
|
6
|
+
* properties it must keep:
|
|
7
|
+
*
|
|
8
|
+
* - **It never fails the tick.** Every path is caught. A prune that cannot run
|
|
9
|
+
* is a log line, never a reason collection did not happen — the same rule the
|
|
10
|
+
* log rotation and the two self-heal follow-ups already live under.
|
|
11
|
+
* - **It never deletes what the ledger did not vouch for.** The plan decides
|
|
12
|
+
* that; this file does not second-guess it, and re-reads nothing.
|
|
13
|
+
* - **It says so on both branches.** `[collector prune]` fires on the boring
|
|
14
|
+
* run too, because "0 deleted, 2.1 GB held, all committed" is the only
|
|
15
|
+
* evidence that the drain is still running at all (the BLI-2528 lesson: what
|
|
16
|
+
* only speaks on failure cannot answer "did anything work today?").
|
|
17
|
+
*
|
|
18
|
+
* Throttled to once a day by a marker file, the same idiom the raw-evidence GC,
|
|
19
|
+
* the scheduled self-update and the BLI Memory install use. Walking a pack tree
|
|
20
|
+
* is cheap; doing it every fifteen minutes for a decision that changes on the
|
|
21
|
+
* scale of hours is not.
|
|
22
|
+
*/
|
|
23
|
+
import fs from "node:fs/promises";
|
|
24
|
+
import path from "node:path";
|
|
25
|
+
import { planStagingRetention, retentionOptionsFromEnv, } from "./disk-retention.js";
|
|
26
|
+
import { RAW_EVIDENCE_DIR, readStagingInventory } from "./disk-usage.js";
|
|
27
|
+
import { describeError } from "./health-detail.js";
|
|
28
|
+
import { readRawEvidenceStagingState, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
|
|
29
|
+
export const STAGING_PRUNE_THROTTLE_MARKER = ".last-staging-prune";
|
|
30
|
+
export const STAGING_PRUNE_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
31
|
+
/**
|
|
32
|
+
* The prune the sync tick runs and `cockpit clean` / `cockpit doctor` reuse.
|
|
33
|
+
* Never throws.
|
|
34
|
+
*/
|
|
35
|
+
export async function runStagingPrune(paths, options = {}) {
|
|
36
|
+
const env = options.env ?? process.env;
|
|
37
|
+
const now = options.now ?? new Date();
|
|
38
|
+
if (env["COCKPIT_DISABLE_GC"] === "1") {
|
|
39
|
+
return skippedResult("disabled");
|
|
40
|
+
}
|
|
41
|
+
if (!options.force && (await prunedRecently(paths, now))) {
|
|
42
|
+
return skippedResult("throttled_recent_run");
|
|
43
|
+
}
|
|
44
|
+
try {
|
|
45
|
+
if (!options.dryRun && !options.force)
|
|
46
|
+
await markPruned(paths, now);
|
|
47
|
+
const plan = await planFromDisk(paths, env, options, now);
|
|
48
|
+
const result = options.dryRun
|
|
49
|
+
? dryRunResult(plan)
|
|
50
|
+
: await applyPlan(paths, plan);
|
|
51
|
+
reportPrune(result, { dryRun: Boolean(options.dryRun) });
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
console.error("[collector prune] the prune could not run; nothing was deleted", JSON.stringify({ reason: "prune_threw", ...describeError(error) }));
|
|
56
|
+
return { ...skippedResult("prune_threw"), status: "fail" };
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
export async function planFromDisk(paths, env, options, now) {
|
|
60
|
+
const fromEnv = retentionOptionsFromEnv(env);
|
|
61
|
+
const inventory = await readStagingInventory(paths, now);
|
|
62
|
+
return planStagingRetention(inventory, {
|
|
63
|
+
retentionMs: options.retentionMs ?? fromEnv.retentionMs,
|
|
64
|
+
capBytes: options.capBytes ?? fromEnv.capBytes,
|
|
65
|
+
allCommitted: options.allCommitted,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
async function applyPlan(paths, plan) {
|
|
69
|
+
const root = path.join(paths.state_dir, RAW_EVIDENCE_DIR);
|
|
70
|
+
const tally = { files: 0, bytes: 0, packs: 0, failed: 0, hashes: [] };
|
|
71
|
+
const removedPackIds = await removeEmptyPacks(root, plan, tally);
|
|
72
|
+
await removeRemainingFiles(root, plan, removedPackIds, tally);
|
|
73
|
+
await forgetStagedObjects(paths, tally.hashes);
|
|
74
|
+
return {
|
|
75
|
+
...dryRunResult(plan),
|
|
76
|
+
status: "ok",
|
|
77
|
+
reason: tally.files > 0 ? "pruned" : "nothing_eligible",
|
|
78
|
+
deleted_files: tally.files,
|
|
79
|
+
deleted_bytes: tally.bytes,
|
|
80
|
+
removed_packs: tally.packs,
|
|
81
|
+
failed_deletions: tally.failed,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* A pack whose every payload file is going is removed whole, in one call, so
|
|
86
|
+
* `manifest.json` goes with it rather than surviving as a description of files
|
|
87
|
+
* that are not there.
|
|
88
|
+
*/
|
|
89
|
+
async function removeEmptyPacks(root, plan, tally) {
|
|
90
|
+
const removed = new Set();
|
|
91
|
+
for (const pack of plan.empty_packs) {
|
|
92
|
+
const gone = await rmQuietly(path.join(root, pack.pack_id), true);
|
|
93
|
+
if (!gone) {
|
|
94
|
+
tally.failed += 1;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
removed.add(pack.pack_id);
|
|
98
|
+
tally.packs += 1;
|
|
99
|
+
tally.bytes += pack.manifest_bytes;
|
|
100
|
+
}
|
|
101
|
+
return removed;
|
|
102
|
+
}
|
|
103
|
+
async function removeRemainingFiles(root, plan, removedPackIds, tally) {
|
|
104
|
+
const emptyPackIds = new Set(plan.empty_packs.map((pack) => pack.pack_id));
|
|
105
|
+
for (const entry of plan.delete) {
|
|
106
|
+
if (emptyPackIds.has(entry.pack_id)) {
|
|
107
|
+
// The pack removal above already took these bytes — or failed, in which
|
|
108
|
+
// case the file is still on disk and must be counted as such.
|
|
109
|
+
if (!removedPackIds.has(entry.pack_id))
|
|
110
|
+
tally.failed += 1;
|
|
111
|
+
else
|
|
112
|
+
countRemoved(tally, entry);
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
const file = path.join(root, entry.pack_id, ...entry.relative_path.split("/").filter(Boolean));
|
|
116
|
+
if (await rmQuietly(file, false))
|
|
117
|
+
countRemoved(tally, entry);
|
|
118
|
+
else
|
|
119
|
+
tally.failed += 1;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function countRemoved(tally, entry) {
|
|
123
|
+
tally.files += 1;
|
|
124
|
+
tally.bytes += entry.byte_size;
|
|
125
|
+
if (entry.content_hash)
|
|
126
|
+
tally.hashes.push(entry.content_hash);
|
|
127
|
+
}
|
|
128
|
+
async function rmQuietly(target, recursive) {
|
|
129
|
+
return fs.rm(target, { recursive, force: true }).then(() => true, () => false);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* The staged-object index points at bytes that are no longer there. Left alone
|
|
133
|
+
* it self-heals — `resolveStagedObject` stats the file and drops the entry —
|
|
134
|
+
* but the index is capped at 5,000 rows, so stale entries evict live ones and
|
|
135
|
+
* a real staged copy stops being found. One atomic write clears them together.
|
|
136
|
+
*/
|
|
137
|
+
async function forgetStagedObjects(paths, hashes) {
|
|
138
|
+
if (hashes.length === 0)
|
|
139
|
+
return;
|
|
140
|
+
try {
|
|
141
|
+
const staging = await readRawEvidenceStagingState(paths.state_dir);
|
|
142
|
+
let removed = 0;
|
|
143
|
+
for (const hash of hashes) {
|
|
144
|
+
if (staging.staged[hash]) {
|
|
145
|
+
delete staging.staged[hash];
|
|
146
|
+
removed += 1;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (removed === 0)
|
|
150
|
+
return;
|
|
151
|
+
staging.updated_at = new Date().toISOString();
|
|
152
|
+
await writeRawEvidenceStagingState(paths.state_dir, staging);
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
// Not fatal and not silent: the bytes are gone either way, and the index
|
|
156
|
+
// heals on the next read. What must not happen is nobody knowing the index
|
|
157
|
+
// is carrying rows for files that no longer exist.
|
|
158
|
+
console.error("[collector prune] staged index not updated after the prune", JSON.stringify({
|
|
159
|
+
reason: "staged_index_write_failed",
|
|
160
|
+
forgotten_count: hashes.length,
|
|
161
|
+
...describeError(error),
|
|
162
|
+
}));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
function dryRunResult(plan) {
|
|
166
|
+
return {
|
|
167
|
+
status: "ok",
|
|
168
|
+
reason: plan.delete.length > 0 ? "would_prune" : "nothing_eligible",
|
|
169
|
+
deleted_files: plan.delete.length,
|
|
170
|
+
deleted_bytes: plan.deleted_bytes + plan.empty_pack_manifest_bytes,
|
|
171
|
+
removed_packs: plan.empty_packs.length,
|
|
172
|
+
kept_uncommitted: plan.kept_uncommitted.count,
|
|
173
|
+
kept_uncommitted_bytes: plan.kept_uncommitted.bytes,
|
|
174
|
+
oldest_uncommitted_age_ms: plan.kept_uncommitted.oldest_age_ms,
|
|
175
|
+
kept_unknown: plan.kept_unknown.count,
|
|
176
|
+
kept_unknown_bytes: plan.kept_unknown.bytes,
|
|
177
|
+
kept_in_window: plan.kept_in_window.count,
|
|
178
|
+
cap_bytes: plan.cap_bytes,
|
|
179
|
+
bytes_before: plan.bytes_before,
|
|
180
|
+
bytes_after: plan.bytes_after,
|
|
181
|
+
cap_blocked_by_uncommitted: plan.cap_blocked_by_uncommitted,
|
|
182
|
+
cap_blocked_count: plan.cap_blocked_count,
|
|
183
|
+
failed_deletions: 0,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
function skippedResult(reason) {
|
|
187
|
+
return {
|
|
188
|
+
status: "skipped",
|
|
189
|
+
reason,
|
|
190
|
+
deleted_files: 0,
|
|
191
|
+
deleted_bytes: 0,
|
|
192
|
+
removed_packs: 0,
|
|
193
|
+
kept_uncommitted: 0,
|
|
194
|
+
kept_uncommitted_bytes: 0,
|
|
195
|
+
oldest_uncommitted_age_ms: 0,
|
|
196
|
+
kept_unknown: 0,
|
|
197
|
+
kept_unknown_bytes: 0,
|
|
198
|
+
kept_in_window: 0,
|
|
199
|
+
cap_bytes: 0,
|
|
200
|
+
bytes_before: 0,
|
|
201
|
+
bytes_after: 0,
|
|
202
|
+
cap_blocked_by_uncommitted: false,
|
|
203
|
+
cap_blocked_count: 0,
|
|
204
|
+
failed_deletions: 0,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* The receipt, on stderr, which launchd captures to `sync.err.log`. Counts,
|
|
209
|
+
* byte totals and reason labels only — never a pack id, never a path.
|
|
210
|
+
*/
|
|
211
|
+
export function reportPrune(result, options = { dryRun: false }) {
|
|
212
|
+
console.error(options.dryRun ? "[collector prune] dry run" : "[collector prune] swept", JSON.stringify({
|
|
213
|
+
reason: result.reason,
|
|
214
|
+
deleted_files: result.deleted_files,
|
|
215
|
+
deleted_bytes: result.deleted_bytes,
|
|
216
|
+
removed_packs: result.removed_packs,
|
|
217
|
+
kept_uncommitted: result.kept_uncommitted,
|
|
218
|
+
staging_uncommitted_bytes: result.kept_uncommitted_bytes,
|
|
219
|
+
oldest_uncommitted_age_ms: result.oldest_uncommitted_age_ms,
|
|
220
|
+
kept_unknown: result.kept_unknown,
|
|
221
|
+
kept_in_window: result.kept_in_window,
|
|
222
|
+
cap_bytes: result.cap_bytes,
|
|
223
|
+
bytes_after: result.bytes_after,
|
|
224
|
+
failed_deletions: result.failed_deletions,
|
|
225
|
+
}));
|
|
226
|
+
if (!result.cap_blocked_by_uncommitted)
|
|
227
|
+
return;
|
|
228
|
+
console.error("[collector prune] still over the cap and nothing committed is left to take", JSON.stringify({
|
|
229
|
+
reason: "staging_cap_blocked_by_uncommitted",
|
|
230
|
+
blocked_object_count: result.cap_blocked_count,
|
|
231
|
+
staging_uncommitted_bytes: result.kept_uncommitted_bytes,
|
|
232
|
+
unknown_bytes: result.kept_unknown_bytes,
|
|
233
|
+
cap_bytes: result.cap_bytes,
|
|
234
|
+
bytes_after: result.bytes_after,
|
|
235
|
+
}));
|
|
236
|
+
}
|
|
237
|
+
async function prunedRecently(paths, now) {
|
|
238
|
+
const marker = path.join(paths.state_dir, STAGING_PRUNE_THROTTLE_MARKER);
|
|
239
|
+
const info = await fs.stat(marker).catch(() => null);
|
|
240
|
+
return Boolean(info && now.getTime() - info.mtimeMs < STAGING_PRUNE_MIN_INTERVAL_MS);
|
|
241
|
+
}
|
|
242
|
+
async function markPruned(paths, now) {
|
|
243
|
+
const marker = path.join(paths.state_dir, STAGING_PRUNE_THROTTLE_MARKER);
|
|
244
|
+
await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
|
|
245
|
+
await fs.writeFile(marker, now.toISOString()).catch(() => undefined);
|
|
246
|
+
}
|