@bli-cockpit/cli 0.2.154 → 0.2.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/clean.js +31 -13
- package/dist/commands/doctor-disk-words.js +81 -3
- package/dist/commands/doctor-pipeline.js +44 -22
- package/dist/commands/heartbeat.js +1 -1
- package/dist/commands/local-help-commands.js +7 -7
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/status-disk-line.js +22 -0
- package/dist/commands/status.js +5 -2
- package/dist/commands/sync-followups-staging.js +13 -10
- package/dist/commands/sync.js +7 -4
- package/dist/disk-prune.js +19 -23
- package/dist/disk-retention.js +12 -9
- package/dist/disk-usage-footprint.js +10 -4
- package/dist/evidence-reconcile-client.js +2 -2
- package/dist/log-rotation.js +31 -10
- package/dist/prune-receipt.js +87 -0
- package/dist/raw-evidence-gc.js +55 -15
- package/dist/state-dir-audit.js +138 -0
- package/package.json +1 -1
package/dist/commands/clean.js
CHANGED
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `cockpit clean` — what this machine is holding, why, and what goes.
|
|
3
3
|
*
|
|
4
|
-
* BLI-3619. The tick prunes on its own
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* BLI-3619 / BLI-11106. The sync tick prunes on its own, every tick, and a
|
|
5
|
+
* staged copy goes the tick its commit is acknowledged; nobody should need
|
|
6
|
+
* this door. It is here for "show me" and for a machine that has been off.
|
|
7
|
+
* It prints the same buckets the retention rule reasons in, one line each with
|
|
8
|
+
* its reason, and only then deletes:
|
|
7
9
|
*
|
|
8
|
-
* committed
|
|
9
|
-
* committed,
|
|
10
|
+
* committed on the server goes
|
|
11
|
+
* committed, inside an opted-in window kept — unless the cap needs it
|
|
12
|
+
* (only when COCKPIT_STAGING_RETENTION_HOURS is set)
|
|
10
13
|
* uncommitted kept, always, and named
|
|
11
14
|
* the ledger cannot say kept, always, and named apart
|
|
12
15
|
*
|
|
13
|
-
* `--dry-run` stops after the printing. `--all-committed` drops
|
|
16
|
+
* `--dry-run` stops after the printing. `--all-committed` drops any opted-in
|
|
14
17
|
* window — every committed spare goes now — and is also the only thing that
|
|
15
18
|
* removes a `manual-study-evidence-vault-*` directory, and then only when every
|
|
16
19
|
* object inside it is committed. Nothing here can delete evidence the upload
|
|
@@ -31,12 +34,13 @@ import crypto from "node:crypto";
|
|
|
31
34
|
import fs from "node:fs/promises";
|
|
32
35
|
import path from "node:path";
|
|
33
36
|
import { writeLine } from "./cli-io.js";
|
|
37
|
+
import { footprintLine } from "./doctor-disk-words.js";
|
|
34
38
|
import { planFromDisk, runStagingPrune } from "../disk-prune.js";
|
|
35
39
|
import { MANUAL_VAULT_PREFIX, readDiskFootprint, } from "../disk-usage.js";
|
|
36
40
|
import { readRawEvidenceCursor } from "../cursors/raw-evidence-cursor.js";
|
|
37
41
|
import { runEvidenceReconcile, } from "../evidence-reconcile-client.js";
|
|
38
42
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
|
|
39
|
-
import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
|
|
43
|
+
import { DEFAULT_LOG_ROTATION_KEEP, orphanedLogArchives, rotateCollectorLogsBestEffort, } from "../log-rotation.js";
|
|
40
44
|
export async function runClean(command, io) {
|
|
41
45
|
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
42
46
|
const now = new Date();
|
|
@@ -53,8 +57,9 @@ export async function runClean(command, io) {
|
|
|
53
57
|
const vaults = command.allCommitted
|
|
54
58
|
? await vaultsSafeToRemove(paths, footprint.vaults)
|
|
55
59
|
: [];
|
|
60
|
+
const logArchives = await orphanedLogArchives(paths);
|
|
56
61
|
if (!command.json) {
|
|
57
|
-
for (const line of cleanLines(footprint, plan, vaults, command)) {
|
|
62
|
+
for (const line of cleanLines(footprint, plan, vaults, command, logArchives)) {
|
|
58
63
|
writeLine(io.stdout, line);
|
|
59
64
|
}
|
|
60
65
|
}
|
|
@@ -67,7 +72,6 @@ export async function runClean(command, io) {
|
|
|
67
72
|
const pruned = await runStagingPrune(paths, {
|
|
68
73
|
env,
|
|
69
74
|
now,
|
|
70
|
-
force: true,
|
|
71
75
|
allCommitted: command.allCommitted,
|
|
72
76
|
reclaimTerminal: true,
|
|
73
77
|
});
|
|
@@ -86,20 +90,32 @@ export async function runClean(command, io) {
|
|
|
86
90
|
return pruned.status === "fail" ? 1 : 0;
|
|
87
91
|
}
|
|
88
92
|
/** The bucket table a person reads before anything is deleted. */
|
|
89
|
-
export function cleanLines(footprint, plan, vaults, command) {
|
|
93
|
+
export function cleanLines(footprint, plan, vaults, command, logArchives = []) {
|
|
90
94
|
const staging = footprint.staging;
|
|
91
95
|
const lines = [
|
|
92
96
|
command.dryRun ? "Tower clean (dry run)" : "Tower clean",
|
|
93
97
|
` staging ${mb(staging.total_bytes)} MB across ${staging.pack_count} pack(s), ${staging.object_count} file(s)`,
|
|
94
98
|
` going ${mb(plan.deleted_bytes + plan.empty_pack_manifest_bytes)} MB, ${plan.delete.length} file(s) — ${command.allCommitted
|
|
95
99
|
? "every committed copy (--all-committed)"
|
|
96
|
-
:
|
|
97
|
-
|
|
100
|
+
: plan.retention_ms > 0
|
|
101
|
+
? `committed and past the opted-in ${Math.round(plan.retention_ms / 3_600_000)} h window`
|
|
102
|
+
: "committed on the server; the source file is still on disk for any retry"}`,
|
|
103
|
+
` kept in window ${mb(plan.kept_in_window.bytes)} MB, ${plan.kept_in_window.count} file(s) — committed, inside an opted-in retry window (none by default)`,
|
|
98
104
|
` kept uncommitted ${mb(plan.kept_uncommitted.bytes)} MB, ${plan.kept_uncommitted.count} file(s) — never delivered; retention will never delete these`,
|
|
99
105
|
` kept unknown ${mb(plan.kept_unknown.bytes)} MB, ${plan.kept_unknown.count} file(s) — the upload ledger no longer reaches back this far and cannot say`,
|
|
100
|
-
` rotated logs ${mb(footprint.logs.total_bytes)} MB (${footprint.logs.archive_count} archive(s))
|
|
106
|
+
` rotated logs ${mb(footprint.logs.total_bytes)} MB (${footprint.logs.archive_count} archive(s))${logArchives.length > 0
|
|
107
|
+
? `; ${logArchives.length} archive(s), ${mb(logArchives.reduce((sum, archive) => sum + archive.byte_size, 0))} MB, sit outside the ${DEFAULT_LOG_ROTATION_KEEP}-archive keep window and go`
|
|
108
|
+
: ""}`,
|
|
101
109
|
` spool ${mb(footprint.spool_bytes)} MB`,
|
|
110
|
+
` state files ${mb(footprint.state.state_bytes)} MB — cursors, indexes and receipts, each bounded`,
|
|
102
111
|
];
|
|
112
|
+
if (footprint.state.other_bytes > 0) {
|
|
113
|
+
lines.push(` unrecognised ${mb(footprint.state.other_bytes)} MB across ${footprint.state.other_entries} entr(ies) — counted, never deleted by clean`);
|
|
114
|
+
}
|
|
115
|
+
if (footprint.state.study_entries > 0) {
|
|
116
|
+
lines.push(` studies ${footprint.state.study_entries} folder(s) an agent left here — not the collector's, not counted, never touched; new ones go in bli-cockpit-studies`);
|
|
117
|
+
}
|
|
118
|
+
lines.push(` ${footprintLine(footprint)}`);
|
|
103
119
|
for (const vault of footprint.vaults) {
|
|
104
120
|
const removable = vaults.some((entry) => entry.name === vault.name);
|
|
105
121
|
lines.push(` ${vault.name} ${mb(vault.byte_size)} MB, ${vault.file_count} file(s) — ${removable
|
|
@@ -158,6 +174,8 @@ function cleanJson(footprint, plan, vaults, pruned, reconciled = null) {
|
|
|
158
174
|
},
|
|
159
175
|
rotated_log_bytes: footprint.logs.total_bytes,
|
|
160
176
|
spool_bytes: footprint.spool_bytes,
|
|
177
|
+
state: footprint.state,
|
|
178
|
+
last_prune: footprint.last_prune,
|
|
161
179
|
vaults: footprint.vaults,
|
|
162
180
|
plan: {
|
|
163
181
|
delete_count: plan.delete.length,
|
|
@@ -28,9 +28,85 @@ export function redeliveryLine(result) {
|
|
|
28
28
|
}
|
|
29
29
|
return "the next `cockpit sync` re-offers them";
|
|
30
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Over this, the whole collector footprint is a needs-fix row (BLI-11106).
|
|
33
|
+
* Delete-on-commit plus the 256 MiB staging cap and the log caps put a healthy
|
|
34
|
+
* machine far under it; crossing it means something is holding bytes, and the
|
|
35
|
+
* row names what.
|
|
36
|
+
*/
|
|
37
|
+
export const FOOTPRINT_NEEDS_FIX_BYTES = 512 * 1024 * 1024;
|
|
38
|
+
/**
|
|
39
|
+
* Every bucket the collector owns, biggest first. Study folders an agent left
|
|
40
|
+
* under the state dir are not here: they are not the collector's, and no
|
|
41
|
+
* prune would ever explain them (they belong in `bli-cockpit-studies`).
|
|
42
|
+
*/
|
|
43
|
+
export function footprintBuckets(footprint) {
|
|
44
|
+
const staging = footprint.staging;
|
|
45
|
+
const buckets = [
|
|
46
|
+
{ label: "staging, committed (goes on the next tick)", bytes: staging.committed_bytes },
|
|
47
|
+
{
|
|
48
|
+
label: "staging, uncommitted (evidence the server has not accepted yet)",
|
|
49
|
+
bytes: staging.uncommitted_bytes + staging.unknown_bytes,
|
|
50
|
+
},
|
|
51
|
+
{ label: "staging, refused by the server", bytes: staging.terminal_bytes ?? 0 },
|
|
52
|
+
{ label: "staging, pack manifests", bytes: stagingOverheadBytes(footprint) },
|
|
53
|
+
{ label: "sync logs", bytes: footprint.logs.total_bytes },
|
|
54
|
+
{ label: "collector state files", bytes: footprint.state.state_bytes },
|
|
55
|
+
{ label: "upload spool", bytes: footprint.spool_bytes },
|
|
56
|
+
{ label: "unrecognised files in the state dir", bytes: footprint.state.other_bytes },
|
|
57
|
+
];
|
|
58
|
+
return buckets.filter((bucket) => bucket.bytes > 0).sort((a, b) => b.bytes - a.bytes);
|
|
59
|
+
}
|
|
60
|
+
export function footprintTotalBytes(footprint) {
|
|
61
|
+
return footprintBuckets(footprint).reduce((sum, bucket) => sum + bucket.bytes, 0);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* The one footprint line doctor and status both print: staging committed /
|
|
65
|
+
* uncommitted, logs, state, and when the prune last ran and what it freed.
|
|
66
|
+
*/
|
|
67
|
+
export function footprintLine(footprint) {
|
|
68
|
+
const staging = footprint.staging;
|
|
69
|
+
const parts = [
|
|
70
|
+
`footprint ${mib(footprintTotalBytes(footprint))} MB`,
|
|
71
|
+
`staging ${mib(staging.committed_bytes)} MB committed / ${mib(staging.uncommitted_bytes + staging.unknown_bytes)} MB uncommitted`,
|
|
72
|
+
`logs ${mib(footprint.logs.total_bytes)} MB`,
|
|
73
|
+
`state ${mib(footprint.state.state_bytes + footprint.spool_bytes)} MB`,
|
|
74
|
+
];
|
|
75
|
+
if (footprint.state.other_bytes > 0) {
|
|
76
|
+
parts.push(`unrecognised ${mib(footprint.state.other_bytes)} MB`);
|
|
77
|
+
}
|
|
78
|
+
parts.push(lastPrunePhrase(footprint));
|
|
79
|
+
return parts.join(", ");
|
|
80
|
+
}
|
|
81
|
+
/** `last prune: <when>, freed <n> MB`, plus the last time it freed anything. */
|
|
82
|
+
export function lastPrunePhrase(footprint) {
|
|
83
|
+
const last = footprint.last_prune;
|
|
84
|
+
if (!last)
|
|
85
|
+
return "last prune: never";
|
|
86
|
+
const base = `last prune: ${minute(last.at)}, freed ${mib(last.freed_bytes)} MB`;
|
|
87
|
+
if (last.freed_bytes > 0 || !last.last_freed_at)
|
|
88
|
+
return base;
|
|
89
|
+
return `${base} (last freed ${mib(last.last_freed_bytes)} MB at ${minute(last.last_freed_at)})`;
|
|
90
|
+
}
|
|
91
|
+
/** The biggest bucket, named, for the needs-fix sentence; null when empty. */
|
|
92
|
+
export function footprintHolder(footprint) {
|
|
93
|
+
return footprintBuckets(footprint)[0] ?? null;
|
|
94
|
+
}
|
|
95
|
+
function stagingOverheadBytes(footprint) {
|
|
96
|
+
const staging = footprint.staging;
|
|
97
|
+
const payload = staging.committed_bytes +
|
|
98
|
+
staging.uncommitted_bytes +
|
|
99
|
+
staging.unknown_bytes +
|
|
100
|
+
(staging.terminal_bytes ?? 0);
|
|
101
|
+
return Math.max(0, staging.total_bytes - payload);
|
|
102
|
+
}
|
|
103
|
+
function minute(iso) {
|
|
104
|
+
return iso.length >= 16 ? `${iso.slice(0, 16)}Z` : iso;
|
|
105
|
+
}
|
|
31
106
|
export function diskRowMessage(footprint, capBytes) {
|
|
32
107
|
const staging = footprint.staging;
|
|
33
108
|
const parts = [
|
|
109
|
+
footprintLine(footprint),
|
|
34
110
|
`staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
|
|
35
111
|
];
|
|
36
112
|
// BLI-3797: "1,916.9 uncommitted" is a number, not a finding. The row now
|
|
@@ -51,9 +127,11 @@ export function diskRowMessage(footprint, capBytes) {
|
|
|
51
127
|
if (staging.unknown_count > 0) {
|
|
52
128
|
parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
|
|
53
129
|
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
130
|
+
// BLI-11106: study folders an agent left under the state dir are named as a
|
|
131
|
+
// count, never sized or added to the footprint. They are not the
|
|
132
|
+
// collector's, and new ones belong in the sibling `bli-cockpit-studies`.
|
|
133
|
+
if (footprint.state.study_entries > 0) {
|
|
134
|
+
parts.push(`${footprint.state.study_entries} study folder(s) under the state dir are not counted (new studies go in bli-cockpit-studies beside it)`);
|
|
57
135
|
}
|
|
58
136
|
return parts.join("; ");
|
|
59
137
|
}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import { withDoctorLockWait } from "./doctor-lock-wait.js";
|
|
2
2
|
import { doctorBackfillBudgetSeconds, doctorLockWaitSeconds } from "./doctor-budget.js";
|
|
3
|
-
import fs from "node:fs/promises";
|
|
4
|
-
import path from "node:path";
|
|
5
3
|
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
6
4
|
import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
7
5
|
import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
|
|
@@ -12,7 +10,7 @@ import { runEvidenceRedelivery } from "../evidence-redelivery.js";
|
|
|
12
10
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
|
|
13
11
|
import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
|
|
14
12
|
import { runBackfillCommand } from "./backfill.js";
|
|
15
|
-
import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
|
|
13
|
+
import { FOOTPRINT_NEEDS_FIX_BYTES, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
16
14
|
import { doctorRoots } from "./doctor-access.js";
|
|
17
15
|
import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
18
16
|
import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
@@ -32,7 +30,6 @@ import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
|
32
30
|
* `doctor-pipeline-verdicts.ts`, and every one of them is re-exported from
|
|
33
31
|
* here so callers keep one address.
|
|
34
32
|
*/
|
|
35
|
-
const GC_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
36
33
|
export async function checkBackfillState(context) {
|
|
37
34
|
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
38
35
|
const roots = await doctorRoots(context);
|
|
@@ -137,18 +134,14 @@ export async function checkGcState(context) {
|
|
|
137
134
|
if (context.io.env["COCKPIT_DISABLE_GC"] === "1") {
|
|
138
135
|
return skipped("gc-checked", "skipped_disabled", "cleanup is switched off");
|
|
139
136
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
if (info && Date.now() - info.mtimeMs < GC_MIN_INTERVAL_MS) {
|
|
144
|
-
return skipped("gc-checked", "skipped_throttled", "cleanup already ran today");
|
|
145
|
-
}
|
|
146
|
-
return needsFix("gc-checked", "due", "cleanup is due");
|
|
137
|
+
// BLI-11106: the GC runs on every sync tick and costs one manifest read per
|
|
138
|
+
// old pack, so there is no "already ran today" to report; doctor runs it too.
|
|
139
|
+
return needsFix("gc-checked", "due", "cleanup runs on every sync; running it now");
|
|
147
140
|
}
|
|
148
141
|
export async function fixGcState(context) {
|
|
149
142
|
const result = await runRawEvidenceLocalGc(getCollectorRuntimePaths(context.command.homeDir), context.io.env);
|
|
150
143
|
if (result.skipped) {
|
|
151
|
-
return skipped("gc-checked", "
|
|
144
|
+
return skipped("gc-checked", "skipped_disabled", "cleanup is switched off");
|
|
152
145
|
}
|
|
153
146
|
if (result.removed_dirs === 0) {
|
|
154
147
|
return ok("gc-checked", "nothing_eligible", rawEvidenceGcSummary(result));
|
|
@@ -156,21 +149,47 @@ export async function fixGcState(context) {
|
|
|
156
149
|
return ok("gc-checked", `removed_${result.removed_dirs}`, rawEvidenceGcSummary(result));
|
|
157
150
|
}
|
|
158
151
|
/**
|
|
159
|
-
* BLI-3619: what
|
|
152
|
+
* BLI-3619 / BLI-11106: what Tower is holding on this machine, in one line.
|
|
160
153
|
*
|
|
161
|
-
* Green means the staging tree fits under the cap
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
154
|
+
* Green means the staging tree fits under the cap AND the whole collector
|
|
155
|
+
* footprint (staging, logs, state, spool) is under 512 MB. Either one crossed
|
|
156
|
+
* is `needs_fix`, and the message names the bucket holding the bytes. The fix
|
|
157
|
+
* is the same prune the tick runs; if it is still over afterwards, the message
|
|
158
|
+
* names the one command that goes further. Every number here is metadata —
|
|
159
|
+
* byte counts and file counts, never a path.
|
|
166
160
|
*/
|
|
167
161
|
export async function checkDiskState(context) {
|
|
168
162
|
const footprint = await readDiskFootprint(getCollectorRuntimePaths(context.command.homeDir));
|
|
169
163
|
const { capBytes } = retentionOptionsFromEnv(context.io.env);
|
|
170
164
|
const message = diskRowMessage(footprint, capBytes);
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
165
|
+
if (footprint.staging.total_bytes - (footprint.staging.terminal_bytes ?? 0) > capBytes) {
|
|
166
|
+
return needsFix("disk-bounded", "over_staging_cap", message);
|
|
167
|
+
}
|
|
168
|
+
const over = footprintOverSentence(footprint);
|
|
169
|
+
if (over)
|
|
170
|
+
return needsFix("disk-bounded", "footprint_over_512mb", `${over}; ${message}`);
|
|
171
|
+
return ok("disk-bounded", "within_staging_cap", message);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* "812.0 MB held, over the 512 MB line; most of it is sync logs (600.0 MB)",
|
|
175
|
+
* or null when the footprint is under the line. Logged either way, because a
|
|
176
|
+
* row that only speaks when it is red cannot say the check ran.
|
|
177
|
+
*/
|
|
178
|
+
export function footprintOverSentence(footprint) {
|
|
179
|
+
const total = footprintTotalBytes(footprint);
|
|
180
|
+
const holder = footprintHolder(footprint);
|
|
181
|
+
const over = total > FOOTPRINT_NEEDS_FIX_BYTES;
|
|
182
|
+
console.error("[cockpit-doctor] collector footprint", JSON.stringify({
|
|
183
|
+
reason: over ? "footprint_over_512mb" : "footprint_within_512mb",
|
|
184
|
+
total_bytes: total,
|
|
185
|
+
limit_bytes: FOOTPRINT_NEEDS_FIX_BYTES,
|
|
186
|
+
holder: holder?.label ?? null,
|
|
187
|
+
holder_bytes: holder?.bytes ?? 0,
|
|
188
|
+
study_entries_not_counted: footprint.state.study_entries,
|
|
189
|
+
}));
|
|
190
|
+
if (!over)
|
|
191
|
+
return null;
|
|
192
|
+
return `${mib(total)} MB held, over the ${mib(FOOTPRINT_NEEDS_FIX_BYTES)} MB line; most of it is ${holder?.label ?? "unknown"} (${mib(holder?.bytes ?? 0)} MB)`;
|
|
174
193
|
}
|
|
175
194
|
export async function fixDiskState(context) {
|
|
176
195
|
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
@@ -188,7 +207,6 @@ export async function fixDiskState(context) {
|
|
|
188
207
|
const redelivered = redeliveryLine(redelivery);
|
|
189
208
|
const pruned = await runStagingPrune(paths, {
|
|
190
209
|
env: context.io.env,
|
|
191
|
-
force: true,
|
|
192
210
|
});
|
|
193
211
|
if (pruned.status === "fail") {
|
|
194
212
|
return fail("disk-bounded", pruned.reason, "the prune could not run; nothing was deleted");
|
|
@@ -198,6 +216,10 @@ export async function fixDiskState(context) {
|
|
|
198
216
|
}
|
|
199
217
|
const footprint = await readDiskFootprint(paths);
|
|
200
218
|
const message = diskRowMessage(footprint, pruned.cap_bytes);
|
|
219
|
+
const over = pruned.cap_blocked_by_uncommitted ? null : footprintOverSentence(footprint);
|
|
220
|
+
if (over) {
|
|
221
|
+
return needsFix("disk-bounded", "footprint_over_512mb", `freed ${mib(pruned.deleted_bytes)} MB; ${over}; ${message}`);
|
|
222
|
+
}
|
|
201
223
|
if (!pruned.cap_blocked_by_uncommitted) {
|
|
202
224
|
return ok("disk-bounded", `freed_${pruned.deleted_files}`, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
|
|
203
225
|
}
|
|
@@ -133,7 +133,7 @@ export function buildCollectorHeartbeat(options) {
|
|
|
133
133
|
* that a green, syncing, up-to-date laptop was sitting on 1.8 GB of evidence the
|
|
134
134
|
* server had never received, and a fleet-wide failure nobody can see is the
|
|
135
135
|
* BLI-2528 shape exactly. The walk is bounded by the pack tree the prune keeps
|
|
136
|
-
* under a
|
|
136
|
+
* under a 256 MiB cap, and it is best-effort: a read that fails returns nulls and
|
|
137
137
|
* says so, so the heartbeat omits both fields rather than claiming zero.
|
|
138
138
|
*/
|
|
139
139
|
export async function readHeartbeatStagingFacts(options) {
|
|
@@ -348,11 +348,11 @@ export function localSubcommandHelp(command) {
|
|
|
348
348
|
"Usage: cockpit clean [--dry-run] [--all-committed] [--reconcile] [--dashboard-url <url>] [--json]",
|
|
349
349
|
"",
|
|
350
350
|
"Reclaims this machine's disk. Prints what it would delete and why, per bucket,",
|
|
351
|
-
"then deletes: staged
|
|
352
|
-
"
|
|
351
|
+
"then deletes: a staged copy Tower has already accepted goes (a retry re-reads",
|
|
352
|
+
"the source file), and anything the upload ledger has NOT vouched for is kept,",
|
|
353
353
|
"counted and named — retention never deletes undelivered evidence.",
|
|
354
|
-
"--dry-run stops after the printing. --all-committed drops
|
|
355
|
-
"so every already-accepted copy goes now, and is the only thing that removes a",
|
|
354
|
+
"--dry-run stops after the printing. --all-committed drops any retry window set",
|
|
355
|
+
"with COCKPIT_STAGING_RETENTION_HOURS, so every already-accepted copy goes now, and is the only thing that removes a",
|
|
356
356
|
"manual-study-evidence-vault folder (and only when every file in it is accepted).",
|
|
357
357
|
"--reconcile asks Tower's own upload ledger about every object this machine's",
|
|
358
358
|
"local memory can no longer classify (\"unknown\") before the buckets print — a",
|
|
@@ -361,9 +361,9 @@ export function localSubcommandHelp(command) {
|
|
|
361
361
|
"object instead of \"unknown\". A server failure never blocks the rest of clean",
|
|
362
362
|
"and is printed on its own line. Combine with --dry-run to ask and print",
|
|
363
363
|
"without deleting.",
|
|
364
|
-
"The scheduled sync does the same prune
|
|
365
|
-
"when there is anything unknown); this is the door for
|
|
366
|
-
"
|
|
364
|
+
"The scheduled sync does the same prune on every tick (reconciling one batch first",
|
|
365
|
+
"when there is anything unknown); this is the door for \"show me\" and for a",
|
|
366
|
+
"laptop that has been off.",
|
|
367
367
|
],
|
|
368
368
|
],
|
|
369
369
|
[
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.155");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit status`'s `Disk:` line (BLI-11106): the same footprint line
|
|
3
|
+
* `cockpit doctor`'s disk row opens with, so a person who pastes `status` into
|
|
4
|
+
* Slack also pastes what the collector holds on disk and when it last cleaned
|
|
5
|
+
* up. Human output only; `status --json` is unchanged. Split out of
|
|
6
|
+
* `status.ts` to keep that file under the readability ratchet.
|
|
7
|
+
*/
|
|
8
|
+
import { footprintLine } from "./doctor-disk-words.js";
|
|
9
|
+
import { readDiskFootprint } from "../disk-usage.js";
|
|
10
|
+
import { describeError } from "../health-detail.js";
|
|
11
|
+
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
12
|
+
/** Never throws: an unreadable footprint is a named line, not a failed status. */
|
|
13
|
+
export async function statusDiskLine(homeDir) {
|
|
14
|
+
try {
|
|
15
|
+
const footprint = await readDiskFootprint(getCollectorRuntimePaths(homeDir));
|
|
16
|
+
return `Disk: ${footprintLine(footprint)}`;
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
console.error("[cockpit-status] could not read the disk footprint", JSON.stringify({ reason: "footprint_read_failed", ...describeError(error) }));
|
|
20
|
+
return "Disk: could not be read (footprint_read_failed)";
|
|
21
|
+
}
|
|
22
|
+
}
|
package/dist/commands/status.js
CHANGED
|
@@ -22,6 +22,7 @@ import { displayTicketId, displayWorkLabel, shortSha, stuckEvidenceLine, } from
|
|
|
22
22
|
import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
23
23
|
import { refreshSetupReceipt } from "./setup-receipt.js";
|
|
24
24
|
import { setupReceiptBlock } from "./setup-receipt-lines.js";
|
|
25
|
+
import { statusDiskLine } from "./status-disk-line.js";
|
|
25
26
|
import { defaultCodexSessionDirs } from "../adapters/codex-attribution.js";
|
|
26
27
|
import { describeError } from "../health-detail.js";
|
|
27
28
|
import { backfillCompletionCovers, emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
@@ -77,6 +78,7 @@ async function writeMultiRepoStatus(command, io, backfillCursor, setupReceipt, w
|
|
|
77
78
|
return writeMultiRepoJson(io, backfillCursor, setupReceipt, statuses);
|
|
78
79
|
writeLine(io.stdout, "Tower status — every repo below this folder");
|
|
79
80
|
writeLine(io.stdout, `Old sessions: ${backfillCursorLine(backfillCursor)}`);
|
|
81
|
+
writeLine(io.stdout, await statusDiskLine(command.homeDir));
|
|
80
82
|
for (const status of statuses) {
|
|
81
83
|
writeLine(io.stdout, `- ${status.repo_label ?? status.repo}/${status.worktree_label ?? "worktree"} · ${status.branch} · head:${shortSha(status.head_sha)} · ${status.upload_state}`);
|
|
82
84
|
}
|
|
@@ -108,14 +110,14 @@ async function writeSingleRepoStatus(command, io, backfillCursor, setupReceipt)
|
|
|
108
110
|
});
|
|
109
111
|
if (command.json)
|
|
110
112
|
return writeSingleRepoJson(io, status, backfillCursor, setupReceipt);
|
|
111
|
-
writeSingleRepoLines(io, status, backfillCursor, setupReceipt);
|
|
113
|
+
writeSingleRepoLines(io, status, backfillCursor, setupReceipt, await statusDiskLine(command.homeDir));
|
|
112
114
|
return 0;
|
|
113
115
|
}
|
|
114
116
|
function writeSingleRepoJson(io, status, backfillCursor, setupReceipt) {
|
|
115
117
|
writeLine(io.stdout, JSON.stringify({ ...status, backfill_cursor: backfillCursor, setup_receipt: setupReceipt?.receipt ?? null }, null, 2));
|
|
116
118
|
return 0;
|
|
117
119
|
}
|
|
118
|
-
function writeSingleRepoLines(io, status, backfillCursor, setupReceipt) {
|
|
120
|
+
function writeSingleRepoLines(io, status, backfillCursor, setupReceipt, disk) {
|
|
119
121
|
writeLine(io.stdout, "Tower status");
|
|
120
122
|
writeLine(io.stdout, `Installed: ${status.installed ? "yes" : "no"}`);
|
|
121
123
|
writeLine(io.stdout, `Signed in: ${signedInLine(status.session_state)}`);
|
|
@@ -142,6 +144,7 @@ function writeSingleRepoLines(io, status, backfillCursor, setupReceipt) {
|
|
|
142
144
|
writeLine(io.stdout, `Last health note problem: ${status.last_health_receipt_failure_reason ?? "none"}`);
|
|
143
145
|
writeLine(io.stdout, `Old sessions: ${backfillCursorLine(backfillCursor)}`);
|
|
144
146
|
writeLine(io.stdout, `Stuck files: ${stuckEvidenceLine(status)}`);
|
|
147
|
+
writeLine(io.stdout, disk);
|
|
145
148
|
for (const detail of status.details)
|
|
146
149
|
writeLine(io.stdout, `- ${detail}`);
|
|
147
150
|
writeConnectedBlock(io, setupReceipt);
|
|
@@ -10,9 +10,11 @@ const RECONCILE_BATCHES_PER_TICK = 1;
|
|
|
10
10
|
* laptop forever.
|
|
11
11
|
*
|
|
12
12
|
* Same shape as the memory install beside it and for the same reasons: it runs
|
|
13
|
-
* only once collection's own outcome is decided and reported,
|
|
14
|
-
*
|
|
15
|
-
*
|
|
13
|
+
* only once collection's own outcome is decided and reported, it cannot throw,
|
|
14
|
+
* and its outcome is its own named receipt rather than a sync failure. Unlike
|
|
15
|
+
* the memory install it runs on EVERY tick (BLI-11106): the upload above has
|
|
16
|
+
* just recorded which objects the server committed, and their staged copies
|
|
17
|
+
* go in this same tick rather than a day or two later. A machine that cannot prune is a machine short of disk, and
|
|
16
18
|
* that must never also be a machine that stops collecting.
|
|
17
19
|
*
|
|
18
20
|
* The rule itself is `disk-retention.ts`: only objects the upload ledger
|
|
@@ -24,7 +26,7 @@ const RECONCILE_BATCHES_PER_TICK = 1;
|
|
|
24
26
|
* ledger's own memory is capped, so a laptop that keeps accumulating
|
|
25
27
|
* `unknown` state needs SOMETHING asking the server on a schedule, not only
|
|
26
28
|
* when a person happens to type `cockpit clean --reconcile`. One batch a tick
|
|
27
|
-
* is
|
|
29
|
+
* is bounded to one batch rather than drained whole — an `unknown` backlog
|
|
28
30
|
* drains at up to 500 hashes/15 min, and the ordinary case (nothing unknown)
|
|
29
31
|
* costs one cheap local read and no network call at all, so it never competes
|
|
30
32
|
* with collection for the tick's time. A reconcile failure never blocks the
|
|
@@ -59,9 +61,10 @@ export async function runStagingPruneAfterSync(command, io, dashboardUrl, option
|
|
|
59
61
|
detail: redactedSyncErrorDetail(error),
|
|
60
62
|
}));
|
|
61
63
|
}
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
|
|
64
|
+
// A tick with nothing to prune is the steady state: `[collector prune]`
|
|
65
|
+
// already logged it locally with the bytes scanned, and posting a receipt on
|
|
66
|
+
// 95 of every 96 ticks would say nothing new.
|
|
67
|
+
if (result.reason !== "nothing_to_prune") {
|
|
65
68
|
events.push(stagingPruneEvent(result));
|
|
66
69
|
}
|
|
67
70
|
if (events.length === 0)
|
|
@@ -78,8 +81,8 @@ export async function runStagingPruneAfterSync(command, io, dashboardUrl, option
|
|
|
78
81
|
/**
|
|
79
82
|
* Never throws (`runEvidenceReconcile` already never does); returns `null`
|
|
80
83
|
* for the boring, common case — nothing this tick was `unknown` — so that
|
|
81
|
-
* case costs no receipt either, the same rule the prune's own
|
|
82
|
-
* follows above.
|
|
84
|
+
* case costs no receipt either, the same rule the prune's own
|
|
85
|
+
* `nothing_to_prune` follows above.
|
|
83
86
|
*/
|
|
84
87
|
async function runReconcileFollowUp(command, io, dashboardUrl, options) {
|
|
85
88
|
const result = await runEvidenceReconcile({
|
|
@@ -102,7 +105,7 @@ async function runReconcileFollowUp(command, io, dashboardUrl, options) {
|
|
|
102
105
|
* Never throws (`runEvidenceRedelivery` already never does); returns `null` for
|
|
103
106
|
* the boring, common case — nothing on this disk is undelivered — so the steady
|
|
104
107
|
* state costs no receipt, the same rule the reconcile follow-up above and the
|
|
105
|
-
* prune's
|
|
108
|
+
* prune's `nothing_to_prune` below both follow.
|
|
106
109
|
*/
|
|
107
110
|
async function runRedeliveryFollowUp(command, io, dashboardUrl, options) {
|
|
108
111
|
const result = await runEvidenceRedelivery({
|
package/dist/commands/sync.js
CHANGED
|
@@ -24,8 +24,10 @@
|
|
|
24
24
|
*
|
|
25
25
|
* Both arms below do the same things in the same order, because a tick that
|
|
26
26
|
* THREW is still a tick a machine survived and the fleet has to be able to tell
|
|
27
|
-
* that from silence (BLI-3551).
|
|
28
|
-
*
|
|
27
|
+
* that from silence (BLI-3551). Both arms also run the staging prune: a tick
|
|
28
|
+
* that threw after uploading still leaves committed copies the prune may take,
|
|
29
|
+
* and waiting for the next clean tick is how a sick machine fills its disk
|
|
30
|
+
* (BLI-11106).
|
|
29
31
|
*
|
|
30
32
|
* Split out of commands/local.ts (BLI-3578), then into the siblings above
|
|
31
33
|
* (BLI-3982). The scan and upload engine underneath is `./session-sync.ts`;
|
|
@@ -78,8 +80,8 @@ export async function runSync(command, io) {
|
|
|
78
80
|
await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
|
|
79
81
|
await reportTickOutcome(command, io, dashboardUrl, result.completion);
|
|
80
82
|
await convergeAfterTick(command, io, dashboardUrl);
|
|
81
|
-
// BLI-3619
|
|
82
|
-
//
|
|
83
|
+
// BLI-3619 / BLI-11106: every tick, the staged copies of whatever the
|
|
84
|
+
// server just committed go; a follow-up never blocks or fails collection.
|
|
83
85
|
await runStagingPruneAfterSync(command, io, dashboardUrl);
|
|
84
86
|
return result.exitCode;
|
|
85
87
|
}
|
|
@@ -94,6 +96,7 @@ export async function runSync(command, io) {
|
|
|
94
96
|
});
|
|
95
97
|
await reportTickThrew(command, io, dashboardUrl, error, errorCode);
|
|
96
98
|
await convergeAfterTick(command, io, dashboardUrl);
|
|
99
|
+
await runStagingPruneAfterSync(command, io, dashboardUrl);
|
|
97
100
|
throw error;
|
|
98
101
|
}
|
|
99
102
|
}
|
package/dist/disk-prune.js
CHANGED
|
@@ -15,19 +15,25 @@
|
|
|
15
15
|
* evidence that the drain is still running at all (the BLI-2528 lesson: what
|
|
16
16
|
* only speaks on failure cannot answer "did anything work today?").
|
|
17
17
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* Runs on EVERY tick (BLI-11106). It used to sit behind a once-a-day marker
|
|
19
|
+
* file, and that is how 534 MB of committed, past-window copies were still on
|
|
20
|
+
* the reference Mac when it was measured: the drain existed and had not run
|
|
21
|
+
* yet. The inventory reads each pack's small `manifest.json` rather than its
|
|
22
|
+
* payload, so a tick with nothing to delete costs one directory walk and logs
|
|
23
|
+
* `nothing_to_prune` with the bytes it scanned. `COCKPIT_DISABLE_GC=1` is the
|
|
24
|
+
* only off switch.
|
|
25
|
+
*
|
|
26
|
+
* Every non-dry run leaves `last-prune.json` (one small object, overwritten)
|
|
27
|
+
* so `cockpit doctor` and `cockpit status` can say when it last ran and what
|
|
28
|
+
* it freed without re-running it.
|
|
22
29
|
*/
|
|
23
30
|
import fs from "node:fs/promises";
|
|
24
31
|
import path from "node:path";
|
|
25
32
|
import { planStagingRetention, retentionOptionsFromEnv, } from "./disk-retention.js";
|
|
26
33
|
import { RAW_EVIDENCE_DIR, readStagingInventory } from "./disk-usage.js";
|
|
27
34
|
import { describeError } from "./health-detail.js";
|
|
35
|
+
import { writeLastPruneReceipt } from "./prune-receipt.js";
|
|
28
36
|
import { readRawEvidenceStagingState, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
|
|
29
|
-
export const STAGING_PRUNE_THROTTLE_MARKER = ".last-staging-prune";
|
|
30
|
-
export const STAGING_PRUNE_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
31
37
|
/**
|
|
32
38
|
* The prune the sync tick runs and `cockpit clean` / `cockpit doctor` reuse.
|
|
33
39
|
* Never throws.
|
|
@@ -38,17 +44,14 @@ export async function runStagingPrune(paths, options = {}) {
|
|
|
38
44
|
if (env["COCKPIT_DISABLE_GC"] === "1") {
|
|
39
45
|
return skippedResult("disabled");
|
|
40
46
|
}
|
|
41
|
-
if (!options.force && (await prunedRecently(paths, now))) {
|
|
42
|
-
return skippedResult("throttled_recent_run");
|
|
43
|
-
}
|
|
44
47
|
try {
|
|
45
|
-
if (!options.dryRun && !options.force)
|
|
46
|
-
await markPruned(paths, now);
|
|
47
48
|
const plan = await planFromDisk(paths, env, options, now);
|
|
48
49
|
const result = options.dryRun
|
|
49
50
|
? dryRunResult(plan)
|
|
50
51
|
: await applyPlan(paths, plan);
|
|
51
52
|
reportPrune(result, { dryRun: Boolean(options.dryRun) });
|
|
53
|
+
if (!options.dryRun)
|
|
54
|
+
await writeLastPruneReceipt(paths, result, now);
|
|
52
55
|
return result;
|
|
53
56
|
}
|
|
54
57
|
catch (error) {
|
|
@@ -75,7 +78,7 @@ async function applyPlan(paths, plan) {
|
|
|
75
78
|
return {
|
|
76
79
|
...dryRunResult(plan),
|
|
77
80
|
status: "ok",
|
|
78
|
-
reason: tally.files > 0 ? "pruned" : "
|
|
81
|
+
reason: tally.files > 0 || tally.packs > 0 ? "pruned" : "nothing_to_prune",
|
|
79
82
|
deleted_files: tally.files,
|
|
80
83
|
deleted_bytes: tally.bytes,
|
|
81
84
|
removed_packs: tally.packs,
|
|
@@ -166,7 +169,9 @@ async function forgetStagedObjects(paths, hashes) {
|
|
|
166
169
|
function dryRunResult(plan) {
|
|
167
170
|
return {
|
|
168
171
|
status: "ok",
|
|
169
|
-
reason: plan.delete.length > 0
|
|
172
|
+
reason: plan.delete.length > 0 || plan.empty_packs.length > 0
|
|
173
|
+
? "would_prune"
|
|
174
|
+
: "nothing_to_prune",
|
|
170
175
|
deleted_files: plan.delete.length,
|
|
171
176
|
deleted_bytes: plan.deleted_bytes + plan.empty_pack_manifest_bytes,
|
|
172
177
|
removed_packs: plan.empty_packs.length,
|
|
@@ -221,6 +226,7 @@ export function reportPrune(result, options = { dryRun: false }) {
|
|
|
221
226
|
kept_unknown: result.kept_unknown,
|
|
222
227
|
kept_in_window: result.kept_in_window,
|
|
223
228
|
cap_bytes: result.cap_bytes,
|
|
229
|
+
bytes_scanned: result.bytes_before,
|
|
224
230
|
bytes_after: result.bytes_after,
|
|
225
231
|
failed_deletions: result.failed_deletions,
|
|
226
232
|
}));
|
|
@@ -234,14 +240,4 @@ export function reportPrune(result, options = { dryRun: false }) {
|
|
|
234
240
|
cap_bytes: result.cap_bytes,
|
|
235
241
|
bytes_after: result.bytes_after,
|
|
236
242
|
}));
|
|
237
|
-
}
|
|
238
|
-
async function prunedRecently(paths, now) {
|
|
239
|
-
const marker = path.join(paths.state_dir, STAGING_PRUNE_THROTTLE_MARKER);
|
|
240
|
-
const info = await fs.stat(marker).catch(() => null);
|
|
241
|
-
return Boolean(info && now.getTime() - info.mtimeMs < STAGING_PRUNE_MIN_INTERVAL_MS);
|
|
242
|
-
}
|
|
243
|
-
async function markPruned(paths, now) {
|
|
244
|
-
const marker = path.join(paths.state_dir, STAGING_PRUNE_THROTTLE_MARKER);
|
|
245
|
-
await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
|
|
246
|
-
await fs.writeFile(marker, now.toISOString()).catch(() => undefined);
|
|
247
243
|
}
|
package/dist/disk-retention.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* How long a committed spare is kept: zero (BLI-11106). A retry re-reads the
|
|
3
|
+
* source file, so the staged copy of a committed object is pure duplication.
|
|
4
|
+
*/
|
|
5
|
+
export const DEFAULT_STAGING_RETENTION_MS = 0;
|
|
3
6
|
/** Ceiling on everything under `raw-evidence`, manifests included. */
|
|
4
|
-
export const DEFAULT_STAGING_CAP_BYTES =
|
|
7
|
+
export const DEFAULT_STAGING_CAP_BYTES = 256 * 1024 * 1024;
|
|
5
8
|
/** Local knobs, read from the environment like `COCKPIT_DISABLE_GC`. */
|
|
6
9
|
export const STAGING_CAP_ENV = "COCKPIT_STAGING_CAP_BYTES";
|
|
7
10
|
export const STAGING_RETENTION_ENV = "COCKPIT_STAGING_RETENTION_HOURS";
|
|
@@ -33,7 +36,7 @@ export function planStagingRetention(inventory, options = {}) {
|
|
|
33
36
|
continue;
|
|
34
37
|
}
|
|
35
38
|
if (options.allCommitted || object.age_ms >= retentionMs) {
|
|
36
|
-
pushDeletion(plan, object, "
|
|
39
|
+
pushDeletion(plan, object, "committed_on_server");
|
|
37
40
|
continue;
|
|
38
41
|
}
|
|
39
42
|
committedInWindow.push(object);
|
|
@@ -61,11 +64,11 @@ export function planStagingRetention(inventory, options = {}) {
|
|
|
61
64
|
* safe elsewhere.
|
|
62
65
|
*
|
|
63
66
|
* It does not run at all when the floor is already above the cap. That branch
|
|
64
|
-
* is not theoretical: on the reference Mac (2026-09-05) 8.0 GB of staging
|
|
65
|
-
* evidence no ledger
|
|
66
|
-
* window
|
|
67
|
-
*
|
|
68
|
-
*
|
|
67
|
+
* is not theoretical: on the reference Mac (2026-09-05) 8.0 GB of staging was
|
|
68
|
+
* evidence no ledger vouched for, against the cap of the day. With the default
|
|
69
|
+
* zero window it rarely matters (committed spares are already gone); it keeps
|
|
70
|
+
* an opted-in window honest by taking a spare only when that can actually get
|
|
71
|
+
* this machine under the cap.
|
|
69
72
|
*/
|
|
70
73
|
function applyByteCap(plan, committedInWindow, capBytes, inventory) {
|
|
71
74
|
let projected = plan.bytes_before - plan.deleted_bytes;
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The whole-machine read: staging, rotated logs, the spool,
|
|
3
|
-
* person made by hand
|
|
2
|
+
* The whole-machine read: staging, rotated logs, the spool, the vaults a
|
|
3
|
+
* person made by hand, and (BLI-11106) the rest of the state directory plus
|
|
4
|
+
* the last prune's receipt.
|
|
4
5
|
*
|
|
5
|
-
* The staging half is the rest of this family;
|
|
6
|
-
*
|
|
6
|
+
* The staging half is the rest of this family; the state-dir audit is
|
|
7
|
+
* `state-dir-audit.ts` and the receipt `prune-receipt.ts`. What lives here is
|
|
8
|
+
* the one function that puts them together.
|
|
7
9
|
*/
|
|
8
10
|
import fs from "node:fs/promises";
|
|
9
11
|
import path from "node:path";
|
|
@@ -11,6 +13,8 @@ import { MANUAL_VAULT_PREFIX, } from "./disk-usage-facts.js";
|
|
|
11
13
|
import { directoryBytes, listPackFiles } from "./disk-usage-files.js";
|
|
12
14
|
import { readStagingInventory } from "./disk-usage-scan.js";
|
|
13
15
|
import { rotatedLogFootprint } from "./log-rotation.js";
|
|
16
|
+
import { readLastPruneReceipt } from "./prune-receipt.js";
|
|
17
|
+
import { readStateDirFootprint } from "./state-dir-audit.js";
|
|
14
18
|
/**
|
|
15
19
|
* Everything on this machine Cockpit put there, in one read. Used by the sync
|
|
16
20
|
* tick's prune, by `cockpit clean` and by `cockpit doctor`'s disk row, so all
|
|
@@ -22,6 +26,8 @@ export async function readDiskFootprint(paths, now = new Date()) {
|
|
|
22
26
|
logs: await rotatedLogFootprint(paths),
|
|
23
27
|
spool_bytes: await directoryBytes(paths.spool_dir),
|
|
24
28
|
vaults: await readManualVaults(paths),
|
|
29
|
+
state: await readStateDirFootprint(paths),
|
|
30
|
+
last_prune: await readLastPruneReceipt(paths),
|
|
25
31
|
};
|
|
26
32
|
}
|
|
27
33
|
/** The hand-made vaults: counted and named so nothing deletes them silently. */
|
|
@@ -35,8 +35,8 @@ export async function runEvidenceReconcile(options) {
|
|
|
35
35
|
if (hashes.length === 0) {
|
|
36
36
|
// No stderr line here on purpose: the sync tick calls this every 15
|
|
37
37
|
// minutes and the steady state is "nothing unknown" — logging that
|
|
38
|
-
// every tick forever is exactly the receipt-on-95-of-96-ticks spam
|
|
39
|
-
//
|
|
38
|
+
// every tick forever is exactly the receipt-on-95-of-96-ticks spam that
|
|
39
|
+
// the follow-ups skip on their `nothing_*` reasons. `cockpit clean --reconcile`
|
|
40
40
|
// still tells a person this on stdout (`reconcileLines` in clean.ts).
|
|
41
41
|
return emptyResult("nothing_unknown");
|
|
42
42
|
}
|
package/dist/log-rotation.js
CHANGED
|
@@ -37,9 +37,16 @@ import path from "node:path";
|
|
|
37
37
|
* of a file operators can read. The tail is capped rather than copied whole, so
|
|
38
38
|
* the archives cannot themselves become the disk problem — worst case on disk
|
|
39
39
|
* is (keep + 1) x maxBytes per stream.
|
|
40
|
+
*
|
|
41
|
+
* BLI-11106 tightened the numbers: at most 2 archives and 20 MB of archive per
|
|
42
|
+
* stream (2 x 10 MiB), so both streams together never pass 60 MiB, live logs
|
|
43
|
+
* included. It was 3 x 20 MiB per stream (160 MiB worst case) and the
|
|
44
|
+
* reference Mac held 147 MB of it on 2026-09-30. An archive above the new keep
|
|
45
|
+
* window (`sync.log.3` on every machine that ran the old numbers) is swept on
|
|
46
|
+
* the first tick by `sweepOrphanedLogArchives` below.
|
|
40
47
|
*/
|
|
41
|
-
export const DEFAULT_LOG_ROTATION_MAX_BYTES =
|
|
42
|
-
export const DEFAULT_LOG_ROTATION_KEEP =
|
|
48
|
+
export const DEFAULT_LOG_ROTATION_MAX_BYTES = 10 * 1024 * 1024;
|
|
49
|
+
export const DEFAULT_LOG_ROTATION_KEEP = 2;
|
|
43
50
|
/** The two files launchd writes for the scheduled tick. */
|
|
44
51
|
export const ROTATED_LOG_NAMES = ["sync.log", "sync.err.log"];
|
|
45
52
|
/**
|
|
@@ -129,27 +136,41 @@ async function rotateOne(filePath, maxBytes, keep) {
|
|
|
129
136
|
* sentences and an operator is owed the second one.
|
|
130
137
|
*/
|
|
131
138
|
export async function sweepOrphanedLogArchives(paths, options) {
|
|
139
|
+
const swept = [];
|
|
140
|
+
for (const archive of await orphanedLogArchives(paths, options)) {
|
|
141
|
+
const removed = await fs
|
|
142
|
+
.rm(path.join(paths.state_dir, archive.name), { force: true })
|
|
143
|
+
.then(() => true, () => false);
|
|
144
|
+
if (removed)
|
|
145
|
+
swept.push(archive);
|
|
146
|
+
}
|
|
147
|
+
return swept;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The archives the sweep above WOULD remove, without removing them — what
|
|
151
|
+
* `cockpit clean --dry-run` prints (BLI-11106). Same rule, one reader.
|
|
152
|
+
*/
|
|
153
|
+
export async function orphanedLogArchives(paths, options = {
|
|
154
|
+
keep: DEFAULT_LOG_ROTATION_KEEP,
|
|
155
|
+
names: ROTATED_LOG_NAMES,
|
|
156
|
+
}) {
|
|
132
157
|
const entries = await fs
|
|
133
158
|
.readdir(paths.state_dir, { withFileTypes: true })
|
|
134
159
|
.catch(() => []);
|
|
135
|
-
const
|
|
160
|
+
const found = [];
|
|
136
161
|
for (const entry of entries) {
|
|
137
162
|
if (!entry.isFile())
|
|
138
163
|
continue;
|
|
139
164
|
const reason = orphanArchiveReason(entry.name, options);
|
|
140
165
|
if (!reason)
|
|
141
166
|
continue;
|
|
142
|
-
const file = path.join(paths.state_dir, entry.name);
|
|
143
167
|
const size = await fs
|
|
144
|
-
.stat(
|
|
168
|
+
.stat(path.join(paths.state_dir, entry.name))
|
|
145
169
|
.then((info) => info.size)
|
|
146
170
|
.catch(() => 0);
|
|
147
|
-
|
|
148
|
-
if (!removed)
|
|
149
|
-
continue;
|
|
150
|
-
swept.push({ name: entry.name, byte_size: size, reason });
|
|
171
|
+
found.push({ name: entry.name, byte_size: size, reason });
|
|
151
172
|
}
|
|
152
|
-
return
|
|
173
|
+
return found;
|
|
153
174
|
}
|
|
154
175
|
/**
|
|
155
176
|
* Which stream this file belongs to and why it is outside the keep window, or
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `last-prune.json`: when the staging prune last ran and what it freed
|
|
3
|
+
* (BLI-11106).
|
|
4
|
+
*
|
|
5
|
+
* The prune runs on every sync tick, so "did it run?" is answered by a file it
|
|
6
|
+
* overwrites, never by a file that grows. One small JSON object, bounded by
|
|
7
|
+
* construction. `cockpit doctor` and `cockpit status` read it for the
|
|
8
|
+
* footprint line ("last prune: <when>, freed <n> MB") without re-running the
|
|
9
|
+
* prune themselves.
|
|
10
|
+
*
|
|
11
|
+
* It also carries the last time a prune actually freed something, because on
|
|
12
|
+
* a healthy machine the newest run is usually `nothing_to_prune` and "freed
|
|
13
|
+
* 0 MB" on its own cannot tell a working drain from a stuck one.
|
|
14
|
+
*
|
|
15
|
+
* The old once-a-day throttle marker is removed on the first write: it gates
|
|
16
|
+
* nothing any more, and a file that gates nothing is exactly the kind of
|
|
17
|
+
* leftover this ticket exists to stop.
|
|
18
|
+
*/
|
|
19
|
+
import fs from "node:fs/promises";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { describeError } from "./health-detail.js";
|
|
22
|
+
export const LAST_PRUNE_FILE = "last-prune.json";
|
|
23
|
+
/** The retired daily throttle marker; deleted on sight. */
|
|
24
|
+
export const LEGACY_STAGING_PRUNE_MARKER = ".last-staging-prune";
|
|
25
|
+
export async function readLastPruneReceipt(paths) {
|
|
26
|
+
const raw = await fs
|
|
27
|
+
.readFile(path.join(paths.state_dir, LAST_PRUNE_FILE), "utf8")
|
|
28
|
+
.catch(() => null);
|
|
29
|
+
if (!raw)
|
|
30
|
+
return null;
|
|
31
|
+
try {
|
|
32
|
+
const parsed = JSON.parse(raw);
|
|
33
|
+
if (typeof parsed.at !== "string" || typeof parsed.reason !== "string") {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
at: parsed.at,
|
|
38
|
+
reason: parsed.reason,
|
|
39
|
+
freed_bytes: numberOr0(parsed.freed_bytes),
|
|
40
|
+
deleted_files: numberOr0(parsed.deleted_files),
|
|
41
|
+
bytes_scanned: numberOr0(parsed.bytes_scanned),
|
|
42
|
+
bytes_after: numberOr0(parsed.bytes_after),
|
|
43
|
+
last_freed_at: typeof parsed.last_freed_at === "string" ? parsed.last_freed_at : null,
|
|
44
|
+
last_freed_bytes: numberOr0(parsed.last_freed_bytes),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// A torn or hand-edited receipt reads as "never ran"; the next tick
|
|
49
|
+
// overwrites it. Doctor prints "last prune: never" rather than guessing.
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Never throws: a receipt that cannot be written is a log line, not a failed tick. */
|
|
54
|
+
export async function writeLastPruneReceipt(paths, result, now) {
|
|
55
|
+
try {
|
|
56
|
+
const previous = await readLastPruneReceipt(paths);
|
|
57
|
+
const freed = result.deleted_bytes > 0;
|
|
58
|
+
const receipt = {
|
|
59
|
+
at: now.toISOString(),
|
|
60
|
+
reason: result.reason,
|
|
61
|
+
freed_bytes: result.deleted_bytes,
|
|
62
|
+
deleted_files: result.deleted_files,
|
|
63
|
+
bytes_scanned: result.bytes_before,
|
|
64
|
+
bytes_after: result.bytes_after,
|
|
65
|
+
last_freed_at: freed ? now.toISOString() : (previous?.last_freed_at ?? null),
|
|
66
|
+
last_freed_bytes: freed
|
|
67
|
+
? result.deleted_bytes
|
|
68
|
+
: (previous?.last_freed_bytes ?? 0),
|
|
69
|
+
};
|
|
70
|
+
await fs.mkdir(paths.state_dir, { recursive: true });
|
|
71
|
+
const file = path.join(paths.state_dir, LAST_PRUNE_FILE);
|
|
72
|
+
const temp = `${file}.${process.pid}.tmp`;
|
|
73
|
+
await fs.writeFile(temp, `${JSON.stringify(receipt)}\n`, { mode: 0o600 });
|
|
74
|
+
await fs.rename(temp, file);
|
|
75
|
+
await fs
|
|
76
|
+
.rm(path.join(paths.state_dir, LEGACY_STAGING_PRUNE_MARKER), { force: true })
|
|
77
|
+
.catch(() => undefined);
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
console.error("[collector prune] could not record the last-prune receipt", JSON.stringify({ reason: "last_prune_write_failed", ...describeError(error) }));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function numberOr0(value) {
|
|
84
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0
|
|
85
|
+
? value
|
|
86
|
+
: 0;
|
|
87
|
+
}
|
package/dist/raw-evidence-gc.js
CHANGED
|
@@ -10,27 +10,33 @@ const RAW_EVIDENCE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
|
10
10
|
* observed on the fleet is minutes, not hours.
|
|
11
11
|
*/
|
|
12
12
|
const STAGING_ORPHAN_MS = 6 * 60 * 60 * 1000;
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
/**
|
|
14
|
+
* The retired once-a-day throttle marker (BLI-11106). The GC runs every tick
|
|
15
|
+
* now, and deletes this file on sight so it stops being state nobody reads.
|
|
16
|
+
*
|
|
17
|
+
* Why every tick is affordable: the GC used to hash every file of every old
|
|
18
|
+
* pack to confirm it was uploaded, which re-read gigabytes on a machine with
|
|
19
|
+
* unconfirmed evidence. It now asks the pack's own `manifest.json` first, and a
|
|
20
|
+
* pack whose manifest names ANY hash the ledger has not committed is skipped
|
|
21
|
+
* without reading a payload byte. Only a pack the manifest says is fully
|
|
22
|
+
* committed is hashed, once, on the way to being deleted.
|
|
23
|
+
*/
|
|
24
|
+
export const LEGACY_GC_THROTTLE_MARKER = ".last-raw-evidence-gc";
|
|
17
25
|
export async function runRawEvidenceLocalGc(paths, env = process.env, now = new Date()) {
|
|
18
26
|
if (env["COCKPIT_DISABLE_GC"] === "1") {
|
|
19
27
|
return { skipped: true, removed_dirs: 0, freed_bytes: 0 };
|
|
20
28
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
return { skipped: true, removed_dirs: 0, freed_bytes: 0 };
|
|
25
|
-
}
|
|
26
|
-
await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
|
|
27
|
-
await fs.writeFile(throttleMarker, now.toISOString()).catch(() => undefined);
|
|
29
|
+
await fs
|
|
30
|
+
.rm(path.join(paths.state_dir, LEGACY_GC_THROTTLE_MARKER), { force: true })
|
|
31
|
+
.catch(() => undefined);
|
|
28
32
|
const cursor = await readRawEvidenceCursor(paths);
|
|
29
33
|
const uploadedHashes = new Set(Object.keys(cursor.objects));
|
|
30
34
|
const rawEvidenceRoot = path.join(paths.state_dir, "raw-evidence");
|
|
31
35
|
const entries = await fs.readdir(rawEvidenceRoot, { withFileTypes: true }).catch(() => []);
|
|
32
36
|
let removedDirs = 0;
|
|
33
37
|
let freedBytes = 0;
|
|
38
|
+
let scannedDirs = 0;
|
|
39
|
+
let skippedByManifest = 0;
|
|
34
40
|
for (const entry of entries) {
|
|
35
41
|
if (!entry.isDirectory() || !entry.name.startsWith("work-"))
|
|
36
42
|
continue;
|
|
@@ -40,6 +46,11 @@ export async function runRawEvidenceLocalGc(paths, env = process.env, now = new
|
|
|
40
46
|
continue;
|
|
41
47
|
if (now.getTime() - dirStats.mtimeMs < RAW_EVIDENCE_RETENTION_MS)
|
|
42
48
|
continue;
|
|
49
|
+
scannedDirs += 1;
|
|
50
|
+
if (await manifestNamesUncommitted(dir, uploadedHashes)) {
|
|
51
|
+
skippedByManifest += 1;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
43
54
|
const inspection = await inspectRawEvidenceDirForGc(dir, uploadedHashes);
|
|
44
55
|
if (!inspection.deletable)
|
|
45
56
|
continue;
|
|
@@ -50,12 +61,18 @@ export async function runRawEvidenceLocalGc(paths, env = process.env, now = new
|
|
|
50
61
|
freedBytes += inspection.byteSize;
|
|
51
62
|
}
|
|
52
63
|
// Log capping used to live here, and that is why it did not work: this GC
|
|
53
|
-
//
|
|
64
|
+
// was throttled to once a day and is skipped entirely by COCKPIT_DISABLE_GC,
|
|
54
65
|
// while sync.log grows ~8 MB an hour on an active machine (measured on the
|
|
55
66
|
// reference Mac, 2026-09-04: 219,866,695 bytes with the last GC 19 hours
|
|
56
|
-
// earlier).
|
|
57
|
-
//
|
|
58
|
-
|
|
67
|
+
// earlier). One owner now, running in the tick preamble: log-rotation.ts
|
|
68
|
+
// (BLI-3553).
|
|
69
|
+
console.error("[raw-evidence] gc sweep", JSON.stringify({
|
|
70
|
+
reason: removedDirs > 0 ? "gc_removed" : "nothing_to_prune",
|
|
71
|
+
scanned_dirs: scannedDirs,
|
|
72
|
+
skipped_by_manifest: skippedByManifest,
|
|
73
|
+
removed_dirs: removedDirs,
|
|
74
|
+
freed_bytes: freedBytes,
|
|
75
|
+
}));
|
|
59
76
|
return {
|
|
60
77
|
skipped: false,
|
|
61
78
|
removed_dirs: removedDirs,
|
|
@@ -241,6 +258,29 @@ async function dirByteSize(dir) {
|
|
|
241
258
|
}
|
|
242
259
|
return total;
|
|
243
260
|
}
|
|
261
|
+
/**
|
|
262
|
+
* True when the pack's own manifest names a content hash the ledger has not
|
|
263
|
+
* committed, so the pack cannot be deleted and no payload needs reading. A
|
|
264
|
+
* missing or unreadable manifest answers false and falls through to the full
|
|
265
|
+
* hash check, which is the old, conservative path.
|
|
266
|
+
*/
|
|
267
|
+
async function manifestNamesUncommitted(dir, uploadedHashes) {
|
|
268
|
+
const manifest = await fs
|
|
269
|
+
.readFile(path.join(dir, "manifest.json"), "utf8")
|
|
270
|
+
.catch(() => null);
|
|
271
|
+
if (!manifest)
|
|
272
|
+
return false;
|
|
273
|
+
try {
|
|
274
|
+
const parsed = JSON.parse(manifest);
|
|
275
|
+
return (parsed.files ?? []).some((file) => typeof file.content_hash_sha256 === "string" &&
|
|
276
|
+
!uploadedHashes.has(file.content_hash_sha256));
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
// Not silent in effect: an unparseable manifest falls through to the full
|
|
280
|
+
// per-file hash check below, which decides on the bytes themselves.
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
244
284
|
async function inspectRawEvidenceDirForGc(dir, uploadedHashes) {
|
|
245
285
|
const files = await listFiles(dir).catch(() => null);
|
|
246
286
|
if (!files)
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Everything the collector's state directory holds, sorted into who owns it
|
|
3
|
+
* and what bounds it (BLI-11106).
|
|
4
|
+
*
|
|
5
|
+
* `~/.local/state/bli-cockpit` (Windows: `%USERPROFILE%\.local\state\bli-cockpit`,
|
|
6
|
+
* the same layout with backslashes) is where the collector keeps its state.
|
|
7
|
+
* By 2026-09-30 it also held a gigabyte of study artifacts written by agents
|
|
8
|
+
* running benches and experiments (`memory-bench`, `memory-extract-*`,
|
|
9
|
+
* `chat-model-bench`, …). Those are not the collector's, the collector never
|
|
10
|
+
* reads them, and counting them as its footprint made `cockpit doctor`
|
|
11
|
+
* alarm about bytes no prune could ever explain.
|
|
12
|
+
*
|
|
13
|
+
* The rule: study artifacts belong in the SIBLING root
|
|
14
|
+
* `~/.local/state/bli-cockpit-studies` (same parent, same layout on Windows),
|
|
15
|
+
* see `studiesStateDir`. Existing study folders already under the state dir are
|
|
16
|
+
* recognised by name, left exactly where they are, and not counted. Anything
|
|
17
|
+
* the table below does not recognise is counted as `other`, so a new
|
|
18
|
+
* collector file that nobody registered still shows up in the footprint
|
|
19
|
+
* rather than hiding.
|
|
20
|
+
*
|
|
21
|
+
* Every collector entry carries the one-line reason it is bounded; the
|
|
22
|
+
* runbook (`docs/runbooks/cockpit-launchd-sync.md`, "What the state
|
|
23
|
+
* directory holds") prints the same table. Metadata only: entry names are
|
|
24
|
+
* fixed strings, never a person's path.
|
|
25
|
+
*/
|
|
26
|
+
import fs from "node:fs/promises";
|
|
27
|
+
import path from "node:path";
|
|
28
|
+
import { directoryBytes } from "./disk-usage-files.js";
|
|
29
|
+
/** The sibling root where agent-written studies live, never under the collector. */
|
|
30
|
+
export const STUDIES_DIR_SUFFIX = "-studies";
|
|
31
|
+
/**
|
|
32
|
+
* The collector's own entries first (checked before the study prefixes), then
|
|
33
|
+
* the studies. Order matters only where a study prefix could shadow a
|
|
34
|
+
* collector name; none does today, and the test pins that.
|
|
35
|
+
*/
|
|
36
|
+
export const STATE_DIR_RULES = [
|
|
37
|
+
{ name: "raw-evidence", match: "exact", class: "staging", bound: "staged copies go the tick the server commits them; 256 MiB cap; uncommitted evidence is never deleted, only counted" },
|
|
38
|
+
{ name: "sync.log", match: "prefix", class: "logs", bound: "rotated every tick at 10 MiB, at most 2 archives (20 MB) per stream" },
|
|
39
|
+
{ name: "sync.err.log", match: "prefix", class: "logs", bound: "rotated every tick at 10 MiB, at most 2 archives (20 MB) per stream" },
|
|
40
|
+
{ name: "spool", match: "exact", class: "spool", bound: "upload queue; each entry is deleted once the ingest door accepts it" },
|
|
41
|
+
{ name: "cursors", match: "exact", class: "state", bound: "raw-evidence ledger and reconcile answers capped at 5,000 rows each; backfill cursors one small file each" },
|
|
42
|
+
{ name: "raw-evidence-staging.json", match: "exact", class: "state", bound: "staged-object index and delivery attempts capped at 5,000 rows each" },
|
|
43
|
+
{ name: "work-contexts", match: "exact", class: "state", bound: "one small file per worktree ever bound with `cockpit start`; overwritten, not pruned (a few KB each)" },
|
|
44
|
+
{ name: "active-work-context.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
45
|
+
{ name: "root-work-scan.json", match: "exact", class: "state", bound: "one file, overwritten by each root scan" },
|
|
46
|
+
{ name: "setup-receipt.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
47
|
+
{ name: "memory-install-receipt.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
48
|
+
{ name: "last-prune.json", match: "exact", class: "state", bound: "one file, overwritten every tick" },
|
|
49
|
+
{ name: "last-turn-trace.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
50
|
+
{ name: "sync.lock", match: "exact", class: "state", bound: "lock file, removed when the tick ends" },
|
|
51
|
+
{ name: "backfill.lock", match: "exact", class: "state", bound: "lock file, removed when the backfill ends" },
|
|
52
|
+
{ name: ".last-", match: "prefix", class: "state", bound: "24-byte throttle markers for self-update, autostart repair and the memory install" },
|
|
53
|
+
{ name: "read-cache", match: "exact", class: "state", bound: "one response file per distinct Tower read, overwritten on refresh" },
|
|
54
|
+
{ name: "memory-hook-stats.json", match: "exact", class: "state", bound: "fixed-size histograms, overwritten (BLI Memory hooks)" },
|
|
55
|
+
{ name: "memory-hook-samples", match: "exact", class: "state", bound: "hourly content-free timing samples (BLI Memory hooks); not yet pruned, a few MB a week" },
|
|
56
|
+
{ name: "memory-recall-seen", match: "exact", class: "state", bound: "one file per agent session, deleted after 7 days (BLI Memory hooks)" },
|
|
57
|
+
{ name: "memory-stop-anchors.json", match: "exact", class: "state", bound: "one offset per transcript, overwritten (BLI Memory hooks)" },
|
|
58
|
+
{ name: "memory-daemon-spawn.json", match: "exact", class: "state", bound: "one spawn stamp, overwritten (BLI Memory daemon)" },
|
|
59
|
+
{ name: "memory-daemon.sock", match: "exact", class: "state", bound: "the daemon's socket, zero bytes" },
|
|
60
|
+
// Studies written by agents and scripts, not by the collector. Counted by
|
|
61
|
+
// name only, never sized, never deleted.
|
|
62
|
+
{ name: "memory-bench", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
63
|
+
{ name: "memory-extract-", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
64
|
+
{ name: "memory-questions", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
65
|
+
{ name: "memory-hook-context-sweep", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
66
|
+
{ name: "memory-anchor-probe", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
67
|
+
{ name: "memory-dup-threshold", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
68
|
+
{ name: "memory-judge-effort", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
69
|
+
{ name: "memory-recurrence-backfill", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
70
|
+
{ name: "memory-supersede-from-record", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
71
|
+
{ name: "memory-reconcile-precision", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
72
|
+
{ name: "memory-harvest-dedup", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
73
|
+
{ name: "chat-model-bench", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
74
|
+
{ name: "gpt6-bench-", match: "prefix", class: "study", bound: "agent study; belongs in bli-cockpit-studies" },
|
|
75
|
+
{ name: "manual-study-evidence-vault", match: "prefix", class: "study", bound: "hand-made evidence vault; `cockpit clean --all-committed` removes it once every file is committed" },
|
|
76
|
+
{ name: "issue-import", match: "exact", class: "study", bound: "one-off import receipts; belongs in bli-cockpit-studies" },
|
|
77
|
+
{ name: "analysis", match: "exact", class: "study", bound: "local harvest-analysis output; belongs in bli-cockpit-studies" },
|
|
78
|
+
{ name: "task-type-backlog", match: "prefix", class: "study", bound: "local harvest-analysis output; belongs in bli-cockpit-studies" },
|
|
79
|
+
];
|
|
80
|
+
/** Which rule an entry name falls under, or null for `other`. */
|
|
81
|
+
export function classifyStateEntry(name) {
|
|
82
|
+
for (const rule of STATE_DIR_RULES) {
|
|
83
|
+
if (rule.match === "exact" ? name === rule.name : name.startsWith(rule.name)) {
|
|
84
|
+
return rule;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* The sibling studies root, on either host family: the state dir's own path
|
|
91
|
+
* with `-studies` appended, so `/Users/x/.local/state/bli-cockpit` becomes
|
|
92
|
+
* `/Users/x/.local/state/bli-cockpit-studies` and
|
|
93
|
+
* `C:\Users\x\.local\state\bli-cockpit` becomes
|
|
94
|
+
* `C:\Users\x\.local\state\bli-cockpit-studies`. No separator is joined, so
|
|
95
|
+
* neither layout can be spelled wrong.
|
|
96
|
+
*/
|
|
97
|
+
export function studiesStateDir(paths) {
|
|
98
|
+
return `${paths.state_dir.replace(/[\\/]+$/u, "")}${STUDIES_DIR_SUFFIX}`;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Sizes the `state` and `other` entries of the state directory. Staging,
|
|
102
|
+
* logs and spool are measured by their own readers and skipped here; studies
|
|
103
|
+
* are counted but never walked, because a gigabyte of somebody's bench run is
|
|
104
|
+
* not worth a stat per file on every doctor call.
|
|
105
|
+
*/
|
|
106
|
+
export async function readStateDirFootprint(paths) {
|
|
107
|
+
const footprint = {
|
|
108
|
+
state_bytes: 0,
|
|
109
|
+
other_bytes: 0,
|
|
110
|
+
other_entries: 0,
|
|
111
|
+
study_entries: 0,
|
|
112
|
+
};
|
|
113
|
+
const entries = await fs
|
|
114
|
+
.readdir(paths.state_dir, { withFileTypes: true })
|
|
115
|
+
.catch(() => []);
|
|
116
|
+
for (const entry of entries) {
|
|
117
|
+
const rule = classifyStateEntry(entry.name);
|
|
118
|
+
const cls = rule?.class ?? "other";
|
|
119
|
+
if (cls === "staging" || cls === "logs" || cls === "spool")
|
|
120
|
+
continue;
|
|
121
|
+
if (cls === "study") {
|
|
122
|
+
footprint.study_entries += 1;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
const full = path.join(paths.state_dir, entry.name);
|
|
126
|
+
const bytes = entry.isDirectory()
|
|
127
|
+
? await directoryBytes(full)
|
|
128
|
+
: await fs.stat(full).then((info) => info.size, () => 0);
|
|
129
|
+
if (cls === "state") {
|
|
130
|
+
footprint.state_bytes += bytes;
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
footprint.other_bytes += bytes;
|
|
134
|
+
footprint.other_entries += 1;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return footprint;
|
|
138
|
+
}
|