@bli-cockpit/cli 0.2.69 → 0.2.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/doctor-disk-words.js +59 -0
- package/dist/commands/doctor-pipeline.js +15 -21
- package/dist/commands/heartbeat.js +64 -1
- package/dist/commands/memory-hook-counts.js +61 -0
- package/dist/commands/ops-render-memory.js +34 -0
- package/dist/commands/ops-render.js +1 -1
- package/dist/commands/ops.js +24 -6
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/status.js +8 -1
- package/dist/commands/sync-followups.js +64 -1
- package/dist/commands/sync.js +13 -2
- package/dist/disk-retention.js +5 -1
- package/dist/disk-usage.js +62 -9
- package/dist/evidence-redelivery-plan.js +277 -0
- package/dist/evidence-redelivery.js +280 -0
- package/dist/local-state-status.js +34 -0
- package/package.json +4 -4
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
export function days(ms) {
|
|
2
|
+
return (ms / 86_400_000).toFixed(1);
|
|
3
|
+
}
|
|
4
|
+
/**
|
|
5
|
+
* What the drain did, in the sentence a person reads (BLI-3797).
|
|
6
|
+
*
|
|
7
|
+
* Until this ticket the row said "run `cockpit sync` to deliver them" about
|
|
8
|
+
* objects no sync would ever offer again: the live sync only sends what it
|
|
9
|
+
* collected THIS tick, so 1.8 GB of orphaned evidence sat behind a fix sentence
|
|
10
|
+
* that could not come true. The drain is real now, `--fix` spends it here, and
|
|
11
|
+
* the row reports what it actually managed rather than issuing an instruction.
|
|
12
|
+
*/
|
|
13
|
+
export function redeliveryLine(result) {
|
|
14
|
+
if (result.reason === "no_device_session") {
|
|
15
|
+
return "they cannot be re-offered until this machine signs in — run `cockpit login`";
|
|
16
|
+
}
|
|
17
|
+
if (result.reason === "disabled") {
|
|
18
|
+
return "re-delivery is switched off on this machine (COCKPIT_DISABLE_EVIDENCE_REDELIVERY=1)";
|
|
19
|
+
}
|
|
20
|
+
if (result.uploaded > 0 || result.reused > 0) {
|
|
21
|
+
return `re-delivered ${result.uploaded + result.reused} of them (${mib(result.uploaded_bytes)} MB); ${result.deferred} left for the next tick`;
|
|
22
|
+
}
|
|
23
|
+
if (result.failed > 0) {
|
|
24
|
+
return `re-offering them was refused: ${result.failure_reasons.join(", ") || "unknown"} — the next tick retries under backoff`;
|
|
25
|
+
}
|
|
26
|
+
if (result.held > 0) {
|
|
27
|
+
return `${result.held} are held by delivery backoff and retry on a later tick`;
|
|
28
|
+
}
|
|
29
|
+
return "the next `cockpit sync` re-offers them";
|
|
30
|
+
}
|
|
31
|
+
export function diskRowMessage(footprint, capBytes) {
|
|
32
|
+
const staging = footprint.staging;
|
|
33
|
+
const parts = [
|
|
34
|
+
`staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
|
|
35
|
+
];
|
|
36
|
+
// BLI-3797: "1,916.9 uncommitted" is a number, not a finding. The row now
|
|
37
|
+
// names WHY the biggest share of it has not landed and HOW LONG those bytes
|
|
38
|
+
// have been here — on the reference Mac the answer was
|
|
39
|
+
// `reconciled_unknown_to_server` on 234 objects whose oldest was 45 days old,
|
|
40
|
+
// which is a different problem from a delivery that failed this morning.
|
|
41
|
+
const dominant = staging.dominant_uncommitted_reason;
|
|
42
|
+
if (dominant) {
|
|
43
|
+
parts.push(`oldest undelivered ${days(dominant.oldest_disk_age_ms)}d; biggest reason ${dominant.reason} on ${dominant.count} object(s), ${mib(dominant.bytes)} MB`);
|
|
44
|
+
}
|
|
45
|
+
// BLI-3619's second half: "unknown" means the local ledger's own capped
|
|
46
|
+
// memory cannot say, never that delivery failed — and the one command that
|
|
47
|
+
// actually answers it is named right here, not left for a person to find.
|
|
48
|
+
if (staging.unknown_count > 0) {
|
|
49
|
+
parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
|
|
50
|
+
}
|
|
51
|
+
parts.push(`logs ${mib(footprint.logs.total_bytes)} MB`, `spool ${mib(footprint.spool_bytes)} MB`);
|
|
52
|
+
for (const vault of footprint.vaults) {
|
|
53
|
+
parts.push(`${vault.name} ${mib(vault.byte_size)} MB (a one-off; \`cockpit clean --all-committed\` removes it only if every file in it is accepted)`);
|
|
54
|
+
}
|
|
55
|
+
return parts.join("; ");
|
|
56
|
+
}
|
|
57
|
+
export function mib(bytes) {
|
|
58
|
+
return (bytes / (1024 * 1024)).toFixed(1);
|
|
59
|
+
}
|
|
@@ -7,9 +7,11 @@ import { describeError } from "../health-detail.js";
|
|
|
7
7
|
import { runStagingPrune } from "../disk-prune.js";
|
|
8
8
|
import { retentionOptionsFromEnv } from "../disk-retention.js";
|
|
9
9
|
import { readDiskFootprint } from "../disk-usage.js";
|
|
10
|
+
import { runEvidenceRedelivery } from "../evidence-redelivery.js";
|
|
10
11
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
|
|
11
12
|
import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
|
|
12
13
|
import { runBackfillCommand } from "./backfill.js";
|
|
14
|
+
import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
|
|
13
15
|
import { doctorRoots } from "./doctor-access.js";
|
|
14
16
|
import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
15
17
|
/**
|
|
@@ -126,6 +128,18 @@ export async function checkDiskState(context) {
|
|
|
126
128
|
}
|
|
127
129
|
export async function fixDiskState(context) {
|
|
128
130
|
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
131
|
+
// BLI-3797, BEFORE the prune: the prune can only delete what the ledger
|
|
132
|
+
// vouches for, so on a machine held over the cap by undelivered evidence it
|
|
133
|
+
// frees nothing at all. Deliver first, then sweep what delivering made
|
|
134
|
+
// durable. This never throws and never decides the row's verdict — a drain
|
|
135
|
+
// that could not run leaves the prune to do exactly what it did before.
|
|
136
|
+
const redelivery = await runEvidenceRedelivery({
|
|
137
|
+
homeDir: context.command.homeDir,
|
|
138
|
+
dashboardUrl: context.command.dashboardUrl,
|
|
139
|
+
env: context.io.env,
|
|
140
|
+
fetch: context.io.fetch,
|
|
141
|
+
});
|
|
142
|
+
const redelivered = redeliveryLine(redelivery);
|
|
129
143
|
const pruned = await runStagingPrune(paths, {
|
|
130
144
|
env: context.io.env,
|
|
131
145
|
force: true,
|
|
@@ -144,27 +158,7 @@ export async function fixDiskState(context) {
|
|
|
144
158
|
// Deliberately still `ok`: staging over the cap because evidence has not been
|
|
145
159
|
// accepted yet is the collector working, not a machine to repair. The row
|
|
146
160
|
// names the blockage and the one command that goes further.
|
|
147
|
-
return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest —
|
|
148
|
-
}
|
|
149
|
-
function diskRowMessage(footprint, capBytes) {
|
|
150
|
-
const staging = footprint.staging;
|
|
151
|
-
const parts = [
|
|
152
|
-
`staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
|
|
153
|
-
];
|
|
154
|
-
// BLI-3619's second half: "unknown" means the local ledger's own capped
|
|
155
|
-
// memory cannot say, never that delivery failed — and the one command that
|
|
156
|
-
// actually answers it is named right here, not left for a person to find.
|
|
157
|
-
if (staging.unknown_count > 0) {
|
|
158
|
-
parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
|
|
159
|
-
}
|
|
160
|
-
parts.push(`logs ${mib(footprint.logs.total_bytes)} MB`, `spool ${mib(footprint.spool_bytes)} MB`);
|
|
161
|
-
for (const vault of footprint.vaults) {
|
|
162
|
-
parts.push(`${vault.name} ${mib(vault.byte_size)} MB (a one-off; \`cockpit clean --all-committed\` removes it only if every file in it is accepted)`);
|
|
163
|
-
}
|
|
164
|
-
return parts.join("; ");
|
|
165
|
-
}
|
|
166
|
-
function mib(bytes) {
|
|
167
|
-
return (bytes / (1024 * 1024)).toFixed(1);
|
|
161
|
+
return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest — ${redelivered} — or \`cockpit clean --all-committed\` to drop every accepted copy now`);
|
|
168
162
|
}
|
|
169
163
|
function capturedIo(io, forward) {
|
|
170
164
|
const stdoutChunks = [];
|
|
@@ -25,10 +25,12 @@ import os from "node:os";
|
|
|
25
25
|
import path from "node:path";
|
|
26
26
|
import { COLLECTOR_HEARTBEAT_SCHEMA_VERSION, MemoryInstallReceiptSchema, memoryInstallGaps, setupReceiptGaps, } from "@bli-cockpit/telemetry-core";
|
|
27
27
|
import { readCachedSetupReceipt } from "./setup-receipt.js";
|
|
28
|
+
import { readStagingInventory } from "../disk-usage.js";
|
|
28
29
|
import { describeError } from "../health-detail.js";
|
|
29
30
|
import { shouldSuppressFleetReceipts, } from "../dev-build.js";
|
|
30
31
|
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
|
|
31
32
|
import { readMemoryReceiptFile } from "./memory-install-receipt.js";
|
|
33
|
+
import { readMemoryHookCounts } from "./memory-hook-counts.js";
|
|
32
34
|
const HEARTBEAT_TIMEOUT_MS = 5_000;
|
|
33
35
|
/**
|
|
34
36
|
* The label form of one approved root.
|
|
@@ -110,10 +112,44 @@ export function buildCollectorHeartbeat(options) {
|
|
|
110
112
|
...(typeof options.facts.sessionsPendingUpload === "number"
|
|
111
113
|
? { sessions_pending_upload: options.facts.sessionsPendingUpload }
|
|
112
114
|
: {}),
|
|
115
|
+
...(typeof options.facts.stagingUncommittedBytes === "number"
|
|
116
|
+
? { staging_uncommitted_bytes: options.facts.stagingUncommittedBytes }
|
|
117
|
+
: {}),
|
|
118
|
+
...(options.facts.stagingUncommittedReason
|
|
119
|
+
? { staging_uncommitted_reason: options.facts.stagingUncommittedReason }
|
|
120
|
+
: {}),
|
|
113
121
|
...(options.memoryInstall ? { memory_install: options.memoryInstall } : {}),
|
|
114
122
|
...(options.setupReceipt ? { setup_receipt: options.setupReceipt } : {}),
|
|
115
123
|
};
|
|
116
124
|
}
|
|
125
|
+
/**
|
|
126
|
+
* What this machine is still holding that never reached storage (BLI-3797).
|
|
127
|
+
*
|
|
128
|
+
* One extra staging walk per tick, on purpose: `cockpit ops` had no way to see
|
|
129
|
+
* that a green, syncing, up-to-date laptop was sitting on 1.8 GB of evidence the
|
|
130
|
+
* server had never received, and a fleet-wide failure nobody can see is the
|
|
131
|
+
* BLI-2528 shape exactly. The walk is bounded by the pack tree the prune keeps
|
|
132
|
+
* under a 2 GB cap, and it is best-effort: a read that fails returns nulls and
|
|
133
|
+
* says so, so the heartbeat omits both fields rather than claiming zero.
|
|
134
|
+
*/
|
|
135
|
+
export async function readHeartbeatStagingFacts(options) {
|
|
136
|
+
const paths = getCollectorRuntimePaths(options.homeDir);
|
|
137
|
+
try {
|
|
138
|
+
const inventory = await readStagingInventory(paths, options.now ?? new Date());
|
|
139
|
+
const dominant = inventory.dominant_uncommitted_reason;
|
|
140
|
+
return {
|
|
141
|
+
bytes: inventory.uncommitted_bytes,
|
|
142
|
+
reason: dominant ? `${dominant.reason}:${dominant.count}`.slice(0, 120) : null,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
console.error("[heartbeat] the staging tree could not be read; this tick reports no backlog figure", JSON.stringify({
|
|
147
|
+
reason: "staging_inventory_unreadable",
|
|
148
|
+
...describeError(error),
|
|
149
|
+
}));
|
|
150
|
+
return { bytes: null, reason: null };
|
|
151
|
+
}
|
|
152
|
+
}
|
|
117
153
|
/**
|
|
118
154
|
* The cached memory receipt for this tick, or null with a named reason
|
|
119
155
|
* (BLI-3729).
|
|
@@ -123,6 +159,13 @@ export function buildCollectorHeartbeat(options) {
|
|
|
123
159
|
* reads and a `--print-config` spawn every fifteen minutes to re-prove a state
|
|
124
160
|
* that changes about once a month. The reason is returned rather than logged
|
|
125
161
|
* here so the ONE heartbeat log line carries it.
|
|
162
|
+
*
|
|
163
|
+
* BLI-3788: the hook COUNTS are merged in here, on the tick, and deliberately
|
|
164
|
+
* not baked into the cached receipt. The five words change about monthly; how
|
|
165
|
+
* often the prompt hook lost its deadline changes every hour, and a number
|
|
166
|
+
* cached for a day would answer yesterday's question. An install receipt this
|
|
167
|
+
* machine could not read means no counts either — there is nowhere to put
|
|
168
|
+
* them — and that case keeps the receipt's own reason.
|
|
126
169
|
*/
|
|
127
170
|
export async function readHeartbeatMemoryReceipt(options) {
|
|
128
171
|
const paths = getCollectorRuntimePaths(options.homeDir);
|
|
@@ -142,7 +185,20 @@ export async function readHeartbeatMemoryReceipt(options) {
|
|
|
142
185
|
return parsed.success ? parsed.data : null;
|
|
143
186
|
},
|
|
144
187
|
});
|
|
145
|
-
|
|
188
|
+
if (!result.receipt)
|
|
189
|
+
return { receipt: null, reason: result.reason };
|
|
190
|
+
const hooks = readMemoryHookCounts({
|
|
191
|
+
homeDir: options.homeDir ?? os.homedir(),
|
|
192
|
+
...(options.now ? { now: options.now } : {}),
|
|
193
|
+
});
|
|
194
|
+
return {
|
|
195
|
+
receipt: {
|
|
196
|
+
...result.receipt,
|
|
197
|
+
...(hooks.counts ?? {}),
|
|
198
|
+
hook_stats_reason: hooks.reason,
|
|
199
|
+
},
|
|
200
|
+
reason: result.reason,
|
|
201
|
+
};
|
|
146
202
|
}
|
|
147
203
|
/**
|
|
148
204
|
* Sends the heartbeat. Returns whether it landed; never throws.
|
|
@@ -248,6 +304,13 @@ export async function sendCollectorHeartbeatBestEffort(options) {
|
|
|
248
304
|
// sync.err.log alone.
|
|
249
305
|
memory_receipt: memory.reason,
|
|
250
306
|
memory_gaps: memory.receipt ? memoryInstallGaps(memory.receipt) : null,
|
|
307
|
+
// BLI-3788. The hooks are registered (above) AND they either worked or
|
|
308
|
+
// did not (here). A prompt hook that misses its deadline prints
|
|
309
|
+
// nothing to the person, so these counts are the only trace one leaves
|
|
310
|
+
// on this machine; `null` means no counter file, never zero misses.
|
|
311
|
+
memory_hook_runs_24h: memory.receipt?.hook_runs_24h ?? null,
|
|
312
|
+
memory_hook_timeouts_24h: memory.receipt?.hook_timeouts_24h ?? null,
|
|
313
|
+
memory_hook_stats: memory.receipt?.hook_stats_reason ?? null,
|
|
251
314
|
}));
|
|
252
315
|
return true;
|
|
253
316
|
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HOW OFTEN DID THE HOOKS MISS? — the reading half (BLI-3788).
|
|
3
|
+
*
|
|
4
|
+
* `bli-memory-mcp` counts every hook run into a counts-only file in this
|
|
5
|
+
* machine's state directory. This module reads it once a sync tick and hands
|
|
6
|
+
* the last 24 hours to the heartbeat, where it rides the memory receipt into
|
|
7
|
+
* `ambient_collector_devices.metadata.memory_install` and out onto `cockpit
|
|
8
|
+
* ops --memory`.
|
|
9
|
+
*
|
|
10
|
+
* Nothing here knows the file's shape: the path, the schema and the windowing
|
|
11
|
+
* rule are `@bli-cockpit/telemetry-core`'s `memory-hook-stats.ts`, which the
|
|
12
|
+
* writing package spends too. That is the whole point of putting them there.
|
|
13
|
+
*
|
|
14
|
+
* ## Which event, and why only one
|
|
15
|
+
*
|
|
16
|
+
* The PROMPT hook. It is the one a person waits on with their sentence typed,
|
|
17
|
+
* the one whose budget QA tick 18 caught it losing, and the one that runs on
|
|
18
|
+
* every turn — so it is the only one whose miss rate means anything as a
|
|
19
|
+
* daily number. The file holds all three; a future gauge that wants
|
|
20
|
+
* SessionStart or Stop reads the same rows with the same function.
|
|
21
|
+
*
|
|
22
|
+
* ## Absent is not zero
|
|
23
|
+
*
|
|
24
|
+
* A machine with no file has no counts and says `hook_stats_absent`. A machine
|
|
25
|
+
* whose file will not parse says `hook_stats_unparseable`. Neither reports
|
|
26
|
+
* zeroes, because a zero here would read as "the hooks ran and never missed",
|
|
27
|
+
* which is the exact false green this ticket exists to remove.
|
|
28
|
+
*/
|
|
29
|
+
import fs from "node:fs";
|
|
30
|
+
import { memoryHookStatsFilePath, parseMemoryHookStats, summariseMemoryHookWindow, } from "@bli-cockpit/telemetry-core";
|
|
31
|
+
export function readMemoryHookCounts(options) {
|
|
32
|
+
const readText = options.readText ?? defaultReadText;
|
|
33
|
+
const file = memoryHookStatsFilePath(options.homeDir);
|
|
34
|
+
const raw = readText(file);
|
|
35
|
+
if (raw === null)
|
|
36
|
+
return { counts: null, reason: "hook_stats_absent" };
|
|
37
|
+
const parsed = parseMemoryHookStats(raw);
|
|
38
|
+
if (!parsed.ok)
|
|
39
|
+
return { counts: null, reason: parsed.reason };
|
|
40
|
+
const window = summariseMemoryHookWindow(parsed.file, "prompt", {
|
|
41
|
+
now: options.now ?? new Date(),
|
|
42
|
+
hours: 24,
|
|
43
|
+
});
|
|
44
|
+
return {
|
|
45
|
+
counts: {
|
|
46
|
+
hook_runs_24h: window.runs,
|
|
47
|
+
hook_timeouts_24h: window.timeouts,
|
|
48
|
+
hook_printed_24h: window.printed,
|
|
49
|
+
hook_failed_24h: window.failed,
|
|
50
|
+
},
|
|
51
|
+
reason: "ok",
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function defaultReadText(file) {
|
|
55
|
+
try {
|
|
56
|
+
return fs.readFileSync(file, "utf8");
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -70,6 +70,40 @@ export function renderMemoryUsage(memory, dim) {
|
|
|
70
70
|
}
|
|
71
71
|
return lines;
|
|
72
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* The hook half: one line per machine, the server's own words.
|
|
75
|
+
*
|
|
76
|
+
* A machine that reports NO counts is printed too, dimmed and named. It is the
|
|
77
|
+
* whole point: "no measurement" and "no misses" are different facts, and the
|
|
78
|
+
* only reason this section exists is that they used to be indistinguishable.
|
|
79
|
+
*/
|
|
80
|
+
export function renderMemoryHooks(hooks, dim) {
|
|
81
|
+
const lines = ["", `HOOKS ${hooks.summary ?? "(no summary)"}`];
|
|
82
|
+
if (hooks.readError) {
|
|
83
|
+
lines.push(` the devices could not be read (${hooks.readError}); nothing is known about whether recalls are arriving`);
|
|
84
|
+
return lines;
|
|
85
|
+
}
|
|
86
|
+
const devices = hooks.devices ?? [];
|
|
87
|
+
if (devices.length === 0) {
|
|
88
|
+
lines.push(dim(" no live collector device to report on"));
|
|
89
|
+
return lines;
|
|
90
|
+
}
|
|
91
|
+
// Worst first: the machine losing recalls is the one somebody acts on, and a
|
|
92
|
+
// machine that reports nothing sinks below both — it is a rollout gap, not a
|
|
93
|
+
// failure.
|
|
94
|
+
const ranked = [...devices].sort((left, right) => {
|
|
95
|
+
const leftKnown = typeof left.runs === "number" ? 0 : 1;
|
|
96
|
+
const rightKnown = typeof right.runs === "number" ? 0 : 1;
|
|
97
|
+
if (leftKnown !== rightKnown)
|
|
98
|
+
return leftKnown - rightKnown;
|
|
99
|
+
return (right.timeouts ?? 0) - (left.timeouts ?? 0);
|
|
100
|
+
});
|
|
101
|
+
for (const device of ranked) {
|
|
102
|
+
const line = ` ${device.line ?? `${device.displayName ?? "?"}: (no line)`}`;
|
|
103
|
+
lines.push((device.timeouts ?? 0) > 0 ? line : dim(line));
|
|
104
|
+
}
|
|
105
|
+
return lines;
|
|
106
|
+
}
|
|
73
107
|
/** The two sources that mean a PERSON's agent used memory. Never the import. */
|
|
74
108
|
function agentSaves(counts) {
|
|
75
109
|
return (counts?.mcp ?? 0) + (counts?.stop_hook ?? 0);
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* artifact does and does not prove — because that is exactly the moment
|
|
13
13
|
* somebody is about to conclude something from it.
|
|
14
14
|
*/
|
|
15
|
-
export { renderMemoryUsage } from "./ops-render-memory.js";
|
|
15
|
+
export { renderMemoryHooks, renderMemoryUsage } from "./ops-render-memory.js";
|
|
16
16
|
/** The word a person reads. Short, fixed width, and never a bare colour. */
|
|
17
17
|
export function verdictWord(verdict) {
|
|
18
18
|
switch (verdict) {
|
package/dist/commands/ops.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
*/
|
|
26
26
|
import { colorEnabled, dim, writeLine } from "./cli-io.js";
|
|
27
27
|
import { renderOpsStatus } from "./ops-render.js";
|
|
28
|
-
import { renderMemoryUsage, } from "./ops-render-memory.js";
|
|
28
|
+
import { renderMemoryHooks, renderMemoryUsage, } from "./ops-render-memory.js";
|
|
29
29
|
import { asRecord, callTower, openTower, writeCommandFailure } from "./tower-command.js";
|
|
30
30
|
/**
|
|
31
31
|
* A whole compile, plus a little. `maxDuration` on `/api/ops/recompile` is 800
|
|
@@ -79,7 +79,7 @@ async function runOpsStatus(command, io, tower) {
|
|
|
79
79
|
// BLI-3762: a job that ran and wrote less than it owed is not healthy.
|
|
80
80
|
row.verdict === "degraded");
|
|
81
81
|
if (command.json) {
|
|
82
|
-
writeLine(io.stdout, JSON.stringify(memory ? { ...payload, memory: memory.section } : payload));
|
|
82
|
+
writeLine(io.stdout, JSON.stringify(memory ? { ...payload, memory: memory.section, hooks: memory.hooks } : payload));
|
|
83
83
|
}
|
|
84
84
|
else {
|
|
85
85
|
const styled = colorEnabled(io);
|
|
@@ -91,7 +91,16 @@ async function runOpsStatus(command, io, tower) {
|
|
|
91
91
|
writeLine(io.stdout, line);
|
|
92
92
|
}
|
|
93
93
|
}
|
|
94
|
-
|
|
94
|
+
// BLI-3788. Printed whenever the door sent it, including when the usage
|
|
95
|
+
// half could not be read: "are the recalls arriving" and "is anybody
|
|
96
|
+
// saving" are two questions, and one being unreadable does not silence
|
|
97
|
+
// the other.
|
|
98
|
+
if (memory?.hooks) {
|
|
99
|
+
for (const line of renderMemoryHooks(memory.hooks, (text) => dim(text, styled))) {
|
|
100
|
+
writeLine(io.stdout, line);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (memory && !memory.section) {
|
|
95
104
|
// Never a silent gap where a section was asked for: the reason the gauge
|
|
96
105
|
// could not be read is printed where the gauge would have been.
|
|
97
106
|
writeLine(io.stdout, "");
|
|
@@ -119,6 +128,12 @@ async function runOpsStatus(command, io, tower) {
|
|
|
119
128
|
memory_agent_saves: memory?.section
|
|
120
129
|
? (memory.section.counts?.mcp ?? 0) + (memory.section.counts?.stop_hook ?? 0)
|
|
121
130
|
: null,
|
|
131
|
+
// BLI-3788: whether the per-turn recalls arrived, on the same line. A
|
|
132
|
+
// null here is "this dashboard does not report it yet", never zero
|
|
133
|
+
// misses.
|
|
134
|
+
memory_hook_runs_24h: memory?.hooks?.totals?.runs ?? null,
|
|
135
|
+
memory_hook_timeouts_24h: memory?.hooks?.totals?.timeouts ?? null,
|
|
136
|
+
memory_hook_devices_reporting: memory?.hooks?.reporting ?? null,
|
|
122
137
|
})}`);
|
|
123
138
|
return unhealthy.length > 0 ? 1 : 0;
|
|
124
139
|
}
|
|
@@ -144,12 +159,15 @@ async function readMemoryUsage(command, io, tower) {
|
|
|
144
159
|
reason: result.reason,
|
|
145
160
|
http_status: result.httpStatus ?? null,
|
|
146
161
|
})}`);
|
|
147
|
-
return { section: null, reason: result.reason };
|
|
162
|
+
return { section: null, hooks: null, reason: result.reason };
|
|
148
163
|
}
|
|
149
164
|
const body = asRecord(result.body);
|
|
165
|
+
// BLI-3788: the hook counts ride the same answer, and their absence is a
|
|
166
|
+
// fact about the SERVER's version rather than about this fleet — an older
|
|
167
|
+
// dashboard sends no `hooks` key, and printing nothing is right there.
|
|
150
168
|
if (!body.memory)
|
|
151
|
-
return { section: null, reason: "memory_section_absent" };
|
|
152
|
-
return { section: body.memory, reason: "ok" };
|
|
169
|
+
return { section: null, hooks: body.hooks ?? null, reason: "memory_section_absent" };
|
|
170
|
+
return { section: body.memory, hooks: body.hooks ?? null, reason: "ok" };
|
|
153
171
|
}
|
|
154
172
|
async function runOpsRecompile(command, io, tower) {
|
|
155
173
|
const person = command.person ?? "";
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.72");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
package/dist/commands/status.js
CHANGED
|
@@ -72,7 +72,14 @@ export async function runStatus(command, io) {
|
|
|
72
72
|
writeConnectedBlock(io, setupReceipt);
|
|
73
73
|
return 0;
|
|
74
74
|
}
|
|
75
|
-
|
|
75
|
+
// The staging walk is opt-in (BLI-3797) and this is the surface that pays for
|
|
76
|
+
// it: one person asking one question. The multi-repo branch above deliberately
|
|
77
|
+
// does not — the backlog is a MACHINE fact, so N repos would walk one tree N
|
|
78
|
+
// times to print the same number.
|
|
79
|
+
const status = await inspectLocalCollectorStatus({
|
|
80
|
+
...command,
|
|
81
|
+
includeStagingBacklog: true,
|
|
82
|
+
});
|
|
76
83
|
if (command.json) {
|
|
77
84
|
writeLine(io.stdout, JSON.stringify({
|
|
78
85
|
...status,
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* What the sync tick does AFTER collection's own outcome is decided and
|
|
3
3
|
* reported: keep this machine's CLI current on npm `latest` (BLI-2601), put a
|
|
4
4
|
* broken scheduler registration back (BLI-2721), keep BLI Memory registered
|
|
5
|
-
* with both agent hosts (BLI-3580),
|
|
5
|
+
* with both agent hosts (BLI-3580), re-offer staged evidence that never landed
|
|
6
|
+
* (BLI-3797), and stop the laptop filling up (BLI-3619).
|
|
6
7
|
*
|
|
7
8
|
* Split out of commands/sync.ts (BLI-3578), moved verbatim. They belong
|
|
8
9
|
* together because they share one rule, and it is the reason both are called
|
|
@@ -20,6 +21,7 @@ import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-sta
|
|
|
20
21
|
import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
|
|
21
22
|
import { runStagingPrune } from "../disk-prune.js";
|
|
22
23
|
import { runEvidenceReconcile, } from "../evidence-reconcile-client.js";
|
|
24
|
+
import { runEvidenceRedelivery, } from "../evidence-redelivery.js";
|
|
23
25
|
import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
|
|
24
26
|
/** How many reconcile batches (of up to 500 hashes each) one sync tick may spend. */
|
|
25
27
|
const RECONCILE_BATCHES_PER_TICK = 1;
|
|
@@ -215,12 +217,20 @@ function memoryInstallEvent(outcome) {
|
|
|
215
217
|
* costs one cheap local read and no network call at all, so it never competes
|
|
216
218
|
* with collection for the tick's time. A reconcile failure never blocks the
|
|
217
219
|
* prune that follows it.
|
|
220
|
+
*
|
|
221
|
+
* BLI-3797 sits between them: the reconcile has just established which staged
|
|
222
|
+
* objects the SERVER says it never received, so the redelivery drain re-offers
|
|
223
|
+
* exactly those, bounded, before the prune runs and can delete whatever landed.
|
|
224
|
+
* Neither of the two can block the prune, and none of the three can fail a tick.
|
|
218
225
|
*/
|
|
219
226
|
export async function runStagingPruneAfterSync(command, io, dashboardUrl, options = {}) {
|
|
220
227
|
const events = [];
|
|
221
228
|
const reconciled = await runReconcileFollowUp(command, io, dashboardUrl, options);
|
|
222
229
|
if (reconciled)
|
|
223
230
|
events.push(reconcileEvent(reconciled));
|
|
231
|
+
const redelivered = await runRedeliveryFollowUp(command, io, dashboardUrl, options);
|
|
232
|
+
if (redelivered)
|
|
233
|
+
events.push(redeliveryEvent(redelivered));
|
|
224
234
|
let result;
|
|
225
235
|
try {
|
|
226
236
|
result = await runStagingPrune(getCollectorRuntimePaths(command.homeDir), {
|
|
@@ -269,6 +279,59 @@ async function runReconcileFollowUp(command, io, dashboardUrl, options) {
|
|
|
269
279
|
});
|
|
270
280
|
return result.reason === "nothing_unknown" ? null : result;
|
|
271
281
|
}
|
|
282
|
+
/**
|
|
283
|
+
* BLI-3797, between the reconcile above and the prune below, and in that order
|
|
284
|
+
* for a reason: reconcile turns `unknown` into a server-backed answer, this
|
|
285
|
+
* re-offers what that answer says never landed, and the prune then deletes
|
|
286
|
+
* whatever this just made durable. Running the drain first would ask the server
|
|
287
|
+
* about hashes it is about to be told the truth about; running it after the
|
|
288
|
+
* prune would leave a tick's worth of freed cap unused.
|
|
289
|
+
*
|
|
290
|
+
* Never throws (`runEvidenceRedelivery` already never does); returns `null` for
|
|
291
|
+
* the boring, common case — nothing on this disk is undelivered — so the steady
|
|
292
|
+
* state costs no receipt, the same rule the reconcile follow-up above and the
|
|
293
|
+
* prune's daily throttle below both follow.
|
|
294
|
+
*/
|
|
295
|
+
async function runRedeliveryFollowUp(command, io, dashboardUrl, options) {
|
|
296
|
+
const result = await runEvidenceRedelivery({
|
|
297
|
+
homeDir: command.homeDir,
|
|
298
|
+
dashboardUrl,
|
|
299
|
+
env: io.env,
|
|
300
|
+
fetch: io.fetch,
|
|
301
|
+
...(options.now ? { now: options.now } : {}),
|
|
302
|
+
});
|
|
303
|
+
return result.reason === "nothing_uncommitted" ? null : result;
|
|
304
|
+
}
|
|
305
|
+
/** Counts and byte totals only; no pack id and no hash travels in a receipt. */
|
|
306
|
+
function redeliveryEvent(result) {
|
|
307
|
+
const detail = [
|
|
308
|
+
`offered ${result.offered} object(s), ${result.offered_bytes}B, across ${result.packs} pack(s)`,
|
|
309
|
+
`uploaded ${result.uploaded} (${result.uploaded_bytes}B), reused ${result.reused}, failed ${result.failed}`,
|
|
310
|
+
`held ${result.held}, deferred ${result.deferred} (${result.deferred_bytes}B)`,
|
|
311
|
+
result.failure_reasons.length > 0
|
|
312
|
+
? `failure_reasons ${result.failure_reasons.join(",")}`
|
|
313
|
+
: null,
|
|
314
|
+
]
|
|
315
|
+
.filter((part) => Boolean(part))
|
|
316
|
+
.join("; ");
|
|
317
|
+
if (result.status === "fail") {
|
|
318
|
+
return {
|
|
319
|
+
step: "evidence_redelivery",
|
|
320
|
+
status: "fail",
|
|
321
|
+
error_code: result.reason,
|
|
322
|
+
error_detail: detail,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
if (result.status === "skipped") {
|
|
326
|
+
return {
|
|
327
|
+
step: "evidence_redelivery",
|
|
328
|
+
status: "skipped",
|
|
329
|
+
error_code: result.reason,
|
|
330
|
+
error_detail: detail,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
return { step: "evidence_redelivery", status: "ok", error_detail: detail };
|
|
334
|
+
}
|
|
272
335
|
function prunedNothing() {
|
|
273
336
|
return {
|
|
274
337
|
status: "skipped",
|
package/dist/commands/sync.js
CHANGED
|
@@ -2,7 +2,7 @@ import { writeLine } from "./cli-io.js";
|
|
|
2
2
|
import { attributedSyncRunStatus, cursorStatusLine, displayTicketId, rawEvidenceSyncLine, shortSha, worktreeSyncRow, writeAgentSessionSummary, } from "./collection-report.js";
|
|
3
3
|
import { collectionRootConsentAliases } from "./collection-roots.js";
|
|
4
4
|
import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
5
|
-
import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
|
|
5
|
+
import { sendCollectorHeartbeatBestEffort, readHeartbeatStagingFacts, } from "./heartbeat.js";
|
|
6
6
|
import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
7
7
|
import { runAttributedWorktreeSync, } from "./session-sync.js";
|
|
8
8
|
import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
|
|
@@ -98,11 +98,22 @@ export async function runSync(command, io) {
|
|
|
98
98
|
*/
|
|
99
99
|
async function sendSyncHeartbeat(command, io, dashboardUrl, facts) {
|
|
100
100
|
const roots = await resolveSyncCollectionRoots(command).catch(() => []);
|
|
101
|
+
// BLI-3797: the backlog figure rides the same check-in as the root labels, so
|
|
102
|
+
// `cockpit ops` learns about undelivered evidence on the SAME tick that proves
|
|
103
|
+
// the machine is alive. Best-effort: nulls omit the fields rather than
|
|
104
|
+
// reporting a zero nobody measured.
|
|
105
|
+
const staging = await readHeartbeatStagingFacts({
|
|
106
|
+
homeDir: command.homeDir,
|
|
107
|
+
}).catch(() => ({ bytes: null, reason: null }));
|
|
101
108
|
await sendCollectorHeartbeatBestEffort({
|
|
102
109
|
homeDir: command.homeDir,
|
|
103
110
|
dashboardUrl,
|
|
104
111
|
roots,
|
|
105
|
-
facts
|
|
112
|
+
facts: {
|
|
113
|
+
...facts,
|
|
114
|
+
stagingUncommittedBytes: staging.bytes,
|
|
115
|
+
stagingUncommittedReason: staging.reason,
|
|
116
|
+
},
|
|
106
117
|
io,
|
|
107
118
|
}).catch((error) => {
|
|
108
119
|
// The sender already swallows everything it knows about; this is the net
|
package/dist/disk-retention.js
CHANGED
|
@@ -131,7 +131,11 @@ function pushDeletion(plan, object, reason) {
|
|
|
131
131
|
function addToBucket(bucket, object) {
|
|
132
132
|
bucket.count += 1;
|
|
133
133
|
bucket.bytes += object.byte_size;
|
|
134
|
-
|
|
134
|
+
// Report the age of the BYTES, not of the last answer about them (BLI-3797).
|
|
135
|
+
// `age_ms` on a reconciled object is the age of the reconcile reply, which
|
|
136
|
+
// made `oldest_uncommitted_age_ms` read 0.94 days for a 45-day-old backlog.
|
|
137
|
+
// Deletion still keys off `age_ms` above — this bucket is report-only.
|
|
138
|
+
bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.disk_age_ms || object.age_ms);
|
|
135
139
|
}
|
|
136
140
|
function removeFromBucket(bucket, object) {
|
|
137
141
|
bucket.count -= 1;
|