@bli-cockpit/cli 0.2.69 → 0.2.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,59 @@
1
+ export function days(ms) {
2
+ return (ms / 86_400_000).toFixed(1);
3
+ }
4
+ /**
5
+ * What the drain did, in the sentence a person reads (BLI-3797).
6
+ *
7
+ * Until this ticket the row said "run `cockpit sync` to deliver them" about
8
+ * objects no sync would ever offer again: the live sync only sends what it
9
+ * collected THIS tick, so 1.8 GB of orphaned evidence sat behind a fix sentence
10
+ * that could not come true. The drain is real now, `--fix` spends it here, and
11
+ * the row reports what it actually managed rather than issuing an instruction.
12
+ */
13
+ export function redeliveryLine(result) {
14
+ if (result.reason === "no_device_session") {
15
+ return "they cannot be re-offered until this machine signs in — run `cockpit login`";
16
+ }
17
+ if (result.reason === "disabled") {
18
+ return "re-delivery is switched off on this machine (COCKPIT_DISABLE_EVIDENCE_REDELIVERY=1)";
19
+ }
20
+ if (result.uploaded > 0 || result.reused > 0) {
21
+ return `re-delivered ${result.uploaded + result.reused} of them (${mib(result.uploaded_bytes)} MB); ${result.deferred} left for the next tick`;
22
+ }
23
+ if (result.failed > 0) {
24
+ return `re-offering them was refused: ${result.failure_reasons.join(", ") || "unknown"} — the next tick retries under backoff`;
25
+ }
26
+ if (result.held > 0) {
27
+ return `${result.held} are held by delivery backoff and retry on a later tick`;
28
+ }
29
+ return "the next `cockpit sync` re-offers them";
30
+ }
31
+ export function diskRowMessage(footprint, capBytes) {
32
+ const staging = footprint.staging;
33
+ const parts = [
34
+ `staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
35
+ ];
36
+ // BLI-3797: "1,916.9 uncommitted" is a number, not a finding. The row now
37
+ // names WHY the biggest share of it has not landed and HOW LONG those bytes
38
+ // have been here — on the reference Mac the answer was
39
+ // `reconciled_unknown_to_server` on 234 objects whose oldest was 45 days old,
40
+ // which is a different problem from a delivery that failed this morning.
41
+ const dominant = staging.dominant_uncommitted_reason;
42
+ if (dominant) {
43
+ parts.push(`oldest undelivered ${days(dominant.oldest_disk_age_ms)}d; biggest reason ${dominant.reason} on ${dominant.count} object(s), ${mib(dominant.bytes)} MB`);
44
+ }
45
+ // BLI-3619's second half: "unknown" means the local ledger's own capped
46
+ // memory cannot say, never that delivery failed — and the one command that
47
+ // actually answers it is named right here, not left for a person to find.
48
+ if (staging.unknown_count > 0) {
49
+ parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
50
+ }
51
+ parts.push(`logs ${mib(footprint.logs.total_bytes)} MB`, `spool ${mib(footprint.spool_bytes)} MB`);
52
+ for (const vault of footprint.vaults) {
53
+ parts.push(`${vault.name} ${mib(vault.byte_size)} MB (a one-off; \`cockpit clean --all-committed\` removes it only if every file in it is accepted)`);
54
+ }
55
+ return parts.join("; ");
56
+ }
57
+ export function mib(bytes) {
58
+ return (bytes / (1024 * 1024)).toFixed(1);
59
+ }
@@ -7,9 +7,11 @@ import { describeError } from "../health-detail.js";
7
7
  import { runStagingPrune } from "../disk-prune.js";
8
8
  import { retentionOptionsFromEnv } from "../disk-retention.js";
9
9
  import { readDiskFootprint } from "../disk-usage.js";
10
+ import { runEvidenceRedelivery } from "../evidence-redelivery.js";
10
11
  import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
11
12
  import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
12
13
  import { runBackfillCommand } from "./backfill.js";
14
+ import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
13
15
  import { doctorRoots } from "./doctor-access.js";
14
16
  import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
15
17
  /**
@@ -126,6 +128,18 @@ export async function checkDiskState(context) {
126
128
  }
127
129
  export async function fixDiskState(context) {
128
130
  const paths = getCollectorRuntimePaths(context.command.homeDir);
131
+ // BLI-3797, BEFORE the prune: the prune can only delete what the ledger
132
+ // vouches for, so on a machine held over the cap by undelivered evidence it
133
+ // frees nothing at all. Deliver first, then sweep what delivering made
134
+ // durable. This never throws and never decides the row's verdict — a drain
135
+ // that could not run leaves the prune to do exactly what it did before.
136
+ const redelivery = await runEvidenceRedelivery({
137
+ homeDir: context.command.homeDir,
138
+ dashboardUrl: context.command.dashboardUrl,
139
+ env: context.io.env,
140
+ fetch: context.io.fetch,
141
+ });
142
+ const redelivered = redeliveryLine(redelivery);
129
143
  const pruned = await runStagingPrune(paths, {
130
144
  env: context.io.env,
131
145
  force: true,
@@ -144,27 +158,7 @@ export async function fixDiskState(context) {
144
158
  // Deliberately still `ok`: staging over the cap because evidence has not been
145
159
  // accepted yet is the collector working, not a machine to repair. The row
146
160
  // names the blockage and the one command that goes further.
147
- return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest — run \`cockpit sync\` to deliver them, or \`cockpit clean --all-committed\` to drop every accepted copy now`);
148
- }
149
- function diskRowMessage(footprint, capBytes) {
150
- const staging = footprint.staging;
151
- const parts = [
152
- `staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
153
- ];
154
- // BLI-3619's second half: "unknown" means the local ledger's own capped
155
- // memory cannot say, never that delivery failed — and the one command that
156
- // actually answers it is named right here, not left for a person to find.
157
- if (staging.unknown_count > 0) {
158
- parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
159
- }
160
- parts.push(`logs ${mib(footprint.logs.total_bytes)} MB`, `spool ${mib(footprint.spool_bytes)} MB`);
161
- for (const vault of footprint.vaults) {
162
- parts.push(`${vault.name} ${mib(vault.byte_size)} MB (a one-off; \`cockpit clean --all-committed\` removes it only if every file in it is accepted)`);
163
- }
164
- return parts.join("; ");
165
- }
166
- function mib(bytes) {
167
- return (bytes / (1024 * 1024)).toFixed(1);
161
+ return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest — ${redelivered} or \`cockpit clean --all-committed\` to drop every accepted copy now`);
168
162
  }
169
163
  function capturedIo(io, forward) {
170
164
  const stdoutChunks = [];
@@ -25,10 +25,12 @@ import os from "node:os";
25
25
  import path from "node:path";
26
26
  import { COLLECTOR_HEARTBEAT_SCHEMA_VERSION, MemoryInstallReceiptSchema, memoryInstallGaps, setupReceiptGaps, } from "@bli-cockpit/telemetry-core";
27
27
  import { readCachedSetupReceipt } from "./setup-receipt.js";
28
+ import { readStagingInventory } from "../disk-usage.js";
28
29
  import { describeError } from "../health-detail.js";
29
30
  import { shouldSuppressFleetReceipts, } from "../dev-build.js";
30
31
  import { getCollectorRuntimePaths, readLocalCollectorSessionFile, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
31
32
  import { readMemoryReceiptFile } from "./memory-install-receipt.js";
33
+ import { readMemoryHookCounts } from "./memory-hook-counts.js";
32
34
  const HEARTBEAT_TIMEOUT_MS = 5_000;
33
35
  /**
34
36
  * The label form of one approved root.
@@ -110,10 +112,44 @@ export function buildCollectorHeartbeat(options) {
110
112
  ...(typeof options.facts.sessionsPendingUpload === "number"
111
113
  ? { sessions_pending_upload: options.facts.sessionsPendingUpload }
112
114
  : {}),
115
+ ...(typeof options.facts.stagingUncommittedBytes === "number"
116
+ ? { staging_uncommitted_bytes: options.facts.stagingUncommittedBytes }
117
+ : {}),
118
+ ...(options.facts.stagingUncommittedReason
119
+ ? { staging_uncommitted_reason: options.facts.stagingUncommittedReason }
120
+ : {}),
113
121
  ...(options.memoryInstall ? { memory_install: options.memoryInstall } : {}),
114
122
  ...(options.setupReceipt ? { setup_receipt: options.setupReceipt } : {}),
115
123
  };
116
124
  }
125
+ /**
126
+ * What this machine is still holding that never reached storage (BLI-3797).
127
+ *
128
+ * One extra staging walk per tick, on purpose: `cockpit ops` had no way to see
129
+ * that a green, syncing, up-to-date laptop was sitting on 1.8 GB of evidence the
130
+ * server had never received, and a fleet-wide failure nobody can see is the
131
+ * BLI-2528 shape exactly. The walk is bounded by the pack tree the prune keeps
132
+ * under a 2 GB cap, and it is best-effort: a read that fails returns nulls and
133
+ * says so, so the heartbeat omits both fields rather than claiming zero.
134
+ */
135
+ export async function readHeartbeatStagingFacts(options) {
136
+ const paths = getCollectorRuntimePaths(options.homeDir);
137
+ try {
138
+ const inventory = await readStagingInventory(paths, options.now ?? new Date());
139
+ const dominant = inventory.dominant_uncommitted_reason;
140
+ return {
141
+ bytes: inventory.uncommitted_bytes,
142
+ reason: dominant ? `${dominant.reason}:${dominant.count}`.slice(0, 120) : null,
143
+ };
144
+ }
145
+ catch (error) {
146
+ console.error("[heartbeat] the staging tree could not be read; this tick reports no backlog figure", JSON.stringify({
147
+ reason: "staging_inventory_unreadable",
148
+ ...describeError(error),
149
+ }));
150
+ return { bytes: null, reason: null };
151
+ }
152
+ }
117
153
  /**
118
154
  * The cached memory receipt for this tick, or null with a named reason
119
155
  * (BLI-3729).
@@ -123,6 +159,13 @@ export function buildCollectorHeartbeat(options) {
123
159
  * reads and a `--print-config` spawn every fifteen minutes to re-prove a state
124
160
  * that changes about once a month. The reason is returned rather than logged
125
161
  * here so the ONE heartbeat log line carries it.
162
+ *
163
+ * BLI-3788: the hook COUNTS are merged in here, on the tick, and deliberately
164
+ * not baked into the cached receipt. The five words change about monthly; how
165
+ * often the prompt hook lost its deadline changes every hour, and a number
166
+ * cached for a day would answer yesterday's question. An install receipt this
167
+ * machine could not read means no counts either — there is nowhere to put
168
+ * them — and that case keeps the receipt's own reason.
126
169
  */
127
170
  export async function readHeartbeatMemoryReceipt(options) {
128
171
  const paths = getCollectorRuntimePaths(options.homeDir);
@@ -142,7 +185,20 @@ export async function readHeartbeatMemoryReceipt(options) {
142
185
  return parsed.success ? parsed.data : null;
143
186
  },
144
187
  });
145
- return { receipt: result.receipt, reason: result.reason };
188
+ if (!result.receipt)
189
+ return { receipt: null, reason: result.reason };
190
+ const hooks = readMemoryHookCounts({
191
+ homeDir: options.homeDir ?? os.homedir(),
192
+ ...(options.now ? { now: options.now } : {}),
193
+ });
194
+ return {
195
+ receipt: {
196
+ ...result.receipt,
197
+ ...(hooks.counts ?? {}),
198
+ hook_stats_reason: hooks.reason,
199
+ },
200
+ reason: result.reason,
201
+ };
146
202
  }
147
203
  /**
148
204
  * Sends the heartbeat. Returns whether it landed; never throws.
@@ -248,6 +304,13 @@ export async function sendCollectorHeartbeatBestEffort(options) {
248
304
  // sync.err.log alone.
249
305
  memory_receipt: memory.reason,
250
306
  memory_gaps: memory.receipt ? memoryInstallGaps(memory.receipt) : null,
307
+ // BLI-3788. The hooks are registered (above) AND they either worked or
308
+ // did not (here). A prompt hook that misses its deadline prints
309
+ // nothing to the person, so these counts are the only trace one leaves
310
+ // on this machine; `null` means no counter file, never zero misses.
311
+ memory_hook_runs_24h: memory.receipt?.hook_runs_24h ?? null,
312
+ memory_hook_timeouts_24h: memory.receipt?.hook_timeouts_24h ?? null,
313
+ memory_hook_stats: memory.receipt?.hook_stats_reason ?? null,
251
314
  }));
252
315
  return true;
253
316
  }
@@ -0,0 +1,61 @@
1
+ /**
2
+ * HOW OFTEN DID THE HOOKS MISS? — the reading half (BLI-3788).
3
+ *
4
+ * `bli-memory-mcp` counts every hook run into a counts-only file in this
5
+ * machine's state directory. This module reads it once a sync tick and hands
6
+ * the last 24 hours to the heartbeat, where it rides the memory receipt into
7
+ * `ambient_collector_devices.metadata.memory_install` and out onto `cockpit
8
+ * ops --memory`.
9
+ *
10
+ * Nothing here knows the file's shape: the path, the schema and the windowing
11
+ * rule are `@bli-cockpit/telemetry-core`'s `memory-hook-stats.ts`, which the
12
+ * writing package spends too. That is the whole point of putting them there.
13
+ *
14
+ * ## Which event, and why only one
15
+ *
16
+ * The PROMPT hook. It is the one a person waits on with their sentence typed,
17
+ * the one whose budget QA tick 18 caught it losing, and the one that runs on
18
+ * every turn — so it is the only one whose miss rate means anything as a
19
+ * daily number. The file holds all three; a future gauge that wants
20
+ * SessionStart or Stop reads the same rows with the same function.
21
+ *
22
+ * ## Absent is not zero
23
+ *
24
+ * A machine with no file has no counts and says `hook_stats_absent`. A machine
25
+ * whose file will not parse says `hook_stats_unparseable`. Neither reports
26
+ * zeroes, because a zero here would read as "the hooks ran and never missed",
27
+ * which is the exact false green this ticket exists to remove.
28
+ */
29
+ import fs from "node:fs";
30
+ import { memoryHookStatsFilePath, parseMemoryHookStats, summariseMemoryHookWindow, } from "@bli-cockpit/telemetry-core";
31
+ export function readMemoryHookCounts(options) {
32
+ const readText = options.readText ?? defaultReadText;
33
+ const file = memoryHookStatsFilePath(options.homeDir);
34
+ const raw = readText(file);
35
+ if (raw === null)
36
+ return { counts: null, reason: "hook_stats_absent" };
37
+ const parsed = parseMemoryHookStats(raw);
38
+ if (!parsed.ok)
39
+ return { counts: null, reason: parsed.reason };
40
+ const window = summariseMemoryHookWindow(parsed.file, "prompt", {
41
+ now: options.now ?? new Date(),
42
+ hours: 24,
43
+ });
44
+ return {
45
+ counts: {
46
+ hook_runs_24h: window.runs,
47
+ hook_timeouts_24h: window.timeouts,
48
+ hook_printed_24h: window.printed,
49
+ hook_failed_24h: window.failed,
50
+ },
51
+ reason: "ok",
52
+ };
53
+ }
54
+ function defaultReadText(file) {
55
+ try {
56
+ return fs.readFileSync(file, "utf8");
57
+ }
58
+ catch {
59
+ return null;
60
+ }
61
+ }
@@ -70,6 +70,40 @@ export function renderMemoryUsage(memory, dim) {
70
70
  }
71
71
  return lines;
72
72
  }
73
+ /**
74
+ * The hook half: one line per machine, the server's own words.
75
+ *
76
+ * A machine that reports NO counts is printed too, dimmed and named. It is the
77
+ * whole point: "no measurement" and "no misses" are different facts, and the
78
+ * only reason this section exists is that they used to be indistinguishable.
79
+ */
80
+ export function renderMemoryHooks(hooks, dim) {
81
+ const lines = ["", `HOOKS ${hooks.summary ?? "(no summary)"}`];
82
+ if (hooks.readError) {
83
+ lines.push(` the devices could not be read (${hooks.readError}); nothing is known about whether recalls are arriving`);
84
+ return lines;
85
+ }
86
+ const devices = hooks.devices ?? [];
87
+ if (devices.length === 0) {
88
+ lines.push(dim(" no live collector device to report on"));
89
+ return lines;
90
+ }
91
+ // Worst first: the machine losing recalls is the one somebody acts on, and a
92
+ // machine that reports nothing sinks below both — it is a rollout gap, not a
93
+ // failure.
94
+ const ranked = [...devices].sort((left, right) => {
95
+ const leftKnown = typeof left.runs === "number" ? 0 : 1;
96
+ const rightKnown = typeof right.runs === "number" ? 0 : 1;
97
+ if (leftKnown !== rightKnown)
98
+ return leftKnown - rightKnown;
99
+ return (right.timeouts ?? 0) - (left.timeouts ?? 0);
100
+ });
101
+ for (const device of ranked) {
102
+ const line = ` ${device.line ?? `${device.displayName ?? "?"}: (no line)`}`;
103
+ lines.push((device.timeouts ?? 0) > 0 ? line : dim(line));
104
+ }
105
+ return lines;
106
+ }
73
107
  /** The two sources that mean a PERSON's agent used memory. Never the import. */
74
108
  function agentSaves(counts) {
75
109
  return (counts?.mcp ?? 0) + (counts?.stop_hook ?? 0);
@@ -12,7 +12,7 @@
12
12
  * artifact does and does not prove — because that is exactly the moment
13
13
  * somebody is about to conclude something from it.
14
14
  */
15
- export { renderMemoryUsage } from "./ops-render-memory.js";
15
+ export { renderMemoryHooks, renderMemoryUsage } from "./ops-render-memory.js";
16
16
  /** The word a person reads. Short, fixed width, and never a bare colour. */
17
17
  export function verdictWord(verdict) {
18
18
  switch (verdict) {
@@ -25,7 +25,7 @@
25
25
  */
26
26
  import { colorEnabled, dim, writeLine } from "./cli-io.js";
27
27
  import { renderOpsStatus } from "./ops-render.js";
28
- import { renderMemoryUsage, } from "./ops-render-memory.js";
28
+ import { renderMemoryHooks, renderMemoryUsage, } from "./ops-render-memory.js";
29
29
  import { asRecord, callTower, openTower, writeCommandFailure } from "./tower-command.js";
30
30
  /**
31
31
  * A whole compile, plus a little. `maxDuration` on `/api/ops/recompile` is 800
@@ -79,7 +79,7 @@ async function runOpsStatus(command, io, tower) {
79
79
  // BLI-3762: a job that ran and wrote less than it owed is not healthy.
80
80
  row.verdict === "degraded");
81
81
  if (command.json) {
82
- writeLine(io.stdout, JSON.stringify(memory ? { ...payload, memory: memory.section } : payload));
82
+ writeLine(io.stdout, JSON.stringify(memory ? { ...payload, memory: memory.section, hooks: memory.hooks } : payload));
83
83
  }
84
84
  else {
85
85
  const styled = colorEnabled(io);
@@ -91,7 +91,16 @@ async function runOpsStatus(command, io, tower) {
91
91
  writeLine(io.stdout, line);
92
92
  }
93
93
  }
94
- else if (memory) {
94
+ // BLI-3788. Printed whenever the door sent it, including when the usage
95
+ // half could not be read: "are the recalls arriving" and "is anybody
96
+ // saving" are two questions, and one being unreadable does not silence
97
+ // the other.
98
+ if (memory?.hooks) {
99
+ for (const line of renderMemoryHooks(memory.hooks, (text) => dim(text, styled))) {
100
+ writeLine(io.stdout, line);
101
+ }
102
+ }
103
+ if (memory && !memory.section) {
95
104
  // Never a silent gap where a section was asked for: the reason the gauge
96
105
  // could not be read is printed where the gauge would have been.
97
106
  writeLine(io.stdout, "");
@@ -119,6 +128,12 @@ async function runOpsStatus(command, io, tower) {
119
128
  memory_agent_saves: memory?.section
120
129
  ? (memory.section.counts?.mcp ?? 0) + (memory.section.counts?.stop_hook ?? 0)
121
130
  : null,
131
+ // BLI-3788: whether the per-turn recalls arrived, on the same line. A
132
+ // null here is "this dashboard does not report it yet", never zero
133
+ // misses.
134
+ memory_hook_runs_24h: memory?.hooks?.totals?.runs ?? null,
135
+ memory_hook_timeouts_24h: memory?.hooks?.totals?.timeouts ?? null,
136
+ memory_hook_devices_reporting: memory?.hooks?.reporting ?? null,
122
137
  })}`);
123
138
  return unhealthy.length > 0 ? 1 : 0;
124
139
  }
@@ -144,12 +159,15 @@ async function readMemoryUsage(command, io, tower) {
144
159
  reason: result.reason,
145
160
  http_status: result.httpStatus ?? null,
146
161
  })}`);
147
- return { section: null, reason: result.reason };
162
+ return { section: null, hooks: null, reason: result.reason };
148
163
  }
149
164
  const body = asRecord(result.body);
165
+ // BLI-3788: the hook counts ride the same answer, and their absence is a
166
+ // fact about the SERVER's version rather than about this fleet — an older
167
+ // dashboard sends no `hooks` key, and printing nothing is right there.
150
168
  if (!body.memory)
151
- return { section: null, reason: "memory_section_absent" };
152
- return { section: body.memory, reason: "ok" };
169
+ return { section: null, hooks: body.hooks ?? null, reason: "memory_section_absent" };
170
+ return { section: body.memory, hooks: body.hooks ?? null, reason: "ok" };
153
171
  }
154
172
  async function runOpsRecompile(command, io, tower) {
155
173
  const person = command.person ?? "";
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
15
15
  }
16
16
 
17
17
  if (command === "--version" || command === "-V" || command === "version") {
18
- writeLine(io?.stdout ?? process.stdout, "0.2.69");
18
+ writeLine(io?.stdout ?? process.stdout, "0.2.72");
19
19
  return 0;
20
20
  }
21
21
 
@@ -72,7 +72,14 @@ export async function runStatus(command, io) {
72
72
  writeConnectedBlock(io, setupReceipt);
73
73
  return 0;
74
74
  }
75
- const status = await inspectLocalCollectorStatus(command);
75
+ // The staging walk is opt-in (BLI-3797) and this is the surface that pays for
76
+ // it: one person asking one question. The multi-repo branch above deliberately
77
+ // does not — the backlog is a MACHINE fact, so N repos would walk one tree N
78
+ // times to print the same number.
79
+ const status = await inspectLocalCollectorStatus({
80
+ ...command,
81
+ includeStagingBacklog: true,
82
+ });
76
83
  if (command.json) {
77
84
  writeLine(io.stdout, JSON.stringify({
78
85
  ...status,
@@ -2,7 +2,8 @@
2
2
  * What the sync tick does AFTER collection's own outcome is decided and
3
3
  * reported: keep this machine's CLI current on npm `latest` (BLI-2601), put a
4
4
  * broken scheduler registration back (BLI-2721), keep BLI Memory registered
5
- * with both agent hosts (BLI-3580), and stop the laptop filling up (BLI-3619).
5
+ * with both agent hosts (BLI-3580), re-offer staged evidence that never landed
6
+ * (BLI-3797), and stop the laptop filling up (BLI-3619).
6
7
  *
7
8
  * Split out of commands/sync.ts (BLI-3578), moved verbatim. They belong
8
9
  * together because they share one rule, and it is the reason both are called
@@ -20,6 +21,7 @@ import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-sta
20
21
  import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
21
22
  import { runStagingPrune } from "../disk-prune.js";
22
23
  import { runEvidenceReconcile, } from "../evidence-reconcile-client.js";
24
+ import { runEvidenceRedelivery, } from "../evidence-redelivery.js";
23
25
  import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
24
26
  /** How many reconcile batches (of up to 500 hashes each) one sync tick may spend. */
25
27
  const RECONCILE_BATCHES_PER_TICK = 1;
@@ -215,12 +217,20 @@ function memoryInstallEvent(outcome) {
215
217
  * costs one cheap local read and no network call at all, so it never competes
216
218
  * with collection for the tick's time. A reconcile failure never blocks the
217
219
  * prune that follows it.
220
+ *
221
+ * BLI-3797 sits between them: the reconcile has just established which staged
222
+ * objects the SERVER says it never received, so the redelivery drain re-offers
223
+ * exactly those, bounded, before the prune runs and can delete whatever landed.
224
+ * Neither of the two can block the prune, and none of the three can fail a tick.
218
225
  */
219
226
  export async function runStagingPruneAfterSync(command, io, dashboardUrl, options = {}) {
220
227
  const events = [];
221
228
  const reconciled = await runReconcileFollowUp(command, io, dashboardUrl, options);
222
229
  if (reconciled)
223
230
  events.push(reconcileEvent(reconciled));
231
+ const redelivered = await runRedeliveryFollowUp(command, io, dashboardUrl, options);
232
+ if (redelivered)
233
+ events.push(redeliveryEvent(redelivered));
224
234
  let result;
225
235
  try {
226
236
  result = await runStagingPrune(getCollectorRuntimePaths(command.homeDir), {
@@ -269,6 +279,59 @@ async function runReconcileFollowUp(command, io, dashboardUrl, options) {
269
279
  });
270
280
  return result.reason === "nothing_unknown" ? null : result;
271
281
  }
282
+ /**
283
+ * BLI-3797, between the reconcile above and the prune below, and in that order
284
+ * for a reason: reconcile turns `unknown` into a server-backed answer, this
285
+ * re-offers what that answer says never landed, and the prune then deletes
286
+ * whatever this just made durable. Running the drain first would ask the server
287
+ * about hashes it is about to be told the truth about; running it after the
288
+ * prune would leave a tick's worth of freed cap unused.
289
+ *
290
+ * Never throws (`runEvidenceRedelivery` already never does); returns `null` for
291
+ * the boring, common case — nothing on this disk is undelivered — so the steady
292
+ * state costs no receipt, the same rule the reconcile follow-up above and the
293
+ * prune's daily throttle below both follow.
294
+ */
295
+ async function runRedeliveryFollowUp(command, io, dashboardUrl, options) {
296
+ const result = await runEvidenceRedelivery({
297
+ homeDir: command.homeDir,
298
+ dashboardUrl,
299
+ env: io.env,
300
+ fetch: io.fetch,
301
+ ...(options.now ? { now: options.now } : {}),
302
+ });
303
+ return result.reason === "nothing_uncommitted" ? null : result;
304
+ }
305
+ /** Counts and byte totals only; no pack id and no hash travels in a receipt. */
306
+ function redeliveryEvent(result) {
307
+ const detail = [
308
+ `offered ${result.offered} object(s), ${result.offered_bytes}B, across ${result.packs} pack(s)`,
309
+ `uploaded ${result.uploaded} (${result.uploaded_bytes}B), reused ${result.reused}, failed ${result.failed}`,
310
+ `held ${result.held}, deferred ${result.deferred} (${result.deferred_bytes}B)`,
311
+ result.failure_reasons.length > 0
312
+ ? `failure_reasons ${result.failure_reasons.join(",")}`
313
+ : null,
314
+ ]
315
+ .filter((part) => Boolean(part))
316
+ .join("; ");
317
+ if (result.status === "fail") {
318
+ return {
319
+ step: "evidence_redelivery",
320
+ status: "fail",
321
+ error_code: result.reason,
322
+ error_detail: detail,
323
+ };
324
+ }
325
+ if (result.status === "skipped") {
326
+ return {
327
+ step: "evidence_redelivery",
328
+ status: "skipped",
329
+ error_code: result.reason,
330
+ error_detail: detail,
331
+ };
332
+ }
333
+ return { step: "evidence_redelivery", status: "ok", error_detail: detail };
334
+ }
272
335
  function prunedNothing() {
273
336
  return {
274
337
  status: "skipped",
@@ -2,7 +2,7 @@ import { writeLine } from "./cli-io.js";
2
2
  import { attributedSyncRunStatus, cursorStatusLine, displayTicketId, rawEvidenceSyncLine, shortSha, worktreeSyncRow, writeAgentSessionSummary, } from "./collection-report.js";
3
3
  import { collectionRootConsentAliases } from "./collection-roots.js";
4
4
  import { discoverCommandWorktrees } from "./local-discovery.js";
5
- import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
5
+ import { sendCollectorHeartbeatBestEffort, readHeartbeatStagingFacts, } from "./heartbeat.js";
6
6
  import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
7
7
  import { runAttributedWorktreeSync, } from "./session-sync.js";
8
8
  import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
@@ -98,11 +98,22 @@ export async function runSync(command, io) {
98
98
  */
99
99
  async function sendSyncHeartbeat(command, io, dashboardUrl, facts) {
100
100
  const roots = await resolveSyncCollectionRoots(command).catch(() => []);
101
+ // BLI-3797: the backlog figure rides the same check-in as the root labels, so
102
+ // `cockpit ops` learns about undelivered evidence on the SAME tick that proves
103
+ // the machine is alive. Best-effort: nulls omit the fields rather than
104
+ // reporting a zero nobody measured.
105
+ const staging = await readHeartbeatStagingFacts({
106
+ homeDir: command.homeDir,
107
+ }).catch(() => ({ bytes: null, reason: null }));
101
108
  await sendCollectorHeartbeatBestEffort({
102
109
  homeDir: command.homeDir,
103
110
  dashboardUrl,
104
111
  roots,
105
- facts,
112
+ facts: {
113
+ ...facts,
114
+ stagingUncommittedBytes: staging.bytes,
115
+ stagingUncommittedReason: staging.reason,
116
+ },
106
117
  io,
107
118
  }).catch((error) => {
108
119
  // The sender already swallows everything it knows about; this is the net
@@ -131,7 +131,11 @@ function pushDeletion(plan, object, reason) {
131
131
  function addToBucket(bucket, object) {
132
132
  bucket.count += 1;
133
133
  bucket.bytes += object.byte_size;
134
- bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.age_ms);
134
+ // Report the age of the BYTES, not of the last answer about them (BLI-3797).
135
+ // `age_ms` on a reconciled object is the age of the reconcile reply, which
136
+ // made `oldest_uncommitted_age_ms` read 0.94 days for a 45-day-old backlog.
137
+ // Deletion still keys off `age_ms` above — this bucket is report-only.
138
+ bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.disk_age_ms || object.age_ms);
135
139
  }
136
140
  function removeFromBucket(bucket, object) {
137
141
  bucket.count -= 1;