@bli-cockpit/cli 0.2.70 → 0.2.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,59 @@
1
+ export function days(ms) {
2
+ return (ms / 86_400_000).toFixed(1);
3
+ }
4
+ /**
5
+ * What the drain did, in the sentence a person reads (BLI-3797).
6
+ *
7
+ * Until this ticket the row said "run `cockpit sync` to deliver them" about
8
+ * objects no sync would ever offer again: the live sync only sends what it
9
+ * collected THIS tick, so 1.8 GB of orphaned evidence sat behind a fix sentence
10
+ * that could not come true. The drain is real now, `--fix` spends it here, and
11
+ * the row reports what it actually managed rather than issuing an instruction.
12
+ */
13
+ export function redeliveryLine(result) {
14
+ if (result.reason === "no_device_session") {
15
+ return "they cannot be re-offered until this machine signs in — run `cockpit login`";
16
+ }
17
+ if (result.reason === "disabled") {
18
+ return "re-delivery is switched off on this machine (COCKPIT_DISABLE_EVIDENCE_REDELIVERY=1)";
19
+ }
20
+ if (result.uploaded > 0 || result.reused > 0) {
21
+ return `re-delivered ${result.uploaded + result.reused} of them (${mib(result.uploaded_bytes)} MB); ${result.deferred} left for the next tick`;
22
+ }
23
+ if (result.failed > 0) {
24
+ return `re-offering them was refused: ${result.failure_reasons.join(", ") || "unknown"} — the next tick retries under backoff`;
25
+ }
26
+ if (result.held > 0) {
27
+ return `${result.held} are held by delivery backoff and retry on a later tick`;
28
+ }
29
+ return "the next `cockpit sync` re-offers them";
30
+ }
31
+ export function diskRowMessage(footprint, capBytes) {
32
+ const staging = footprint.staging;
33
+ const parts = [
34
+ `staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
35
+ ];
36
+ // BLI-3797: "1,916.9 uncommitted" is a number, not a finding. The row now
37
+ // names WHY the biggest share of it has not landed and HOW LONG those bytes
38
+ // have been here — on the reference Mac the answer was
39
+ // `reconciled_unknown_to_server` on 234 objects whose oldest was 45 days old,
40
+ // which is a different problem from a delivery that failed this morning.
41
+ const dominant = staging.dominant_uncommitted_reason;
42
+ if (dominant) {
43
+ parts.push(`oldest undelivered ${days(dominant.oldest_disk_age_ms)}d; biggest reason ${dominant.reason} on ${dominant.count} object(s), ${mib(dominant.bytes)} MB`);
44
+ }
45
+ // BLI-3619's second half: "unknown" means the local ledger's own capped
46
+ // memory cannot say, never that delivery failed — and the one command that
47
+ // actually answers it is named right here, not left for a person to find.
48
+ if (staging.unknown_count > 0) {
49
+ parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
50
+ }
51
+ parts.push(`logs ${mib(footprint.logs.total_bytes)} MB`, `spool ${mib(footprint.spool_bytes)} MB`);
52
+ for (const vault of footprint.vaults) {
53
+ parts.push(`${vault.name} ${mib(vault.byte_size)} MB (a one-off; \`cockpit clean --all-committed\` removes it only if every file in it is accepted)`);
54
+ }
55
+ return parts.join("; ");
56
+ }
57
+ export function mib(bytes) {
58
+ return (bytes / (1024 * 1024)).toFixed(1);
59
+ }
@@ -7,9 +7,11 @@ import { describeError } from "../health-detail.js";
7
7
  import { runStagingPrune } from "../disk-prune.js";
8
8
  import { retentionOptionsFromEnv } from "../disk-retention.js";
9
9
  import { readDiskFootprint } from "../disk-usage.js";
10
+ import { runEvidenceRedelivery } from "../evidence-redelivery.js";
10
11
  import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
11
12
  import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
12
13
  import { runBackfillCommand } from "./backfill.js";
14
+ import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
13
15
  import { doctorRoots } from "./doctor-access.js";
14
16
  import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
15
17
  /**
@@ -126,6 +128,18 @@ export async function checkDiskState(context) {
126
128
  }
127
129
  export async function fixDiskState(context) {
128
130
  const paths = getCollectorRuntimePaths(context.command.homeDir);
131
+ // BLI-3797, BEFORE the prune: the prune can only delete what the ledger
132
+ // vouches for, so on a machine held over the cap by undelivered evidence it
133
+ // frees nothing at all. Deliver first, then sweep what delivering made
134
+ // durable. This never throws and never decides the row's verdict — a drain
135
+ // that could not run leaves the prune to do exactly what it did before.
136
+ const redelivery = await runEvidenceRedelivery({
137
+ homeDir: context.command.homeDir,
138
+ dashboardUrl: context.command.dashboardUrl,
139
+ env: context.io.env,
140
+ fetch: context.io.fetch,
141
+ });
142
+ const redelivered = redeliveryLine(redelivery);
129
143
  const pruned = await runStagingPrune(paths, {
130
144
  env: context.io.env,
131
145
  force: true,
@@ -144,27 +158,7 @@ export async function fixDiskState(context) {
144
158
  // Deliberately still `ok`: staging over the cap because evidence has not been
145
159
  // accepted yet is the collector working, not a machine to repair. The row
146
160
  // names the blockage and the one command that goes further.
147
- return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest — run \`cockpit sync\` to deliver them, or \`cockpit clean --all-committed\` to drop every accepted copy now`);
148
- }
149
- function diskRowMessage(footprint, capBytes) {
150
- const staging = footprint.staging;
151
- const parts = [
152
- `staging ${mib(staging.total_bytes)} MB (${mib(staging.committed_bytes)} committed / ${mib(staging.uncommitted_bytes)} uncommitted / ${mib(staging.unknown_bytes)} unknown) against a ${mib(capBytes)} MB cap`,
153
- ];
154
- // BLI-3619's second half: "unknown" means the local ledger's own capped
155
- // memory cannot say, never that delivery failed — and the one command that
156
- // actually answers it is named right here, not left for a person to find.
157
- if (staging.unknown_count > 0) {
158
- parts.push(`${staging.unknown_count} object(s) unknown to this laptop's own ledger — run \`cockpit clean --reconcile\` to ask the server`);
159
- }
160
- parts.push(`logs ${mib(footprint.logs.total_bytes)} MB`, `spool ${mib(footprint.spool_bytes)} MB`);
161
- for (const vault of footprint.vaults) {
162
- parts.push(`${vault.name} ${mib(vault.byte_size)} MB (a one-off; \`cockpit clean --all-committed\` removes it only if every file in it is accepted)`);
163
- }
164
- return parts.join("; ");
165
- }
166
- function mib(bytes) {
167
- return (bytes / (1024 * 1024)).toFixed(1);
161
+ return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest — ${redelivered} or \`cockpit clean --all-committed\` to drop every accepted copy now`);
168
162
  }
169
163
  function capturedIo(io, forward) {
170
164
  const stdoutChunks = [];
@@ -25,6 +25,7 @@ import os from "node:os";
25
25
  import path from "node:path";
26
26
  import { COLLECTOR_HEARTBEAT_SCHEMA_VERSION, MemoryInstallReceiptSchema, memoryInstallGaps, setupReceiptGaps, } from "@bli-cockpit/telemetry-core";
27
27
  import { readCachedSetupReceipt } from "./setup-receipt.js";
28
+ import { readStagingInventory } from "../disk-usage.js";
28
29
  import { describeError } from "../health-detail.js";
29
30
  import { shouldSuppressFleetReceipts, } from "../dev-build.js";
30
31
  import { getCollectorRuntimePaths, readLocalCollectorSessionFile, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
@@ -111,10 +112,44 @@ export function buildCollectorHeartbeat(options) {
111
112
  ...(typeof options.facts.sessionsPendingUpload === "number"
112
113
  ? { sessions_pending_upload: options.facts.sessionsPendingUpload }
113
114
  : {}),
115
+ ...(typeof options.facts.stagingUncommittedBytes === "number"
116
+ ? { staging_uncommitted_bytes: options.facts.stagingUncommittedBytes }
117
+ : {}),
118
+ ...(options.facts.stagingUncommittedReason
119
+ ? { staging_uncommitted_reason: options.facts.stagingUncommittedReason }
120
+ : {}),
114
121
  ...(options.memoryInstall ? { memory_install: options.memoryInstall } : {}),
115
122
  ...(options.setupReceipt ? { setup_receipt: options.setupReceipt } : {}),
116
123
  };
117
124
  }
125
+ /**
126
+ * What this machine is still holding that never reached storage (BLI-3797).
127
+ *
128
+ * One extra staging walk per tick, on purpose: `cockpit ops` had no way to see
129
+ * that a green, syncing, up-to-date laptop was sitting on 1.8 GB of evidence the
130
+ * server had never received, and a fleet-wide failure nobody can see is the
131
+ * BLI-2528 shape exactly. The walk is bounded by the pack tree the prune keeps
132
+ * under a 2 GB cap, and it is best-effort: a read that fails returns nulls and
133
+ * says so, so the heartbeat omits both fields rather than claiming zero.
134
+ */
135
+ export async function readHeartbeatStagingFacts(options) {
136
+ const paths = getCollectorRuntimePaths(options.homeDir);
137
+ try {
138
+ const inventory = await readStagingInventory(paths, options.now ?? new Date());
139
+ const dominant = inventory.dominant_uncommitted_reason;
140
+ return {
141
+ bytes: inventory.uncommitted_bytes,
142
+ reason: dominant ? `${dominant.reason}:${dominant.count}`.slice(0, 120) : null,
143
+ };
144
+ }
145
+ catch (error) {
146
+ console.error("[heartbeat] the staging tree could not be read; this tick reports no backlog figure", JSON.stringify({
147
+ reason: "staging_inventory_unreadable",
148
+ ...describeError(error),
149
+ }));
150
+ return { bytes: null, reason: null };
151
+ }
152
+ }
118
153
  /**
119
154
  * The cached memory receipt for this tick, or null with a named reason
120
155
  * (BLI-3729).
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
15
15
  }
16
16
 
17
17
  if (command === "--version" || command === "-V" || command === "version") {
18
- writeLine(io?.stdout ?? process.stdout, "0.2.70");
18
+ writeLine(io?.stdout ?? process.stdout, "0.2.72");
19
19
  return 0;
20
20
  }
21
21
 
@@ -72,7 +72,14 @@ export async function runStatus(command, io) {
72
72
  writeConnectedBlock(io, setupReceipt);
73
73
  return 0;
74
74
  }
75
- const status = await inspectLocalCollectorStatus(command);
75
+ // The staging walk is opt-in (BLI-3797) and this is the surface that pays for
76
+ // it: one person asking one question. The multi-repo branch above deliberately
77
+ // does not — the backlog is a MACHINE fact, so N repos would walk one tree N
78
+ // times to print the same number.
79
+ const status = await inspectLocalCollectorStatus({
80
+ ...command,
81
+ includeStagingBacklog: true,
82
+ });
76
83
  if (command.json) {
77
84
  writeLine(io.stdout, JSON.stringify({
78
85
  ...status,
@@ -2,7 +2,8 @@
2
2
  * What the sync tick does AFTER collection's own outcome is decided and
3
3
  * reported: keep this machine's CLI current on npm `latest` (BLI-2601), put a
4
4
  * broken scheduler registration back (BLI-2721), keep BLI Memory registered
5
- * with both agent hosts (BLI-3580), and stop the laptop filling up (BLI-3619).
5
+ * with both agent hosts (BLI-3580), re-offer staged evidence that never landed
6
+ * (BLI-3797), and stop the laptop filling up (BLI-3619).
6
7
  *
7
8
  * Split out of commands/sync.ts (BLI-3578), moved verbatim. They belong
8
9
  * together because they share one rule, and it is the reason both are called
@@ -20,6 +21,7 @@ import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-sta
20
21
  import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
21
22
  import { runStagingPrune } from "../disk-prune.js";
22
23
  import { runEvidenceReconcile, } from "../evidence-reconcile-client.js";
24
+ import { runEvidenceRedelivery, } from "../evidence-redelivery.js";
23
25
  import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
24
26
  /** How many reconcile batches (of up to 500 hashes each) one sync tick may spend. */
25
27
  const RECONCILE_BATCHES_PER_TICK = 1;
@@ -215,12 +217,20 @@ function memoryInstallEvent(outcome) {
215
217
  * costs one cheap local read and no network call at all, so it never competes
216
218
  * with collection for the tick's time. A reconcile failure never blocks the
217
219
  * prune that follows it.
220
+ *
221
+ * BLI-3797 sits between them: the reconcile has just established which staged
222
+ * objects the SERVER says it never received, so the redelivery drain re-offers
223
+ * exactly those, bounded, before the prune runs and can delete whatever landed.
224
+ * Neither of the two can block the prune, and none of the three can fail a tick.
218
225
  */
219
226
  export async function runStagingPruneAfterSync(command, io, dashboardUrl, options = {}) {
220
227
  const events = [];
221
228
  const reconciled = await runReconcileFollowUp(command, io, dashboardUrl, options);
222
229
  if (reconciled)
223
230
  events.push(reconcileEvent(reconciled));
231
+ const redelivered = await runRedeliveryFollowUp(command, io, dashboardUrl, options);
232
+ if (redelivered)
233
+ events.push(redeliveryEvent(redelivered));
224
234
  let result;
225
235
  try {
226
236
  result = await runStagingPrune(getCollectorRuntimePaths(command.homeDir), {
@@ -269,6 +279,59 @@ async function runReconcileFollowUp(command, io, dashboardUrl, options) {
269
279
  });
270
280
  return result.reason === "nothing_unknown" ? null : result;
271
281
  }
282
+ /**
283
+ * BLI-3797, between the reconcile above and the prune below, and in that order
284
+ * for a reason: reconcile turns `unknown` into a server-backed answer, this
285
+ * re-offers what that answer says never landed, and the prune then deletes
286
+ * whatever this just made durable. Running the drain first would ask the server
287
+ * about hashes it is about to be told the truth about; running it after the
288
+ * prune would leave a tick's worth of freed cap unused.
289
+ *
290
+ * Never throws (`runEvidenceRedelivery` already never does); returns `null` for
291
+ * the boring, common case — nothing on this disk is undelivered — so the steady
292
+ * state costs no receipt, the same rule the reconcile follow-up above and the
293
+ * prune's daily throttle below both follow.
294
+ */
295
+ async function runRedeliveryFollowUp(command, io, dashboardUrl, options) {
296
+ const result = await runEvidenceRedelivery({
297
+ homeDir: command.homeDir,
298
+ dashboardUrl,
299
+ env: io.env,
300
+ fetch: io.fetch,
301
+ ...(options.now ? { now: options.now } : {}),
302
+ });
303
+ return result.reason === "nothing_uncommitted" ? null : result;
304
+ }
305
+ /** Counts and byte totals only; no pack id and no hash travels in a receipt. */
306
+ function redeliveryEvent(result) {
307
+ const detail = [
308
+ `offered ${result.offered} object(s), ${result.offered_bytes}B, across ${result.packs} pack(s)`,
309
+ `uploaded ${result.uploaded} (${result.uploaded_bytes}B), reused ${result.reused}, failed ${result.failed}`,
310
+ `held ${result.held}, deferred ${result.deferred} (${result.deferred_bytes}B)`,
311
+ result.failure_reasons.length > 0
312
+ ? `failure_reasons ${result.failure_reasons.join(",")}`
313
+ : null,
314
+ ]
315
+ .filter((part) => Boolean(part))
316
+ .join("; ");
317
+ if (result.status === "fail") {
318
+ return {
319
+ step: "evidence_redelivery",
320
+ status: "fail",
321
+ error_code: result.reason,
322
+ error_detail: detail,
323
+ };
324
+ }
325
+ if (result.status === "skipped") {
326
+ return {
327
+ step: "evidence_redelivery",
328
+ status: "skipped",
329
+ error_code: result.reason,
330
+ error_detail: detail,
331
+ };
332
+ }
333
+ return { step: "evidence_redelivery", status: "ok", error_detail: detail };
334
+ }
272
335
  function prunedNothing() {
273
336
  return {
274
337
  status: "skipped",
@@ -2,7 +2,7 @@ import { writeLine } from "./cli-io.js";
2
2
  import { attributedSyncRunStatus, cursorStatusLine, displayTicketId, rawEvidenceSyncLine, shortSha, worktreeSyncRow, writeAgentSessionSummary, } from "./collection-report.js";
3
3
  import { collectionRootConsentAliases } from "./collection-roots.js";
4
4
  import { discoverCommandWorktrees } from "./local-discovery.js";
5
- import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
5
+ import { sendCollectorHeartbeatBestEffort, readHeartbeatStagingFacts, } from "./heartbeat.js";
6
6
  import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
7
7
  import { runAttributedWorktreeSync, } from "./session-sync.js";
8
8
  import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
@@ -98,11 +98,22 @@ export async function runSync(command, io) {
98
98
  */
99
99
  async function sendSyncHeartbeat(command, io, dashboardUrl, facts) {
100
100
  const roots = await resolveSyncCollectionRoots(command).catch(() => []);
101
+ // BLI-3797: the backlog figure rides the same check-in as the root labels, so
102
+ // `cockpit ops` learns about undelivered evidence on the SAME tick that proves
103
+ // the machine is alive. Best-effort: nulls omit the fields rather than
104
+ // reporting a zero nobody measured.
105
+ const staging = await readHeartbeatStagingFacts({
106
+ homeDir: command.homeDir,
107
+ }).catch(() => ({ bytes: null, reason: null }));
101
108
  await sendCollectorHeartbeatBestEffort({
102
109
  homeDir: command.homeDir,
103
110
  dashboardUrl,
104
111
  roots,
105
- facts,
112
+ facts: {
113
+ ...facts,
114
+ stagingUncommittedBytes: staging.bytes,
115
+ stagingUncommittedReason: staging.reason,
116
+ },
106
117
  io,
107
118
  }).catch((error) => {
108
119
  // The sender already swallows everything it knows about; this is the net
@@ -131,7 +131,11 @@ function pushDeletion(plan, object, reason) {
131
131
  function addToBucket(bucket, object) {
132
132
  bucket.count += 1;
133
133
  bucket.bytes += object.byte_size;
134
- bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.age_ms);
134
+ // Report the age of the BYTES, not of the last answer about them (BLI-3797).
135
+ // `age_ms` on a reconciled object is the age of the reconcile reply, which
136
+ // made `oldest_uncommitted_age_ms` read 0.94 days for a 45-day-old backlog.
137
+ // Deletion still keys off `age_ms` above — this bucket is report-only.
138
+ bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.disk_age_ms || object.age_ms);
135
139
  }
136
140
  function removeFromBucket(bucket, object) {
137
141
  bucket.count -= 1;
@@ -104,6 +104,7 @@ export async function readStagingInventory(paths, now = new Date()) {
104
104
  inventory.packs.push(pack);
105
105
  accumulatePack(inventory, pack);
106
106
  }
107
+ rankUncommittedReasons(inventory);
107
108
  return inventory;
108
109
  }
109
110
  /**
@@ -150,15 +151,23 @@ async function readPack(dir, packId, options) {
150
151
  manifest_present: manifest !== null,
151
152
  };
152
153
  for (const relative of await listPackFiles(dir)) {
153
- const size = await fs
154
- .stat(path.join(dir, relative))
155
- .then((info) => (info.isFile() ? info.size : 0))
156
- .catch(() => 0);
154
+ // One stat, two facts: how big the staged copy is and WHEN it was written.
155
+ // The mtime is the only unfalsifiable record of how long undelivered
156
+ // evidence has been sitting here (BLI-3797) every other timestamp on the
157
+ // machine records when something last had an opinion about it.
158
+ const info = await fs.stat(path.join(dir, relative)).catch(() => null);
159
+ const size = info?.isFile() ? info.size : 0;
157
160
  if (relative === "manifest.json") {
158
161
  pack.manifest_bytes += size;
159
162
  continue;
160
163
  }
161
- pack.objects.push(await classifyObject(dir, packId, relative, size, manifest, options));
164
+ const stagedOnDiskAt = info && Number.isFinite(info.mtimeMs)
165
+ ? new Date(info.mtimeMs).toISOString()
166
+ : null;
167
+ pack.objects.push(await classifyObject(dir, packId, relative, size, manifest, {
168
+ ...options,
169
+ stagedOnDiskAt,
170
+ }));
162
171
  }
163
172
  return pack;
164
173
  }
@@ -167,7 +176,13 @@ async function classifyObject(dir, packId, relative, size, manifest, options) {
167
176
  (await hashUnnamedFile(path.join(dir, relative), size, options.budget));
168
177
  const stagedAt = options.stagedAt.get(`${packId}/${relative}`) ?? null;
169
178
  const committedAt = hash ? options.ledger.committed.get(hash) : undefined;
170
- const base = { pack_id: packId, relative_path: relative, byte_size: size };
179
+ const base = {
180
+ pack_id: packId,
181
+ relative_path: relative,
182
+ byte_size: size,
183
+ staged_on_disk_at: options.stagedOnDiskAt,
184
+ disk_age_ms: ageMs(options.stagedOnDiskAt, options.now),
185
+ };
171
186
  if (committedAt) {
172
187
  return {
173
188
  ...base,
@@ -359,13 +374,49 @@ function accumulatePack(inventory, pack) {
359
374
  }
360
375
  inventory.uncommitted_count += 1;
361
376
  inventory.uncommitted_bytes += object.byte_size;
362
- if (object.decided_at &&
377
+ tallyUncommittedReason(inventory, object);
378
+ // The age of the BYTES, not of the answer about them (BLI-3797). Falls back
379
+ // to `decided_at` only when the file has no readable mtime, which is the
380
+ // one case where the disk itself cannot say.
381
+ const stagedAt = object.staged_on_disk_at ?? object.decided_at;
382
+ if (stagedAt &&
363
383
  (!inventory.oldest_uncommitted_at ||
364
- object.decided_at.localeCompare(inventory.oldest_uncommitted_at) < 0)) {
365
- inventory.oldest_uncommitted_at = object.decided_at;
384
+ stagedAt.localeCompare(inventory.oldest_uncommitted_at) < 0)) {
385
+ inventory.oldest_uncommitted_at = stagedAt;
366
386
  }
367
387
  }
368
388
  }
389
+ /**
390
+ * One more object against its reason label. Kept as a mutable array rather than
391
+ * a Map so the inventory stays a plain serializable value — it rides `--json`
392
+ * output and a receipt, and a Map would silently become `{}` in both.
393
+ */
394
+ function tallyUncommittedReason(inventory, object) {
395
+ const existing = inventory.uncommitted_reasons.find((tally) => tally.reason === object.reason);
396
+ const tally = existing ?? {
397
+ reason: object.reason,
398
+ count: 0,
399
+ bytes: 0,
400
+ oldest_disk_age_ms: 0,
401
+ };
402
+ tally.count += 1;
403
+ tally.bytes += object.byte_size;
404
+ tally.oldest_disk_age_ms = Math.max(tally.oldest_disk_age_ms, object.disk_age_ms || object.age_ms);
405
+ if (!existing)
406
+ inventory.uncommitted_reasons.push(tally);
407
+ }
408
+ /**
409
+ * Rank the reasons by BYTES, then name the biggest one.
410
+ *
411
+ * Bytes, not object count, because the question the ranking answers is "what is
412
+ * holding this laptop over its cap" — 234 objects at 1,836 MB and 17 at 81 MB
413
+ * are not the same finding, and counting rows would have called them close.
414
+ */
415
+ function rankUncommittedReasons(inventory) {
416
+ inventory.uncommitted_reasons.sort((left, right) => right.bytes - left.bytes || right.count - left.count);
417
+ inventory.dominant_uncommitted_reason =
418
+ inventory.uncommitted_reasons[0] ?? null;
419
+ }
369
420
  function emptyStagingInventory() {
370
421
  return {
371
422
  pack_count: 0,
@@ -376,6 +427,8 @@ function emptyStagingInventory() {
376
427
  uncommitted_count: 0,
377
428
  uncommitted_bytes: 0,
378
429
  oldest_uncommitted_at: null,
430
+ uncommitted_reasons: [],
431
+ dominant_uncommitted_reason: null,
379
432
  unknown_count: 0,
380
433
  unknown_bytes: 0,
381
434
  manifest_bytes: 0,
@@ -0,0 +1,277 @@
1
+ /**
2
+ * Rebuilding an upload offer for evidence that is already on this disk.
3
+ *
4
+ * BLI-3797. The live sync only ever offers what THIS tick collected: it reads
5
+ * a transcript, hashes it, stages a copy, uploads that copy, and moves on. If
6
+ * the upload never happens — the process is killed between staging and
7
+ * delivery, the envelope is refused, the source file grows so next tick mints a
8
+ * different hash into a different pack — the staged bytes are orphaned. Nothing
9
+ * offers them again, ever. `disk-retention.ts` will not delete them (correctly:
10
+ * the ledger cannot vouch for them), so they sit until the disk fills.
11
+ *
12
+ * Measured on the reference Mac 2026-09-06, not reasoned about: 251 objects and
13
+ * 1,916.9 MB uncommitted against a 2,048 MB cap, 234 of them (1,835.8 MB)
14
+ * carrying `reconciled_unknown_to_server` — the SERVER was asked and answered
15
+ * that it has never held those hashes. `raw-evidence-staging.json` held six
16
+ * delivery-attempt rows in total, so those objects had not failed delivery;
17
+ * they had never been offered. The oldest pack was written 2026-07-23, 45 days
18
+ * earlier, and `cockpit doctor`'s fix sentence said "run `cockpit sync` to
19
+ * deliver them" — a sentence that could not come true.
20
+ *
21
+ * The repair is possible because each pack's own `manifest.json` already
22
+ * carries everything an offer needs: `object_key`, `content_hash_sha256`,
23
+ * `byte_size`, `media_type` and `redacted_summary` per file, and the pack's
24
+ * `work_context_id` / `session_id` / `operator_id` / `repo_basename` / `branch`
25
+ * beside them. This module rebuilds a `RawEvidenceUploadFile` from those
26
+ * recorded facts and NEVER re-derives them: the bytes on disk were sanitized
27
+ * when they were staged, and re-reading the original source would both cost the
28
+ * IO and risk offering content the redactor of the day would have masked
29
+ * differently.
30
+ *
31
+ * Metadata only, as everywhere in this family: pack ids, pack-relative paths,
32
+ * hashes, byte counts, reason labels. Never content, never an operator path.
33
+ */
34
+ import fs from "node:fs/promises";
35
+ import path from "node:path";
36
+ import { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./adapters/raw-evidence-manifest.js";
37
+ import { RAW_EVIDENCE_DIR } from "./disk-usage.js";
38
+ import { describeError } from "./health-detail.js";
39
+ import { LOCAL_COLLECTOR_VERSION } from "./local-state.js";
40
+ import { deliveryHold, } from "./raw-evidence-staging.js";
41
+ /**
42
+ * How much undelivered backlog one tick may re-offer.
43
+ *
44
+ * A tick is fifteen minutes and the backlog is measured in gigabytes, so this
45
+ * is a drain rate, not a ceiling on what will eventually be sent: 512 MB a tick
46
+ * clears the reference Mac's 1.9 GB in four ticks (one hour) without competing
47
+ * with the tick's own collection, which owns a separate 2 GB budget. The object
48
+ * count bounds the number of `begin` round-trips for the opposite case — a
49
+ * backlog of thousands of small objects.
50
+ */
51
+ export const REDELIVERY_MAX_BYTES_PER_TICK = 512 * 1024 * 1024;
52
+ export const REDELIVERY_MAX_OBJECTS_PER_TICK = 64;
53
+ /**
54
+ * Which undelivered objects this tick will re-offer, oldest bytes first.
55
+ *
56
+ * Oldest first is deliberate: the objects that have been undelivered longest
57
+ * are the ones most likely to be the only remaining copy of a session whose
58
+ * source transcript has since been rotated away by its agent host, and they are
59
+ * also the ones the disk cap will start refusing to keep. A newest-first drain
60
+ * would leave a 45-day-old orphan permanently at the back of the queue.
61
+ *
62
+ * `unknown` objects are deliberately NOT planned. "The ledger cannot say" is
63
+ * `cockpit clean --reconcile`'s question, it already runs one bounded batch per
64
+ * tick ahead of this, and it turns each answer into `committed` or
65
+ * `uncommitted` — so an unknown object reaches this planner on a later tick,
66
+ * with the server's own answer behind it, rather than being uploaded again on a
67
+ * guess.
68
+ */
69
+ export async function planEvidenceRedelivery(options) {
70
+ const plan = {
71
+ packs: [],
72
+ object_count: 0,
73
+ bytes: 0,
74
+ held_count: 0,
75
+ held_bytes: 0,
76
+ unplannable: {},
77
+ deferred_count: 0,
78
+ deferred_bytes: 0,
79
+ };
80
+ const maxBytes = options.maxBytes ?? REDELIVERY_MAX_BYTES_PER_TICK;
81
+ const maxObjects = options.maxObjects ?? REDELIVERY_MAX_OBJECTS_PER_TICK;
82
+ const root = path.join(options.stateDir, RAW_EVIDENCE_DIR);
83
+ const candidates = uncommittedOldestFirst(options.inventory);
84
+ const manifests = new Map();
85
+ const packPlans = new Map();
86
+ for (const object of candidates) {
87
+ if (!object.content_hash) {
88
+ countUnplannable(plan, "no_content_hash");
89
+ continue;
90
+ }
91
+ if (deliveryHold(options.staging, object.content_hash, options.now)) {
92
+ plan.held_count += 1;
93
+ plan.held_bytes += object.byte_size;
94
+ continue;
95
+ }
96
+ if (plan.object_count >= maxObjects ||
97
+ plan.bytes + object.byte_size > maxBytes) {
98
+ plan.deferred_count += 1;
99
+ plan.deferred_bytes += object.byte_size;
100
+ continue;
101
+ }
102
+ if (!manifests.has(object.pack_id)) {
103
+ manifests.set(object.pack_id, await readPackManifest(path.join(root, object.pack_id), object.pack_id));
104
+ }
105
+ const manifest = manifests.get(object.pack_id) ?? null;
106
+ if (!manifest) {
107
+ countUnplannable(plan, "manifest_unreadable");
108
+ continue;
109
+ }
110
+ const row = manifest.files.get(object.relative_path);
111
+ if (!row) {
112
+ countUnplannable(plan, "no_manifest_row");
113
+ continue;
114
+ }
115
+ let packPlan = packPlans.get(object.pack_id);
116
+ if (!packPlan) {
117
+ packPlan = {
118
+ pack_id: object.pack_id,
119
+ provenance: manifest.provenance,
120
+ files: [],
121
+ bytes: 0,
122
+ };
123
+ packPlans.set(object.pack_id, packPlan);
124
+ plan.packs.push(packPlan);
125
+ }
126
+ packPlan.files.push({
127
+ pointer: {
128
+ raw_evidence_pointer_id: row.object_key,
129
+ privacy_classification: "remote_durable_raw_evidence",
130
+ retention_policy: {
131
+ mode: RAW_EVIDENCE_RETENTION_MODE,
132
+ privacy_classification: "remote_durable_raw_evidence",
133
+ },
134
+ storage_scope: "remote_object",
135
+ storage_bucket: RAW_EVIDENCE_BUCKET,
136
+ object_key: row.object_key,
137
+ content_hash_sha256: row.content_hash_sha256,
138
+ byte_size: row.byte_size,
139
+ media_type: row.media_type,
140
+ redacted_summary: row.redacted_summary,
141
+ },
142
+ local_path: path.join(root, object.pack_id, ...object.relative_path.split("/").filter(Boolean)),
143
+ kind: row.kind,
144
+ codex_session_id: null,
145
+ });
146
+ packPlan.bytes += object.byte_size;
147
+ plan.object_count += 1;
148
+ plan.bytes += object.byte_size;
149
+ }
150
+ return plan;
151
+ }
152
+ /**
153
+ * Every uncommitted object, oldest staged bytes first.
154
+ *
155
+ * Sorted on `disk_age_ms` (the file's own mtime) rather than `age_ms`, because
156
+ * `age_ms` for a reconciled object is the age of the reconcile ANSWER — the
157
+ * same confusion that had 45-day-old orphans reading as one day old.
158
+ */
159
+ function uncommittedOldestFirst(inventory) {
160
+ return inventory.packs
161
+ .flatMap((pack) => pack.objects)
162
+ .filter((object) => object.state === "uncommitted")
163
+ .sort((left, right) => (right.disk_age_ms || right.age_ms) - (left.disk_age_ms || left.age_ms));
164
+ }
165
+ function countUnplannable(plan, reason) {
166
+ plan.unplannable[reason] = (plan.unplannable[reason] ?? 0) + 1;
167
+ }
168
+ /**
169
+ * A pack's recorded provenance and file rows.
170
+ *
171
+ * Every field is read back from what the collector WROTE when it staged the
172
+ * pack; nothing is re-derived from the machine's current state. A pack staged
173
+ * six weeks ago under a work context that no longer exists still knows which
174
+ * repo, branch, operator and work context it belonged to, and that is the only
175
+ * honest label for those bytes.
176
+ */
177
+ async function readPackManifest(dir, packId) {
178
+ const raw = await fs
179
+ .readFile(path.join(dir, "manifest.json"), "utf8")
180
+ .catch(() => null);
181
+ if (!raw)
182
+ return null;
183
+ try {
184
+ const parsed = JSON.parse(raw);
185
+ const provenance = provenanceFromManifest(parsed, packId);
186
+ if (!provenance)
187
+ return null;
188
+ const files = new Map();
189
+ for (const entry of asArray(parsed["files"])) {
190
+ const row = manifestFileRow(entry);
191
+ if (row)
192
+ files.set(row.relative_path, row.value);
193
+ }
194
+ return { provenance, files };
195
+ }
196
+ catch (error) {
197
+ console.error("[evidence-redelivery] pack manifest unreadable; its objects cannot be re-offered", JSON.stringify({
198
+ reason: "manifest_unreadable",
199
+ pack_id: packId,
200
+ ...describeError(error),
201
+ }));
202
+ return null;
203
+ }
204
+ }
205
+ function provenanceFromManifest(manifest, packId) {
206
+ const workContextId = asText(manifest["work_context_id"]);
207
+ const sessionId = asText(manifest["session_id"]);
208
+ const operatorId = asText(manifest["operator_id"]);
209
+ const repo = asText(manifest["repo_basename"]) ?? asText(manifest["repo_label"]);
210
+ if (!workContextId || !sessionId || !operatorId || !repo) {
211
+ console.error("[evidence-redelivery] pack manifest names no work context; its objects cannot be re-offered", JSON.stringify({
212
+ reason: "manifest_provenance_incomplete",
213
+ pack_id: packId,
214
+ has_work_context: Boolean(workContextId),
215
+ has_session: Boolean(sessionId),
216
+ has_operator: Boolean(operatorId),
217
+ has_repo: Boolean(repo),
218
+ }));
219
+ return null;
220
+ }
221
+ const repoLabel = asText(manifest["repo_label"]);
222
+ const worktreeLabel = asText(manifest["worktree_label"]);
223
+ return {
224
+ capture_source: "collector_runtime",
225
+ // The version that is re-offering, not the one that staged: this IS a
226
+ // delivery by today's collector, and pretending otherwise would misreport
227
+ // which build put the object in the bucket.
228
+ capture_adapter_version: LOCAL_COLLECTOR_VERSION,
229
+ collector_version: LOCAL_COLLECTOR_VERSION,
230
+ repo,
231
+ // A pack staged on a detached HEAD or a deleted worktree can have no branch
232
+ // recorded. `unknown` is a label, not a guess at which branch it was.
233
+ branch: asText(manifest["branch"]) ?? "unknown",
234
+ ...(repoLabel ? { repo_label: repoLabel } : {}),
235
+ ...(worktreeLabel ? { worktree_label: worktreeLabel } : {}),
236
+ operator_id: operatorId,
237
+ session_id: sessionId,
238
+ work_context_id: workContextId,
239
+ };
240
+ }
241
+ function manifestFileRow(entry) {
242
+ if (!entry || typeof entry !== "object")
243
+ return null;
244
+ const row = entry;
245
+ const relativePath = asText(row["relative_path"]);
246
+ const objectKey = asText(row["object_key"]);
247
+ const hash = asText(row["content_hash_sha256"]);
248
+ const mediaType = asText(row["media_type"]);
249
+ const summary = asText(row["redacted_summary"]);
250
+ const byteSize = row["byte_size"];
251
+ if (!relativePath ||
252
+ !objectKey ||
253
+ !hash ||
254
+ hash.length !== 64 ||
255
+ !mediaType ||
256
+ !summary ||
257
+ typeof byteSize !== "number") {
258
+ return null;
259
+ }
260
+ return {
261
+ relative_path: relativePath,
262
+ value: {
263
+ object_key: objectKey,
264
+ content_hash_sha256: hash,
265
+ byte_size: byteSize,
266
+ media_type: mediaType,
267
+ redacted_summary: summary,
268
+ kind: asText(row["kind"]) ?? "unknown",
269
+ },
270
+ };
271
+ }
272
+ function asArray(value) {
273
+ return Array.isArray(value) ? value : [];
274
+ }
275
+ function asText(value) {
276
+ return typeof value === "string" && value.trim() ? value : null;
277
+ }
@@ -0,0 +1,280 @@
1
+ /**
2
+ * Offering undelivered staged evidence again, on the next tick and every tick
3
+ * after, until it lands or names why it cannot.
4
+ *
5
+ * BLI-3797. `evidence-redelivery-plan.ts` says WHICH bytes and rebuilds the
6
+ * offer; this file spends it against the same chunked upload client the live
7
+ * sync uses, writes each success into the commit ledger and each failure into
8
+ * the delivery-attempt history (so the existing 15-min → 6-h backoff applies
9
+ * and one doomed object is not re-offered 96 times a day), and says what
10
+ * happened on every branch.
11
+ *
12
+ * Four properties, the same ones the reconcile and prune follow-ups keep:
13
+ *
14
+ * - **It never throws and never fails a tick.** A drain that cannot run is a
15
+ * log line and a receipt, never a reason collection did not happen.
16
+ * - **It never deletes.** Nothing here removes a staged byte; the prune that
17
+ * follows it decides that, and it only ever takes what the ledger vouches
18
+ * for — which now includes whatever this just delivered.
19
+ * - **It is bounded.** One drain rate per tick (see the plan module), so the
20
+ * backlog clears in hours without competing with the tick's own collection.
21
+ * - **It says so on both branches.** A tick with nothing undelivered is silent
22
+ * (the steady state, 96 times a day); every other outcome, including a clean
23
+ * drain, gets one `[evidence-redelivery]` line with counts and reasons.
24
+ */
25
+ import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
26
+ import { planEvidenceRedelivery, } from "./evidence-redelivery-plan.js";
27
+ import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
28
+ import { readStagingInventory } from "./disk-usage.js";
29
+ import { describeError } from "./health-detail.js";
30
+ import { getCollectorRuntimePaths, readLocalCollectorSessionFile, } from "./local-state.js";
31
+ import { clearDeliveryAttempt, readRawEvidenceStagingState, recordDeliveryFailure, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
32
+ /**
33
+ * The drain the sync tick runs and `cockpit doctor`'s disk fix reuses.
34
+ * Never throws.
35
+ */
36
+ export async function runEvidenceRedelivery(options) {
37
+ const env = options.env ?? process.env;
38
+ const now = options.now ?? new Date();
39
+ if (env["COCKPIT_DISABLE_EVIDENCE_REDELIVERY"] === "1") {
40
+ return emptyResult("disabled", "skipped");
41
+ }
42
+ const paths = getCollectorRuntimePaths(options.homeDir);
43
+ try {
44
+ const inventory = options.inventory ?? (await readStagingInventory(paths, now));
45
+ if (inventory.uncommitted_count === 0) {
46
+ // The steady state, 96 ticks a day. Saying so every time would bury the
47
+ // ticks that matter; `cockpit doctor`'s disk row still says it out loud
48
+ // to a person who asks.
49
+ return emptyResult("nothing_uncommitted", "ok");
50
+ }
51
+ const staging = await readRawEvidenceStagingState(paths.state_dir);
52
+ const plan = await planEvidenceRedelivery({
53
+ stateDir: paths.state_dir,
54
+ inventory,
55
+ staging,
56
+ now,
57
+ ...(options.maxBytes != null ? { maxBytes: options.maxBytes } : {}),
58
+ ...(options.maxObjects != null ? { maxObjects: options.maxObjects } : {}),
59
+ });
60
+ if (plan.object_count === 0) {
61
+ const result = emptyPlanResult(plan);
62
+ reportRedelivery(result, plan);
63
+ return result;
64
+ }
65
+ const session = await readLocalCollectorSessionFile(paths).catch(() => null);
66
+ if (!session ||
67
+ session.session_state !== "valid" ||
68
+ typeof session.device_token !== "string" ||
69
+ !session.device_token) {
70
+ // Not a failure of the backlog: this machine simply cannot speak to the
71
+ // dashboard right now. The bytes stay, the reason is named, and the next
72
+ // tick after a `cockpit login` picks them straight back up.
73
+ const result = {
74
+ ...emptyPlanResult(plan),
75
+ status: "skipped",
76
+ reason: "no_device_session",
77
+ };
78
+ reportRedelivery(result, plan);
79
+ return result;
80
+ }
81
+ const outcome = await deliverPlan({
82
+ plan,
83
+ dashboardUrl: options.dashboardUrl,
84
+ deviceToken: session.device_token,
85
+ fetchImpl: options.fetch ?? fetch,
86
+ now,
87
+ });
88
+ await recordOutcomes(paths, staging, outcome.outcomes, now);
89
+ const result = {
90
+ status: outcome.failed > 0 && outcome.uploaded === 0 ? "fail" : "ok",
91
+ reason: "redelivered",
92
+ offered: plan.object_count,
93
+ offered_bytes: plan.bytes,
94
+ uploaded: outcome.uploaded,
95
+ uploaded_bytes: outcome.uploadedBytes,
96
+ reused: outcome.reused,
97
+ failed: outcome.failed,
98
+ held: plan.held_count,
99
+ deferred: plan.deferred_count,
100
+ deferred_bytes: plan.deferred_bytes,
101
+ packs: plan.packs.length,
102
+ failure_reasons: outcome.failureReasons,
103
+ };
104
+ reportRedelivery(result, plan);
105
+ return result;
106
+ }
107
+ catch (error) {
108
+ console.error("[evidence-redelivery] the drain could not run; nothing was re-offered", JSON.stringify({ reason: "redelivery_threw", ...describeError(error) }));
109
+ return { ...emptyResult("redelivery_threw", "fail") };
110
+ }
111
+ }
112
+ /**
113
+ * One `uploadRawEvidenceFilesChunked` call per pack, because provenance is a
114
+ * PACK fact: the repo, branch, operator, session and work context these bytes
115
+ * belonged to were recorded when they were staged, and sending one pack's
116
+ * objects under another pack's labels would file six-week-old evidence against
117
+ * today's ticket.
118
+ */
119
+ async function deliverPlan(options) {
120
+ const result = {
121
+ outcomes: [],
122
+ uploaded: 0,
123
+ uploadedBytes: 0,
124
+ reused: 0,
125
+ failed: 0,
126
+ failureReasons: [],
127
+ };
128
+ for (const pack of options.plan.packs) {
129
+ const uploaded = await deliverOnePack(pack, options);
130
+ for (const outcome of uploaded) {
131
+ result.outcomes.push(outcome);
132
+ if (outcome.upload_state === "uploaded") {
133
+ result.uploaded += 1;
134
+ result.uploadedBytes += outcome.pointer.byte_size ?? 0;
135
+ continue;
136
+ }
137
+ if (outcome.upload_state === "reused_existing") {
138
+ result.reused += 1;
139
+ continue;
140
+ }
141
+ result.failed += 1;
142
+ const reason = outcome.reason ?? "unknown";
143
+ if (!result.failureReasons.includes(reason)) {
144
+ result.failureReasons.push(reason);
145
+ }
146
+ }
147
+ }
148
+ return result;
149
+ }
150
+ async function deliverOnePack(pack, options) {
151
+ try {
152
+ const uploaded = await uploadRawEvidenceFilesChunked({
153
+ fetchImpl: options.fetchImpl,
154
+ dashboardUrl: options.dashboardUrl,
155
+ deviceToken: options.deviceToken,
156
+ provenance: pack.provenance,
157
+ generatedAt: options.now.toISOString(),
158
+ files: pack.files,
159
+ });
160
+ return uploaded.outcomes;
161
+ }
162
+ catch (error) {
163
+ // The upload client already reports its own transport failures; this is the
164
+ // net for anything it does not, and it must not abandon the packs behind
165
+ // this one. Each file is reported failed so the backoff history records the
166
+ // attempt rather than re-offering all of them again in fifteen minutes.
167
+ console.error("[evidence-redelivery] a pack's re-offer threw; the rest of the drain continues", JSON.stringify({
168
+ reason: "pack_redelivery_threw",
169
+ pack_id: pack.pack_id,
170
+ object_count: pack.files.length,
171
+ ...describeError(error),
172
+ }));
173
+ return pack.files.map((file) => ({
174
+ pointer: file.pointer,
175
+ object_key: file.pointer.object_key ?? "",
176
+ codex_session_id: null,
177
+ kind: file.kind ?? "unknown",
178
+ upload_state: "upload_failed",
179
+ reason: "pack_redelivery_threw",
180
+ uploaded_chunk_count: 0,
181
+ }));
182
+ }
183
+ }
184
+ /**
185
+ * Write what happened where the next tick will read it: the commit ledger for
186
+ * anything that landed, the delivery-attempt history for anything that did not.
187
+ *
188
+ * Both writes happen even when the run partly failed, and the ledger is written
189
+ * FIRST — a delivered object whose ledger row was lost would be re-offered
190
+ * forever, which is the failure this whole module exists to end.
191
+ */
192
+ async function recordOutcomes(paths, staging, outcomes, now) {
193
+ const cursor = await readRawEvidenceCursor(paths);
194
+ let ledgerChanged = false;
195
+ let stagingChanged = false;
196
+ for (const outcome of outcomes) {
197
+ const hash = outcome.pointer.content_hash_sha256;
198
+ if (!hash)
199
+ continue;
200
+ if (outcome.upload_state === "upload_failed") {
201
+ recordDeliveryFailure(staging, hash, {
202
+ reason: outcome.reason ?? "unknown",
203
+ attemptedAt: now,
204
+ byteSize: outcome.pointer.byte_size ?? 0,
205
+ });
206
+ stagingChanged = true;
207
+ continue;
208
+ }
209
+ markObjectCommitted(cursor, hash, {
210
+ object_key: outcome.object_key,
211
+ byte_size: outcome.pointer.byte_size ?? 0,
212
+ committed_at: now.toISOString(),
213
+ });
214
+ ledgerChanged = true;
215
+ if (clearDeliveryAttempt(staging, hash))
216
+ stagingChanged = true;
217
+ }
218
+ if (ledgerChanged)
219
+ await writeRawEvidenceCursor(paths, cursor);
220
+ if (stagingChanged) {
221
+ await writeRawEvidenceStagingState(paths.state_dir, staging);
222
+ }
223
+ }
224
+ /**
225
+ * One line per non-steady-state tick. Counts, byte totals and reason labels
226
+ * only — no pack id, no hash, no path, per the logging contract.
227
+ */
228
+ function reportRedelivery(result, plan) {
229
+ const fields = {
230
+ reason: result.reason,
231
+ status: result.status,
232
+ offered: result.offered,
233
+ offered_bytes: result.offered_bytes,
234
+ uploaded: result.uploaded,
235
+ uploaded_bytes: result.uploaded_bytes,
236
+ reused: result.reused,
237
+ failed: result.failed,
238
+ held_by_backoff: result.held,
239
+ deferred_to_next_tick: result.deferred,
240
+ deferred_bytes: result.deferred_bytes,
241
+ packs: result.packs,
242
+ unplannable: plan.unplannable,
243
+ failure_reasons: result.failure_reasons,
244
+ };
245
+ if (result.status === "fail") {
246
+ console.error("[evidence-redelivery] every re-offered object was refused", JSON.stringify(fields));
247
+ return;
248
+ }
249
+ console.error(result.uploaded > 0
250
+ ? "[evidence-redelivery] undelivered staged evidence landed"
251
+ : "[evidence-redelivery] undelivered staged evidence was not re-offered this tick", JSON.stringify(fields));
252
+ }
253
+ function emptyPlanResult(plan) {
254
+ const nothingPlannable = plan.held_count > 0 && plan.deferred_count === 0
255
+ ? "all_held_by_backoff"
256
+ : "nothing_plannable";
257
+ return {
258
+ ...emptyResult(nothingPlannable, "ok"),
259
+ held: plan.held_count,
260
+ deferred: plan.deferred_count,
261
+ deferred_bytes: plan.deferred_bytes,
262
+ };
263
+ }
264
+ function emptyResult(reason, status) {
265
+ return {
266
+ status,
267
+ reason,
268
+ offered: 0,
269
+ offered_bytes: 0,
270
+ uploaded: 0,
271
+ uploaded_bytes: 0,
272
+ reused: 0,
273
+ failed: 0,
274
+ held: 0,
275
+ deferred: 0,
276
+ deferred_bytes: 0,
277
+ packs: 0,
278
+ failure_reasons: [],
279
+ };
280
+ }
@@ -1,5 +1,7 @@
1
1
  import os from "node:os";
2
2
  import path from "node:path";
3
+ import { readStagingInventory } from "./disk-usage.js";
4
+ import { describeError } from "./health-detail.js";
3
5
  import { summarizeLocalUploadSpool } from "./spool/local-spool.js";
4
6
  import { summarizeInstallEventOutbox } from "./spool/install-event-outbox.js";
5
7
  import { readRawEvidenceStagingState, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
@@ -55,6 +57,10 @@ export async function inspectLocalCollectorStatus(options = {}) {
55
57
  stuck_evidence_max_attempts: stuckEvidence.max_attempts,
56
58
  stuck_evidence_oldest_failure_at: stuckEvidence.oldest_first_failed_at,
57
59
  stuck_evidence_reasons: stuckEvidence.reasons,
60
+ staging_uncommitted_count: reading.staging?.uncommitted_count ?? null,
61
+ staging_uncommitted_bytes: reading.staging?.uncommitted_bytes ?? null,
62
+ staging_uncommitted_reason: reading.staging?.dominant_uncommitted_reason?.reason ?? null,
63
+ staging_oldest_uncommitted_at: reading.staging?.oldest_uncommitted_at ?? null,
58
64
  details: describeCollectorStatus(reading),
59
65
  };
60
66
  }
@@ -76,7 +82,20 @@ async function readCollectorStatus(options, now) {
76
82
  readRawEvidenceStagingState(paths.state_dir),
77
83
  ]);
78
84
  const stuckEvidence = summarizeStuckEvidence(stagingState, now);
85
+ // Best-effort by construction: a status read must never fail because the
86
+ // staging tree could not be walked, and an absent backlog reads as "this read
87
+ // did not ask", never as "there is none".
88
+ const staging = options.includeStagingBacklog
89
+ ? await readStagingInventory(paths, now).catch((error) => {
90
+ console.error("[collector status] the staging tree could not be read; the undelivered backlog is unknown", JSON.stringify({
91
+ reason: "staging_inventory_unreadable",
92
+ ...describeError(error),
93
+ }));
94
+ return null;
95
+ })
96
+ : null;
79
97
  return {
98
+ staging,
80
99
  paths,
81
100
  config,
82
101
  session,
@@ -134,6 +153,18 @@ function describeCollectorStatus(reading) {
134
153
  if (healthOutbox.pending_count > 0) {
135
154
  details.push(`Collector health retry pending: ${healthOutbox.pending_count} sanitized receipt(s) queued since ${healthOutbox.oldest_created_at ?? "unknown"}.`);
136
155
  }
156
+ // BLI-3797: the line above only sees objects whose delivery FAILED. This one
157
+ // sees evidence that is on this disk and not in the bucket, however it got
158
+ // that way — which is the only question that answers "has everything landed?".
159
+ if (reading.staging && reading.staging.uncommitted_count > 0) {
160
+ const dominant = reading.staging.dominant_uncommitted_reason;
161
+ details.push(`Staged evidence not yet accepted: ${reading.staging.uncommitted_count} object(s), ${mib(reading.staging.uncommitted_bytes)} MB, oldest staged ${reading.staging.oldest_uncommitted_at ?? "unknown"}${dominant
162
+ ? ` — biggest reason ${dominant.reason} on ${dominant.count} object(s)`
163
+ : ""}. \`cockpit sync\` re-offers them each tick; \`cockpit doctor --fix\` does it now.`);
164
+ }
165
+ else if (reading.staging) {
166
+ details.push("Staged evidence: every object on this disk is accepted.");
167
+ }
137
168
  if (stuckEvidence.stuck_object_count > 0) {
138
169
  details.push(`Raw evidence stuck: ${stuckEvidence.stuck_object_count} object(s) have never been accepted (${stuckEvidence.held_object_count} waiting on backoff, worst ${stuckEvidence.max_attempts} attempt(s) since ${stuckEvidence.oldest_first_failed_at ?? "unknown"}) — reasons: ${stuckEvidence.reasons.join(", ") || "unknown"}.`);
139
170
  }
@@ -155,6 +186,9 @@ export function classifyCollectorFreshness(context, lastUploadSuccessAt, now) {
155
186
  return "stale";
156
187
  return now.getTime() - latestActivityAt <= 5 * 60 * 1000 ? "fresh" : "stale";
157
188
  }
189
+ function mib(bytes) {
190
+ return (bytes / (1024 * 1024)).toFixed(1);
191
+ }
158
192
  function latestTimestamp(values) {
159
193
  const timestamps = values
160
194
  .map((value) => Date.parse(value ?? ""))
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.70",
3
+ "version": "0.2.72",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -27,8 +27,8 @@
27
27
  "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
28
28
  },
29
29
  "dependencies": {
30
- "@bli-cockpit/memory-mcp": "0.1.10",
31
- "@bli-cockpit/mcp": "0.1.11",
32
- "@bli-cockpit/telemetry-core": "0.1.31"
30
+ "@bli-cockpit/memory-mcp": "0.1.11",
31
+ "@bli-cockpit/mcp": "0.1.12",
32
+ "@bli-cockpit/telemetry-core": "0.1.32"
33
33
  }
34
34
  }